Skip to content

fix(distributed): reject wrong-model requests on the remaining modalities #10990

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mudler merged 1 commit into from
Jul 20, 2026
Merged

fix(distributed): reject wrong-model requests on the remaining modalities #10990

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 100 additions & 1 deletion backend/backend.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,10 @@ message MetricsResponse {
message TokenClassifyRequest {
string text = 1;
float threshold = 2;
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 3;
}

// TokenClassifyEntity is one detected entity span. Byte offsets are
Expand Down Expand Up @@ -173,6 +177,10 @@ message ScoreRequest {
// candidates differ in length and the consumer wants a per-token
// measure comparable across them (PMI-style scoring).
bool length_normalize = 4;
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 5;
}

// CandidateScore is one row in the ScoreResponse, matching by index
Expand Down Expand Up @@ -204,6 +212,10 @@ message RerankRequest {
string query = 1;
repeated string documents = 2;
int32 top_n = 3;
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 4;
}

message RerankResult {
Expand Down Expand Up @@ -336,7 +348,14 @@ message PredictOptions {
// paths (core/services/nodes/file_staging_client.go), so in distributed mode
// they already differ from the load-time value and comparing them would
// reject valid requests. Extending identity to those RPCs needs a separate
// field carrying the untranslated value.
// field carrying the untranslated value - which is exactly what
// TTSRequest.ModelIdentity and SoundGenerationRequest.ModelIdentity are.
//
// Every other request message that reaches a backend through the distributed
// router now carries the same ModelIdentity field, populated from the same
// ModelConfig.Model. FileStagingClient rewrites Src/Dst/Voice/Model/
// StartImage/EndImage/Audio and never ModelIdentity, so what the backend
// compares is always what the controller sent.
string ModelIdentity = 54;

// 24 was never assigned; reserve it so it is not silently reused.
Expand Down Expand Up @@ -510,6 +529,10 @@ message TranscriptRequest {
float temperature = 8;
repeated string timestamp_granularities = 9;
bool stream = 10;
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 11;
}

message TranscriptResult {
Expand Down Expand Up @@ -588,6 +611,10 @@ message GenerateImageRequest {

// Reference images for models that support them (e.g., Flux Kontext)
repeated string ref_images = 12;
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 13;
}

message GenerateVideoRequest {
Expand All @@ -607,6 +634,10 @@ message GenerateVideoRequest {
// Backend-specific per-request generation parameters. Values are strings
// and are validated/coerced by the selected backend.
map<string, string> params = 14;
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 15;
}

message TTSRequest {
Expand All @@ -624,10 +655,26 @@ message TTSRequest {
// (e.g. Chatterbox exaggeration/cfg_weight/temperature). Values are strings and
// coerced by the backend; unset leaves the backend's configured defaults.
map<string, string> params = 7;
// ModelIdentity is a SEPARATE field from `model` above and carries the
// UNTRANSLATED controller-side ModelConfig.Model, so a backend can reject a
// request that reached it through a stale distributed route (#10952).
//
// `model` cannot be reused for this: FileStagingClient.TTS/.TTSStream and the
// SoundGeneration path rewrite it into a worker-local absolute path
// (core/services/nodes/file_staging_client.go), while the load-time value is
// untranslated. In distributed mode - exactly the configuration this guards -
// the two already differ, so comparing them would reject valid requests.
//
// Empty means "no identity supplied" and backends MUST skip the check.
string ModelIdentity = 8;
}

message VADRequest {
repeated float audio = 1;
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 2;
}

message VADSegment {
Expand Down Expand Up @@ -659,6 +706,10 @@ message DiarizeRequest {
float min_duration_on = 8; // discard segments shorter than this (seconds); 0 = backend default
float min_duration_off = 9; // merge gaps shorter than this (seconds); 0 = backend default
bool include_text = 10; // when the backend can emit per-segment transcript for free, ask it to populate `text`
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 11;
}

message DiarizeSegment {
Expand Down Expand Up @@ -693,6 +744,18 @@ message SoundGenerationRequest {
optional string language = 14;
optional string timesignature = 15;
optional bool instrumental = 17;
// ModelIdentity is a SEPARATE field from `model` above and carries the
// UNTRANSLATED controller-side ModelConfig.Model, so a backend can reject a
// request that reached it through a stale distributed route (#10952).
//
// `model` cannot be reused for this: FileStagingClient.TTS/.TTSStream and the
// SoundGeneration path rewrite it into a worker-local absolute path
// (core/services/nodes/file_staging_client.go), while the load-time value is
// untranslated. In distributed mode - exactly the configuration this guards -
// the two already differ, so comparing them would reject valid requests.
//
// Empty means "no identity supplied" and backends MUST skip the check.
string ModelIdentity = 18;
}

message TokenizationResponse {
Expand Down Expand Up @@ -732,6 +795,10 @@ message DetectOptions {
repeated float points = 3; // Point coordinates as [x1, y1, label1, x2, y2, label2, ...] (label: 1=pos, 0=neg)
repeated float boxes = 4; // Box coordinates as [x1, y1, x2, y2, ...]
float threshold = 5; // Detection confidence threshold
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 6;
}

message Detection {
Expand All @@ -754,6 +821,10 @@ message SoundDetectionRequest {
string src = 1; // audio file path (LocalAI writes the upload to disk)
int32 top_k = 2; // number of top tags to return (0 = all classes)
float threshold = 3; // optional: drop tags scoring below this
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 4;
}

message SoundClass {
Expand All @@ -778,6 +849,10 @@ message DepthRequest {
bool include_points = 7; // back-project to a 3D point cloud (DualDPT)
float points_conf_thresh = 8; // keep points with confidence >= this threshold
repeated string exports = 9; // requested exports: "glb", "colmap"
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 10;
}

message DepthResponse {
Expand Down Expand Up @@ -809,6 +884,10 @@ message FaceVerifyRequest {
string img2 = 2; // base64-encoded image
float threshold = 3; // cosine-distance threshold; 0 = use backend default
bool anti_spoofing = 4; // run MiniFASNet liveness on each image; failed liveness forces verified=false
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 5;
}

message FaceVerifyResponse {
Expand All @@ -830,6 +909,10 @@ message FaceAnalyzeRequest {
string img = 1; // base64-encoded image
repeated string actions = 2; // subset of ["age","gender","emotion","race"]; empty = all-supported
bool anti_spoofing = 3;
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 4;
}

message FaceAnalysis {
Expand Down Expand Up @@ -862,6 +945,10 @@ message VoiceVerifyRequest {
string audio2 = 2; // path to second audio clip
float threshold = 3; // cosine-distance threshold; 0 = use backend default
bool anti_spoofing = 4; // reserved for future AASIST bolt-on
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 5;
}

message VoiceVerifyResponse {
Expand All @@ -876,6 +963,10 @@ message VoiceVerifyResponse {
message VoiceAnalyzeRequest {
string audio = 1; // path to audio clip
repeated string actions = 2; // subset of ["age","gender","emotion"]; empty = all-supported
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 3;
}

message VoiceAnalysis {
Expand All @@ -894,6 +985,10 @@ message VoiceAnalyzeResponse {

message VoiceEmbedRequest {
string audio = 1; // path to audio clip
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 2;
}

message VoiceEmbedResponse {
Expand Down Expand Up @@ -988,6 +1083,10 @@ message AudioTransformRequest {
string reference_path = 2; // optional auxiliary; empty => zero-fill
string dst = 3; // required, output file path
map<string, string> params = 4; // backend-specific tuning
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 5;
}

message AudioTransformResult {
Expand Down
14 changes: 13 additions & 1 deletion backend/cpp/llama-cpp/grpc-server.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1416,7 +1416,11 @@ class BackendServiceImpl final : public backend::Backend::Service {
// field and for the synthetic PredictOptions this server builds internally
// for ASR, and the loaded side is empty when such a controller performed
// the load. A false rejection is worse than the miss it prevents.
grpc::Status checkModelIdentity(const backend::PredictOptions* request) {
// Templated over the request type: every guarded request message exposes
// modelidentity(), and one body keeps the rule identical across modalities
// rather than repeating it per RPC.
template <typename Request>
grpc::Status checkModelIdentity(const Request* request) {
if (request == nullptr || request->modelidentity().empty()) {
return grpc::Status::OK;
}
Expand Down Expand Up @@ -2846,6 +2850,8 @@ class BackendServiceImpl final : public backend::Backend::Service {
}

grpc::Status Rerank(ServerContext* context, const backend::RerankRequest* request, backend::RerankResult* rerankResult) override {
auto identity = checkModelIdentity(request);
if (!identity.ok()) return identity;
if (!params_base.embedding || params_base.pooling_type != LLAMA_POOLING_TYPE_RANK) {
return grpc::Status(grpc::StatusCode::UNIMPLEMENTED, "This server does not support reranking. Start it with `--reranking` and without `--embedding`");
}
Expand Down Expand Up @@ -2970,6 +2976,8 @@ class BackendServiceImpl final : public backend::Backend::Service {
grpc::Status Score(ServerContext* context, const backend::ScoreRequest* request, backend::ScoreResponse* response) override {
auto auth = checkAuth(context);
if (!auth.ok()) return auth;
auto identity = checkModelIdentity(request);
if (!identity.ok()) return identity;
if (params_base.model.path.empty()) {
return grpc::Status(grpc::StatusCode::FAILED_PRECONDITION, "Model not loaded");
}
Expand Down Expand Up @@ -3428,6 +3436,8 @@ class BackendServiceImpl final : public backend::Backend::Service {
backend::TranscriptResult* response) override {
auto auth = checkAuth(context);
if (!auth.ok()) return auth;
auto identity = checkModelIdentity(request);
if (!identity.ok()) return identity;

backend::Reply reply;
grpc::Status st = runTranscriptionAsCompletion(context, request, &reply);
Expand All @@ -3446,6 +3456,8 @@ class BackendServiceImpl final : public backend::Backend::Service {
grpc::ServerWriter<backend::TranscriptStreamResponse>* writer) override {
auto auth = checkAuth(context);
if (!auth.ok()) return auth;
auto identity = checkModelIdentity(request);
if (!identity.ok()) return identity;

// Buffered streaming: run the transcription as a normal chat
// completion, then emit one delta + one final event. Real
Expand Down
36 changes: 36 additions & 0 deletions backend/cpp/privacy-filter/grpc-server.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ namespace {
// per loaded model. g_mu guards (re)load against in-flight classification.
std::mutex g_mu;
pf_ctx * g_ctx = nullptr;
// The ModelOptions.Model this process loaded, compared against
// TokenClassifyRequest.ModelIdentity so a request that arrived through a stale
// distributed route is rejected rather than answered from the wrong model
// (#10952). Guarded by g_mu like the rest of the engine state.
std::string g_loaded_model_identity;
std::atomic<Server *> g_server{nullptr};

// Resolve the device string the engine expects ("cpu" / "gpu" / "cuda" /
Expand Down Expand Up @@ -113,17 +118,48 @@ class PrivacyFilterBackend final : public backend::Backend::Service {
}

g_ctx = ctx;
// Record what we loaded so TokenClassify can reject a request meant
// for a different model. request->model(), not modelfile(): it is the
// value the controller also sends as ModelIdentity, and the two are
// read from the same ModelConfig.Model (#10952).
g_loaded_model_identity = request->model();
result->set_success(true);
result->set_message("privacy-filter loaded (" + device + ")");
return GStatus::OK;
}

// checkModelIdentity mirrors pkg/grpc/server.go,
// backend/python/common/model_identity.py and the llama-cpp server. In
// distributed mode a worker can recycle a stopped backend's gRPC port for
// another model's backend, and the controller's liveness-only probe cannot
// tell a stale cached route from a valid one, so the backend has to catch
// it. Either side empty means "skip": the request side is empty for a
// controller that predates the field, the loaded side when such a
// controller performed the load. A false rejection is worse than the miss.
// Callers must already hold g_mu.
GStatus checkModelIdentity(const backend::TokenClassifyRequest * request) {
if (request == nullptr || request->modelidentity().empty()) {
return GStatus::OK;
}
if (g_loaded_model_identity.empty() ||
g_loaded_model_identity == request->modelidentity()) {
return GStatus::OK;
}
// NOT_FOUND plus this exact sentinel is the cross-language contract
// the router matches on (grpcerrors.ModelMismatchSentinel).
return GStatus(StatusCode::NOT_FOUND,
"privacy-filter: model identity mismatch: loaded \"" +
g_loaded_model_identity + "\", requested \"" +
request->modelidentity() + "\"");
}

GStatus TokenClassify(ServerContext *, const backend::TokenClassifyRequest * request,
backend::TokenClassifyResponse * response) override {
std::lock_guard<std::mutex> lock(g_mu);
if (!g_ctx) {
return GStatus(StatusCode::FAILED_PRECONDITION, "Model not loaded");
}
if (GStatus id = checkModelIdentity(request); !id.ok()) return id;

const std::string & text = request->text();
if (text.empty()) {
Expand Down
Loading
Loading