diff --git a/src/dstack/_internal/cli/services/presets/verify.py b/src/dstack/_internal/cli/services/presets/verify.py index 36fcda5fd..e0614f057 100644 --- a/src/dstack/_internal/cli/services/presets/verify.py +++ b/src/dstack/_internal/cli/services/presets/verify.py @@ -135,6 +135,46 @@ def _verified_run_service(run: Run, report: PresetAgentSuccess) -> ServiceConfig return service +#: Characters that may follow the base name in a variant repo. A variant adds a +#: suffix — a quantisation, a format, a precision — and these are what separate +#: it, so requiring one stops ``Qwen3.5-27B`` from accepting ``Qwen3.5-27Bx``. +_VARIANT_SUFFIX_SEPARATORS = ("-", "_", ".") + + +def _model_name(repo: str) -> str: + """The model half of a repo reference, without its owner. + + Deliberately owner-blind. A quantisation of a model is routinely published + by someone other than the original author — this repository's own fixtures + pair a ``Qwen/Qwen3.5-27B`` base with a ``community/Qwen3.5-27B-GPTQ-Int4`` + repo — so requiring the owner to match would reject the ordinary case. + """ + return repo.rsplit("/", 1)[-1].strip() + + +def _is_variant_of(repo: str, base: str) -> bool: + """Is ``repo`` a variant of ``base``, rather than a different model? + + A variant is the base plus a suffix: ``Qwen3.5-27B`` also answers to + ``Qwen3.5-27B-GPTQ-Int4`` and ``Qwen3.5-27B-AWQ``. A different generation + is not a variant, however similar the name — ``Qwen3.8-27B`` does not + answer a request for ``Qwen3.5-27B``, which is exactly the substitution + that verified clean before. + + Compared case-insensitively, and on the model name alone, so the check + stays about which model was served rather than about who published it. + """ + served = _model_name(repo).lower() + wanted = _model_name(base).lower() + if not served or not wanted: + return False + if served == wanted: + return True + if not served.startswith(wanted): + return False + return served[len(wanted)] in _VARIANT_SUFFIX_SEPARATORS + + def _check_report_answers_request( report: PresetAgentSuccess, configuration: PresetConfiguration ) -> None: @@ -145,6 +185,17 @@ def _check_report_answers_request( if configuration.model.allows_variant_selection: if report.base != configuration.model.api_model_name: raise CLIError("Claude final report base does not match the requested model") + # The base must constrain which repos are acceptable, and echoing it + # back does not: the served repo is what the agent chose, and it was + # only ever checked against the *advertised* name — which a + # substitution preserves. `vllm serve Qwen/Qwen3.5-27B-GPTQ-Int4 + # --served-model-name Qwen/Qwen3.8-27B` answered a request for + # Qwen3.8 with a different model generation and verified clean. + if not _is_variant_of(report.model, configuration.model.api_model_name): + raise CLIError( + f"Claude served {report.model!r}, which is not a variant of the requested" + f" base {configuration.model.api_model_name!r}" + ) elif report.model != configuration.model.exact_repo: raise CLIError("Claude changed an exact model request") diff --git a/src/tests/_internal/cli/services/presets/test_verify.py b/src/tests/_internal/cli/services/presets/test_verify.py index 26750c7b7..6214db1d0 100644 --- a/src/tests/_internal/cli/services/presets/test_verify.py +++ b/src/tests/_internal/cli/services/presets/test_verify.py @@ -304,6 +304,74 @@ def test_rejects_benchmark_on_a_different_dataset(self, tmp_path, reported): created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), ) + @pytest.mark.parametrize( + "served", + [ + "Qwen/Qwen3.8-27B-GPTQ-Int4", # the reported substitution: another generation + "Qwen/Qwen3.8-27B", + "meta-llama/Llama-3-8B", # another family entirely + "Qwen/Qwen3.5-27Bx", # a longer name that merely starts the same + ], + ) + def test_rejects_a_served_repo_that_is_not_a_variant_of_the_base(self, tmp_path, served): + """`base` has to constrain which repos are acceptable. + + Verification checked that the service advertised the requested name and + that the report echoed the requested base — both of which a substitution + preserves. `vllm serve Qwen/Qwen3.5-27B-GPTQ-Int4 --served-model-name + Qwen/Qwen3.8-27B` answered a request for Qwen3.8 with a different model + generation and verified clean. + """ + run = get_running_service_run() + report = get_successful_preset_report(run).model_copy(update={"model": served}) + + # Both values named, as elsewhere here: "not a variant" alone is not actionable. + with pytest.raises(CLIError, match="is not a variant of the requested base"): + build_verified_preset( + run=run, + preset_configuration=PresetConfiguration( + name="qwen-build", base="Qwen/Qwen3.5-27B" + ), + report=report, + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + @pytest.mark.parametrize( + "served", + [ + "community/Qwen3.5-27B-GPTQ-Int4", # a quantisation by another publisher + "Qwen/Qwen3.5-27B-AWQ", + "Qwen/Qwen3.5-27B", # the base itself + "qwen/qwen3.5-27b-gptq-int4", # repo references are not case-sensitive + ], + ) + def test_accepts_a_genuine_variant_of_the_base(self, tmp_path, served): + """The check must not cost the freedom `base` exists to grant. + + A quantisation is routinely published by someone other than the model's + author, so the owner is deliberately not compared — only the model name. + """ + run = get_running_service_run() + report = get_successful_preset_report(run).model_copy(update={"model": served}) + + preset = build_verified_preset( + run=run, + preset_configuration=PresetConfiguration(name="qwen-build", base="Qwen/Qwen3.5-27B"), + report=report, + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + assert preset.repo == served + assert preset.base == "Qwen/Qwen3.5-27B" + def test_rejects_variant_for_exact_model_request(self, tmp_path): run = get_running_service_run() report = get_successful_preset_report(run).model_copy(update={"model": "other/model"})