fix(presets): enforce that the served repo is a variant of the requested base

A preset declaring `base: X` was verified against two things that a
substitution preserves: that the service advertises the requested name,
and that the report echoes the requested base. Neither looks at the repo
the agent actually served.

So `vllm serve Qwen/Qwen3.5-27B-GPTQ-Int4 --served-model-name
Qwen/Qwen3.8-27B` answered a request for Qwen3.8 with a different model
generation, verified clean, and was reported successful.

`base` is documented as "the base model for which the agent may select a
compatible variant", so verification now checks compatibility: the
served repo must be the base, or the base plus a suffix at a separator
boundary. A different generation is not a variant however similar the
name.

The comparison is on the model name alone, ignoring the owner. A
quantisation is routinely published by someone other than the model's
author -- this repository's own fixtures pair a Qwen/Qwen3.5-27B base
with a community/Qwen3.5-27B-GPTQ-Int4 repo -- so comparing owners would
reject the ordinary case and cost `base` the freedom it exists to grant.
It is also case-insensitive, since repo references are.

Eight tests: four substitutions that must be rejected, including the
reported one and a name that merely starts the same, and four genuine
variants that must still pass, including the third-party quantisation
and a lowercased reference.

Author: vaibhav8a

src/dstack/_internal/cli/services/presets/verify.py

@@ -135,6 +135,46 @@ def _verified_run_service(run: Run, report: PresetAgentSuccess) -> ServiceConfig
     return service
 
 
+#: Characters that may follow the base name in a variant repo. A variant adds a
+#: suffix — a quantisation, a format, a precision — and these are what separate
+#: it, so requiring one stops ``Qwen3.5-27B`` from accepting ``Qwen3.5-27Bx``.
+_VARIANT_SUFFIX_SEPARATORS = ("-", "_", ".")
+
+
+def _model_name(repo: str) -> str:
+    """The model half of a repo reference, without its owner.
+
+    Deliberately owner-blind. A quantisation of a model is routinely published
+    by someone other than the original author — this repository's own fixtures
+    pair a ``Qwen/Qwen3.5-27B`` base with a ``community/Qwen3.5-27B-GPTQ-Int4``
+    repo — so requiring the owner to match would reject the ordinary case.
+    """
+    return repo.rsplit("/", 1)[-1].strip()
+
+
+def _is_variant_of(repo: str, base: str) -> bool:
+    """Is ``repo`` a variant of ``base``, rather than a different model?
+
+    A variant is the base plus a suffix: ``Qwen3.5-27B`` also answers to
+    ``Qwen3.5-27B-GPTQ-Int4`` and ``Qwen3.5-27B-AWQ``. A different generation
+    is not a variant, however similar the name — ``Qwen3.8-27B`` does not
+    answer a request for ``Qwen3.5-27B``, which is exactly the substitution
+    that verified clean before.
+
+    Compared case-insensitively, and on the model name alone, so the check
+    stays about which model was served rather than about who published it.
+    """
+    served = _model_name(repo).lower()
+    wanted = _model_name(base).lower()
+    if not served or not wanted:
+        return False
+    if served == wanted:
+        return True
+    if not served.startswith(wanted):
+        return False
+    return served[len(wanted)] in _VARIANT_SUFFIX_SEPARATORS
+
+
 def _check_report_answers_request(
     report: PresetAgentSuccess, configuration: PresetConfiguration
 ) -> None:
@@ -145,6 +185,17 @@ def _check_report_answers_request(
     if configuration.model.allows_variant_selection:
         if report.base != configuration.model.api_model_name:
             raise CLIError("Claude final report base does not match the requested model")
+        # The base must constrain which repos are acceptable, and echoing it
+        # back does not: the served repo is what the agent chose, and it was
+        # only ever checked against the *advertised* name — which a
+        # substitution preserves. `vllm serve Qwen/Qwen3.5-27B-GPTQ-Int4
+        # --served-model-name Qwen/Qwen3.8-27B` answered a request for
+        # Qwen3.8 with a different model generation and verified clean.
+        if not _is_variant_of(report.model, configuration.model.api_model_name):
+            raise CLIError(
+                f"Claude served {report.model!r}, which is not a variant of the requested"
+                f" base {configuration.model.api_model_name!r}"
+            )
     elif report.model != configuration.model.exact_repo:
         raise CLIError("Claude changed an exact model request")
 

src/tests/_internal/cli/services/presets/test_verify.py

@@ -304,6 +304,74 @@ def test_rejects_benchmark_on_a_different_dataset(self, tmp_path, reported):
                 created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc),
             )
 
+    @pytest.mark.parametrize(
+        "served",
+        [
+            "Qwen/Qwen3.8-27B-GPTQ-Int4",  # the reported substitution: another generation
+            "Qwen/Qwen3.8-27B",
+            "meta-llama/Llama-3-8B",  # another family entirely
+            "Qwen/Qwen3.5-27Bx",  # a longer name that merely starts the same
+        ],
+    )
+    def test_rejects_a_served_repo_that_is_not_a_variant_of_the_base(self, tmp_path, served):
+        """`base` has to constrain which repos are acceptable.
+
+        Verification checked that the service advertised the requested name and
+        that the report echoed the requested base — both of which a substitution
+        preserves. `vllm serve Qwen/Qwen3.5-27B-GPTQ-Int4 --served-model-name
+        Qwen/Qwen3.8-27B` answered a request for Qwen3.8 with a different model
+        generation and verified clean.
+        """
+        run = get_running_service_run()
+        report = get_successful_preset_report(run).model_copy(update={"model": served})
+
+        # Both values named, as elsewhere here: "not a variant" alone is not actionable.
+        with pytest.raises(CLIError, match="is not a variant of the requested base"):
+            build_verified_preset(
+                run=run,
+                preset_configuration=PresetConfiguration(
+                    name="qwen-build", base="Qwen/Qwen3.5-27B"
+                ),
+                report=report,
+                workspace_path=tmp_path,
+                session_path=tmp_path,
+                preset_id="ab12cd34",
+                name=None,
+                created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc),
+            )
+
+    @pytest.mark.parametrize(
+        "served",
+        [
+            "community/Qwen3.5-27B-GPTQ-Int4",  # a quantisation by another publisher
+            "Qwen/Qwen3.5-27B-AWQ",
+            "Qwen/Qwen3.5-27B",  # the base itself
+            "qwen/qwen3.5-27b-gptq-int4",  # repo references are not case-sensitive
+        ],
+    )
+    def test_accepts_a_genuine_variant_of_the_base(self, tmp_path, served):
+        """The check must not cost the freedom `base` exists to grant.
+
+        A quantisation is routinely published by someone other than the model's
+        author, so the owner is deliberately not compared — only the model name.
+        """
+        run = get_running_service_run()
+        report = get_successful_preset_report(run).model_copy(update={"model": served})
+
+        preset = build_verified_preset(
+            run=run,
+            preset_configuration=PresetConfiguration(name="qwen-build", base="Qwen/Qwen3.5-27B"),
+            report=report,
+            workspace_path=tmp_path,
+            session_path=tmp_path,
+            preset_id="ab12cd34",
+            name=None,
+            created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc),
+        )
+
+        assert preset.repo == served
+        assert preset.base == "Qwen/Qwen3.5-27B"
+
     def test_rejects_variant_for_exact_model_request(self, tmp_path):
         run = get_running_service_run()
         report = get_successful_preset_report(run).model_copy(update={"model": "other/model"})