From f588efb93155f6ce6f03be6c455e8578cb7eefe7 Mon Sep 17 00:00:00 2001 From: "liang.feng" Date: Mon, 20 Jul 2026 23:45:45 -0700 Subject: [PATCH 1/3] feat(inference): Cosmos3-Edge num_frames default + Edge GPU smoke tests Default num_frames to 121 for Cosmos3-Edge video generation (keyed on the reasoner vlm model_name), mirroring the existing per-model _RESOLUTION_SHIFT_DEFAULTS pattern; other models keep the 189 modality default. resolution (480, from model config) and fps (24, shared) are already correct for Edge, so only num_frames needed a model-specific default. Add two GPU smoke tests mirroring the Nano suite, wired into the existing generator/reasoner inference-smoke CI jobs: - tests/edge_inference_smoke_test.py (8-GPU): t2v + policy + forward_dynamics. Edge has no sound, so t2vs -> plain t2v; the t2v sample gets the stronger whole-clip check and asserts the Edge 480/121/24 generation defaults. - tests/edge_reasoner_inference_smoke_test.py (4-GPU): image-conditioned reasoner. Golden-free smoke that validates the generated reasoner_text is non-empty, coherent, and on-topic (references the image's robotic subject). Verified on 4xGB200: Edge produced an accurate robotic-scene description. Also add a unit test for the Edge=121 / Nano=189 / explicit-override behavior and note the Edge default in docs/inference.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/gpu-tests.yml | 28 ++- cosmos_framework/inference/args.py | 20 ++ cosmos_framework/inference/args_test.py | 26 ++ docs/inference.md | 2 +- tests/edge_inference_smoke_test.py | 250 ++++++++++++++++++++ tests/edge_reasoner_inference_smoke_test.py | 172 ++++++++++++++ 6 files changed, 495 insertions(+), 3 deletions(-) create mode 100644 tests/edge_inference_smoke_test.py create mode 100644 tests/edge_reasoner_inference_smoke_test.py diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml index 31119267..09b8f292 100644 --- a/.github/workflows/gpu-tests.yml +++ b/.github/workflows/gpu-tests.yml @@ -9,8 +9,8 @@ # The five GPU jobs then run (one at a time on the single runner): # * training-smoke — Nano SFT pipeline (convert -> train 5 -> export -> t2i) # * generator-training-regression — vision_sft_nano loss vs goldens (4-GPU subset) -# * generator-inference-smoke — Nano multi-modality inference (t2vs + policy + forward_dynamics) -# * reasoner-inference-smoke — Nano reasoner inference golden + Qwen conversion coverage (4-GPU) +# * generator-inference-smoke — Nano + Edge multi-modality inference (t2vs/t2v + policy + forward_dynamics) +# * reasoner-inference-smoke — Nano reasoner golden + Edge reasoner smoke + Qwen conversion coverage (4-GPU) # * reasoner-training-regression — llava_ov loss vs goldens (4-GPU subset) # # Requires: @@ -132,6 +132,18 @@ jobs: uv run --all-extras --group=cu128-train python -m pytest -v -s \ tests/nano_inference_smoke_test.py --num-gpus=8 --levels=2 -o addopts= + # Same three-modality smoke on the smaller Cosmos3-Edge checkpoint: a plain + # t2v (Edge has no sound, so t2vs is swapped for t2v and its 480/121/24 + # generation defaults are asserted) plus action policy + forward_dynamics. + # Reuses the same input-asset cache dir; the Edge checkpoint downloads once + # and is reused afterward. + - name: Edge inference smoke (t2v + action policy + forward_dynamics, 8 GPU) + run: | + export LD_LIBRARY_PATH= + export COSMOS_DOWNLOAD_CACHE_DIR="$RUNNER_WORKSPACE/cosmos_input_cache" + uv run --all-extras --group=cu128-train python -m pytest -v -s \ + tests/edge_inference_smoke_test.py --num-gpus=8 --levels=2 -o addopts= + # Inference writes only the pytest tmp dir (the t2vs video + logs); the # checkpoint download stays in the HF cache (kept). No examples/ artifacts. - name: Clean up run outputs @@ -171,6 +183,18 @@ jobs: tests/nano_reasoner_inference_smoke_test.py::test_nano_reasoner_image_first_token_logits \ --num-gpus=4 --levels=2 -o addopts= + # Image-conditioned Cosmos3-Edge reasoner inference at smoke level: assert a + # non-empty reasoner_text (no per-checkpoint logits golden, unlike the Nano + # case above). Edge's Nemotron reasoner is vision-capable. Reuse the shared + # input-asset cache dir for reasoner_image.json's remote vision_path. + - name: Edge reasoner inference smoke (image-conditioned reasoner_text, 4 GPU) + run: | + export LD_LIBRARY_PATH= + export COSMOS_DOWNLOAD_CACHE_DIR="$RUNNER_WORKSPACE/cosmos_input_cache" + uv run --all-extras --group=cu128-train python -m pytest -v -s \ + tests/edge_reasoner_inference_smoke_test.py::test_edge_reasoner_image_reasoner_text \ + --num-gpus=4 --levels=2 -o addopts= + # Merge Cosmos3-Nano into the public Qwen3-VL shell and verify that every # Qwen tensor is present and both the vision and language towers come from # Cosmos3-Nano. The exact node ID avoids collecting the training regressions diff --git a/cosmos_framework/inference/args.py b/cosmos_framework/inference/args.py index 2d634406..b82be6e5 100644 --- a/cosmos_framework/inference/args.py +++ b/cosmos_framework/inference/args.py @@ -1045,6 +1045,13 @@ class OmniSampleOverrides( "nvidia/Cosmos3-Edge-Reasoner": "2B", } + _NUM_FRAMES_DEFAULTS: ClassVar[dict[str, int]] = { + # Cosmos3-Edge defaults to a shorter 121-frame clip for video + # generation; every other model keeps the per-modality JSON default + # (189). Keyed by ``vlm_config.model_name`` so only Edge is affected. + "nvidia/Cosmos3-Edge-Reasoner": 121, + } + _RESOLUTION_SHIFT_DEFAULTS: ClassVar[dict[(ModelSize, Resolution), float]] = { # 2B rows mirror Cosmos3-Edge's training shift (Cosmos3-Edge.yaml # rectified_flow_training_config.shift: 256->3, 480->5, 720->10). @@ -1082,6 +1089,19 @@ def build_sample(self, *, model_config: Any) -> OmniSampleArgs: self.__dict__.update(merged.__dict__) self.model_mode = sample_meta.model_mode + # Model-specific num_frames default for video generation (e.g. + # Cosmos3-Edge -> 121). Applies only when the user did not request a + # frame count and the mode produces a plain video; image and action + # modes keep their own num_frames handling. + if ( + "num_frames" not in user_fields + and sample_meta.vision_mode == VisionMode.VIDEO + and not sample_meta.model_mode.is_action + ): + num_frames_default = self._NUM_FRAMES_DEFAULTS.get(model_config.vlm_config.model_name) + if num_frames_default is not None: + self.num_frames = num_frames_default + self._build_sample() self._build_sampling(model_config=model_config, sample_meta=sample_meta) self._build_text_data(model_config=model_config, sample_meta=sample_meta) diff --git a/cosmos_framework/inference/args_test.py b/cosmos_framework/inference/args_test.py index 512d3dca..66b4ad84 100644 --- a/cosmos_framework/inference/args_test.py +++ b/cosmos_framework/inference/args_test.py @@ -255,6 +255,32 @@ def test_sample_args(tmp_path: Path): assert text2image_args.shift == 3.0 +def test_edge_num_frames_default(tmp_path: Path): + def _t2v_num_frames(checkpoint: str, label: str, **overrides: object) -> int: + setup_args = OmniSetupOverrides( + checkpoint_path=checkpoint, + output_dir=tmp_path / f"outputs_{label}", + ).build_setup() + model_dict: "OmniMoTModel" = structure_config( + setup_args.load_model_config_dict(), + omegaconf.DictConfig, + ) + args = OmniSampleOverrides( + name=label, + output_dir=tmp_path / label, + model_mode=ModelMode.TEXT2VIDEO, + **overrides, + ).build_sample(model_config=model_dict.config) + return args.num_frames + + # Cosmos3-Edge defaults to a shorter 121-frame clip for video generation. + assert _t2v_num_frames("Cosmos3-Edge", "edge_default") == 121 + # Other models keep the per-modality JSON default (189). + assert _t2v_num_frames("Cosmos3-Nano", "nano_default") == 189 + # An explicit user value always wins over the model-specific default. + assert _t2v_num_frames("Cosmos3-Edge", "edge_override", num_frames=189) == 189 + + def test_build_sound_data_requires_sound_path_for_a2v(): model_config = types.SimpleNamespace(sound_gen=True) sample_meta = types.SimpleNamespace(model_mode=ModelMode.AUDIO_IMAGE2VIDEO) diff --git a/docs/inference.md b/docs/inference.md index 0f580432..214b2bac 100644 --- a/docs/inference.md +++ b/docs/inference.md @@ -219,7 +219,7 @@ Condition arguments: Generation arguments: -- `num_frames`: Number of output frames. `1` = image; `≥24` = video. Default 189; resolution-dependent max — see [FAQ § How many frames can I generate?](./faq.md#q-how-many-frames-can-i-generate). +- `num_frames`: Number of output frames. `1` = image; `≥24` = video. Default 189 (Cosmos3-Edge defaults to 121); resolution-dependent max — see [FAQ § How many frames can I generate?](./faq.md#q-how-many-frames-can-i-generate). Outputs `vision.jpg` or `vision.mp4` depending on `num_frames`. diff --git a/tests/edge_inference_smoke_test.py b/tests/edge_inference_smoke_test.py new file mode 100644 index 00000000..142f12be --- /dev/null +++ b/tests/edge_inference_smoke_test.py @@ -0,0 +1,250 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: OpenMDW-1.1 + +"""8-GPU multi-modality inference smoke test for Cosmos3-Edge. + +Mirrors ``nano_inference_smoke_test.py`` but targets the smaller Cosmos3-Edge +checkpoint, adapted to its capabilities: Edge has ``sound_gen=false`` (so the +``t2vs`` text2video+sound sample is replaced by a plain ``t2v`` sample) and a +native 480p resolution, but keeps ``action_gen=true``. + +A single ``throughput`` ``cosmos_framework.scripts.inference`` call runs three +input samples of different modalities (the ``-i`` flag takes a list of files) +and validates each output: + + * ``inputs/omni/t2v.json`` (text2video, no explicit resolution/num_frames/fps) + -> a ``vision.mp4`` validated with the stronger whole-clip check (decodes the + full clip; asserts frame count + real pixel variation, so a collapsed / + near-constant video fails). Because the input pins none of those fields, this + sample also exercises the Cosmos3-Edge generation defaults resolved in + ``args.py`` -- ``resolution=480``, ``num_frames=121`` (the Edge-specific + default; other models default to 189), ``fps=24`` -- which the test asserts + from the serialized ``args``. + * ``inputs/omni/action_forward_dynamics_camera.json`` (forward_dynamics) -> a + ``vision.mp4`` that decodes to at least one valid video frame (``action_path`` + is an input, not an output). + * ``inputs/omni/action_policy_robot.json`` (policy) -> BOTH a ``vision.mp4`` and + a finite, non-empty predicted ``action`` array in ``sample_outputs.json``. + +Every sample produces a video; the policy sample additionally produces an action. +Unlike the Nano suite there is no sound sample (Edge has no sound generation) and +no transfer/multi-control run. + +Smoke-level only (output validity + the Edge default triple, not numeric +goldens). The checkpoint + its tokenizers download from the HF Hub on first run +and are reused afterward. + +Invocation (inside the inference container, from the repo root, on an 8-GPU +node):: + + pytest -s tests/edge_inference_smoke_test.py --num-gpus=8 --levels=2 -o addopts= + +Without ``--num-gpus``/``--levels`` (e.g. the no-GPU pre-commit CI) the test is +not collected. +""" + +import json +import os +import shutil +import socket +import subprocess +import sys +from pathlib import Path + +import pytest + +from cosmos_framework.inference.fixtures.args import MAX_GPUS + +REPO_ROOT = Path(__file__).resolve().parents[1] + +_INPUTS = [ + "inputs/omni/t2v.json", + "inputs/omni/action_policy_robot.json", + "inputs/omni/action_forward_dynamics_camera.json", +] + +# Cosmos3-Edge generation defaults for a plain text2video sample that pins none +# of these fields (see ``_NUM_FRAMES_DEFAULTS`` + the model config resolution and +# the per-modality fps default in ``cosmos_framework/inference/args.py``). +_EDGE_T2V_RESOLUTION = "480" +_EDGE_T2V_NUM_FRAMES = 121 +_EDGE_T2V_FPS = 24 + + +def _free_port() -> int: + """Return a currently-free TCP port for torchrun's rendezvous (avoids + EADDRINUSE from a hardcoded port / lingering process).""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +def _run(cmd: list[str], log_file: Path) -> str: + """Run ``cmd`` from the repo root, tee combined output (live to stdout under + ``pytest -s`` + into ``log_file``). Inherits the caller's env (HF cache, ...) + plus ``PYTHONPATH=.``. Fails with the log tail on a non-zero exit.""" + env = os.environ.copy() + env["PYTHONPATH"] = f".:{env.get('PYTHONPATH', '')}" + log_file.parent.mkdir(parents=True, exist_ok=True) + captured: list[str] = [] + with log_file.open("w") as fp: + proc = subprocess.Popen( + cmd, env=env, cwd=str(REPO_ROOT), + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, + ) + assert proc.stdout is not None + for line in proc.stdout: + sys.stdout.write(line) + sys.stdout.flush() + fp.write(line) + captured.append(line) + returncode = proc.wait() + text = "".join(captured) + if returncode != 0: + pytest.fail(f"inference failed with exit code {returncode}:\n {' '.join(cmd)}\nLog tail:\n{text[-3000:]}") + return text + + +def _assert_valid_video(mp4_path: Path) -> None: + """Assert ``mp4_path`` decodes to at least one valid, non-degenerate video frame.""" + import av + + assert mp4_path.is_file() and mp4_path.stat().st_size > 1024, f"video missing/too small: {mp4_path}" + with av.open(str(mp4_path)) as container: + vstreams = container.streams.video + assert vstreams, f"no video stream in {mp4_path}" + width = height = frames = 0 + for frame in container.decode(vstreams[0]): + width, height, frames = frame.width, frame.height, frames + 1 + break + assert frames >= 1 and width > 0 and height > 0, f"no decodable video frame in {mp4_path}" + + +def _assert_video_has_content(mp4_path: Path, *, min_frames: int = 16) -> None: + """Assert ``mp4_path`` decodes to enough non-degenerate frames. + + Stronger than ``_assert_valid_video`` (which only inspects the first frame): + decodes the whole clip and checks the frame count plus real pixel variation, + so a run that produced a well-formed container but collapsed to a constant / + blank video fails instead of passing. + """ + import av + import numpy as np + + with av.open(str(mp4_path)) as container: + vstreams = container.streams.video + assert vstreams, f"no video stream in {mp4_path}" + frames = [frame.to_ndarray(format="rgb24") for frame in container.decode(vstreams[0])] + assert len(frames) >= min_frames, f"{mp4_path}: expected >= {min_frames} frames, got {len(frames)}" + arr = np.stack(frames).astype(np.float64) + assert np.all(np.isfinite(arr)), f"{mp4_path}: decoded video has non-finite pixels" + # Both spatial and temporal flatness collapse global std toward 0; a real + # generated clip sits well above this floor (typically tens on a 0-255 scale). + assert arr.std() > 3.0, f"{mp4_path}: degenerate/near-constant video (pixel std={arr.std():.3f})" + + +def _assert_valid_action(content: dict, where: str) -> None: + """Assert a policy sample's predicted ``action`` is a non-empty, all-finite array.""" + import numpy as np + + assert isinstance(content, dict) and content.get("action") is not None, ( + f"no 'action' in policy output ({where}); content keys={list(content) if isinstance(content, dict) else content}" + ) + arr = np.asarray(content["action"], dtype=np.float64) + assert arr.size > 0, f"empty action output ({where})" + assert np.all(np.isfinite(arr)), f"action output has NaN/Inf ({where})" + + +@pytest.fixture(scope="module", autouse=True) +def _require_8_gpus() -> None: + """Skip the module unless we can launch an 8-GPU run here.""" + if shutil.which("torchrun") is None: + pytest.skip("torchrun not on PATH -- must run inside the inference container") + try: + import torch + except Exception as exc: # pragma: no cover -- surfaces during dev only + pytest.skip(f"torch unavailable ({exc!r})") + if not torch.cuda.is_available() or torch.cuda.device_count() < 8: + pytest.skip(f"requires 8 visible CUDA devices, found {torch.cuda.device_count()}") + + +# Defined only when the active MAX_GPUS is 8 -- the conftest rejects ``gpus(N)`` +# markers outside ``ALL_NUM_GPUS = (0, 1, MAX_GPUS)``. +if MAX_GPUS == 8: + + @pytest.mark.level(2) + @pytest.mark.gpus(8) + def test_edge_inference_omni(tmp_path: Path) -> None: + """Throughput run over t2v + policy + forward_dynamics on Cosmos3-Edge.""" + out_dir = tmp_path / "out" + cmd = [ + "torchrun", + "--nproc_per_node=8", + f"--master_port={_free_port()}", + "-m", + "cosmos_framework.scripts.inference", + "--parallelism-preset=throughput", + "-i", + *_INPUTS, + "-o", + str(out_dir), + "--checkpoint-path", + "Cosmos3-Edge", + "--seed=0", + ] + _run(cmd, tmp_path / "inference.log") + + results = sorted(out_dir.rglob("sample_outputs.json")) + assert len(results) == len(_INPUTS), ( + f"expected {len(_INPUTS)} sample_outputs.json (one per input), found {[str(p) for p in results]}" + ) + + # Dispatch validation by what each sample produced (robust to model_mode + # string formatting): a vision.mp4 -> valid video; an `action` content -> + # valid action array. The plain t2v sample (no conditioning `vision_path`) + # gets the stronger whole-clip check and additionally verifies the + # Cosmos3-Edge generation defaults. + n_video = n_action = 0 + edge_t2v_checked = False + for so in results: + data = json.loads(so.read_text()) + args = data.get("args", {}) + content = data["outputs"][0]["content"] + sample_dir = so.parent + video = sample_dir / "vision.mp4" + # The t2v sample is the only one with no conditioning input (policy and + # forward_dynamics both set `vision_path`); use it to exercise both the + # stronger video check and the Edge generation defaults. + is_t2v = not args.get("vision_path") + if video.is_file(): + # t2v -> whole-clip check (frame count + real pixel variation, so a + # collapsed/near-constant clip fails); conditioned modes keep the + # first-frame check, matching the Nano suite. + if is_t2v: + _assert_video_has_content(video) + else: + _assert_valid_video(video) + n_video += 1 + if isinstance(content, dict) and content.get("action") is not None: + _assert_valid_action(content, str(so)) + n_action += 1 + if is_t2v: + # t2v.json pins none of these, so the resolved args should carry the + # Edge defaults. + assert str(args.get("resolution")) == _EDGE_T2V_RESOLUTION, ( + f"expected Edge t2v resolution {_EDGE_T2V_RESOLUTION}, got {args.get('resolution')} ({so})" + ) + assert args.get("num_frames") == _EDGE_T2V_NUM_FRAMES, ( + f"expected Edge t2v num_frames {_EDGE_T2V_NUM_FRAMES}, got {args.get('num_frames')} ({so})" + ) + assert args.get("fps") == _EDGE_T2V_FPS, ( + f"expected Edge t2v fps {_EDGE_T2V_FPS}, got {args.get('fps')} ({so})" + ) + edge_t2v_checked = True + + # Every sample produces a valid video (t2v, forward_dynamics, policy); the + # policy sample additionally yields an action, and the t2v sample pins the + # Edge generation defaults. + assert n_video == len(_INPUTS), f"expected every sample to produce a valid video, got {n_video}/{len(_INPUTS)}" + assert n_action >= 1, f"expected the policy sample's action to be checked, got {n_action}" + assert edge_t2v_checked, "expected the t2v sample's Edge defaults (resolution/num_frames/fps) to be checked" diff --git a/tests/edge_reasoner_inference_smoke_test.py b/tests/edge_reasoner_inference_smoke_test.py new file mode 100644 index 00000000..18bb5d8f --- /dev/null +++ b/tests/edge_reasoner_inference_smoke_test.py @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: OpenMDW-1.1 + +"""4-GPU reasoner inference smoke test for Cosmos3-Edge. + +The Edge counterpart of ``nano_reasoner_inference_smoke_test.py``, but at +smoke level: instead of comparing first-token logits against a committed golden +(the Nano suite's tight regression), it validates the generated ``reasoner_text`` +directly -- non-empty, coherent (min length + lexical diversity), and on-topic +for the input image (references the robotic subject that dominates the frame). +This keeps the Edge case runnable in CI without a per-checkpoint golden bootstrap +while still checking the generated content, not just that something was emitted. + +One case, image-conditioned reasoner inference (matching what the Nano CI job +actually runs): a single ``cosmos_framework.scripts.inference`` torchrun over +``inputs/reasoner/reasoner_image.json``. Edge's Nemotron reasoner is +vision-capable (it loads a SigLIP2 tower lazily -- see ``_reasoner_vision_capable`` +in ``cosmos_framework/inference/args.py``), so the image prompt is encoded and +the run must emit non-empty generated text. + +Smoke-level only (output validity, not numeric goldens). The checkpoint + its +reasoner backbone download from the HF Hub on first run and are reused afterward. + +Invocation (inside the inference container, from the repo root, on a >=4-GPU +node):: + + TEST_MAX_GPUS=4 pytest -s tests/edge_reasoner_inference_smoke_test.py \ + --num-gpus=4 --levels=2 -o addopts= + +Without ``--num-gpus``/``--levels`` (e.g. the no-GPU pre-commit CI) the test is +not collected. +""" + +import json +import os +import re +import shutil +import socket +import subprocess +import sys +from pathlib import Path + +import pytest + +from cosmos_framework.inference.fixtures.args import MAX_GPUS + +REPO_ROOT = Path(__file__).resolve().parents[1] + +# The image prompt (``inputs/reasoner/reasoner_image.json``) is +# "Describe what is happening in this image in one sentence." over ``robot_153.jpg``: +# an ego-view of two robotic hands/arms reaching over a table (red apple + lab +# equipment + a person in the background). Decoding is greedy (``do_sample: false`` +# in ``defaults/reasoner/sample_args.json``), so a correct description reliably +# references that dominant robotic subject. The stems below are broad enough to +# survive phrasing ("robotic hands", "mechanical arms", "grippers", ...) yet fail +# on garbage, an empty/degenerate reply, or a description of some other scene. +_SUBJECT_PATTERN = re.compile( + r"\b(robot\w*|mechanical|gripper\w*|prosthetic|arm\w*|hand\w*|finger\w*)\b", + re.IGNORECASE, +) +# A real one-sentence description clears these easily; the floors only catch a +# collapsed/degenerate generation (empty-ish, or one token repeated). +_MIN_TEXT_CHARS = 15 +_MIN_UNIQUE_WORDS = 3 + + +def _free_port() -> int: + """Return a currently-free TCP port for torchrun's rendezvous.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +def _run(cmd: list[str], log_file: Path) -> str: + """Run ``cmd`` from the repo root, tee combined output (live under ``-s`` + + into ``log_file``). Inherits the caller's env plus ``PYTHONPATH=.``. Fails + with the log tail on a non-zero exit.""" + env = os.environ.copy() + env["PYTHONPATH"] = f".:{env.get('PYTHONPATH', '')}" + log_file.parent.mkdir(parents=True, exist_ok=True) + captured: list[str] = [] + with log_file.open("w") as fp: + proc = subprocess.Popen( + cmd, env=env, cwd=str(REPO_ROOT), + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, + ) + assert proc.stdout is not None + for line in proc.stdout: + sys.stdout.write(line) + sys.stdout.flush() + fp.write(line) + captured.append(line) + returncode = proc.wait() + text = "".join(captured) + if returncode != 0: + pytest.fail(f"inference failed with exit code {returncode}:\n {' '.join(cmd)}\nLog tail:\n{text[-3000:]}") + return text + + +def _assert_reasoner_text(out_dir: Path) -> None: + """Assert the single sample produced a valid, on-topic ``reasoner_text``. + + Three gates: (1) a non-empty string was generated; (2) it is coherent, not a + collapsed/degenerate reply (min length + lexical diversity); (3) its content + is correct for the input image -- it references the robotic subject that + dominates ``robot_153.jpg`` (see ``_SUBJECT_PATTERN``). + """ + results = sorted(out_dir.rglob("sample_outputs.json")) + assert len(results) == 1, f"expected one sample_outputs.json, found {[str(p) for p in results]}" + content = json.loads(results[0].read_text())["outputs"][0]["content"] + text = content.get("reasoner_text") if isinstance(content, dict) else None + + # (1) non-empty string. + assert isinstance(text, str) and text.strip(), f"empty/missing reasoner_text in {results[0]}: {content!r}" + + # (2) coherent, non-degenerate text (not empty-ish, not a single repeated token). + stripped = text.strip() + words = re.findall(r"[a-zA-Z]+", stripped.lower()) + assert len(stripped) >= _MIN_TEXT_CHARS, ( + f"reasoner_text too short to be a real description ({len(stripped)} chars): {stripped!r}" + ) + assert len(set(words)) >= _MIN_UNIQUE_WORDS, ( + f"reasoner_text is degenerate/repetitive ({len(set(words))} unique words): {stripped!r}" + ) + + # (3) content correctness: the reply describes the actual image -- it must + # reference the robotic hands/arms that dominate the frame. + assert _SUBJECT_PATTERN.search(stripped), ( + f"reasoner_text does not describe the image's robotic subject " + f"(expected a term like robot/robotic/arm/hand/gripper): {stripped!r}" + ) + + +@pytest.fixture(scope="module", autouse=True) +def _require_4_gpus() -> None: + """Skip the module unless we can launch a 4-GPU run here.""" + if shutil.which("torchrun") is None: + pytest.skip("torchrun not on PATH -- must run inside the inference container") + try: + import torch + except Exception as exc: # pragma: no cover -- surfaces during dev only + pytest.skip(f"torch unavailable ({exc!r})") + if not torch.cuda.is_available() or torch.cuda.device_count() < 4: + pytest.skip(f"requires 4 visible CUDA devices, found {torch.cuda.device_count()}") + + +# Defined only when the active MAX_GPUS is 4 -- the conftest rejects ``gpus(N)`` +# markers outside ``ALL_NUM_GPUS = (0, 1, MAX_GPUS)``. Run with TEST_MAX_GPUS=4. +if MAX_GPUS == 4: + + @pytest.mark.level(2) + @pytest.mark.gpus(4) + def test_edge_reasoner_image_reasoner_text(tmp_path: Path) -> None: + """Image-conditioned reasoner inference on Cosmos3-Edge; assert non-empty reasoner_text.""" + out_dir = tmp_path / "out" + cmd = [ + "torchrun", + "--nproc_per_node=4", + f"--master_port={_free_port()}", + "-m", + "cosmos_framework.scripts.inference", + "--parallelism-preset=throughput", + "-i", + "inputs/reasoner/reasoner_image.json", + "-o", + str(out_dir), + "--checkpoint-path", + "Cosmos3-Edge", + "--seed=0", + ] + _run(cmd, tmp_path / "inference.log") + _assert_reasoner_text(out_dir) From b73ed6c32dabe76e707689f8e3825a8a00e6cbe5 Mon Sep 17 00:00:00 2001 From: "liang.feng" Date: Mon, 20 Jul 2026 23:53:39 -0700 Subject: [PATCH 2/3] test(edge-reasoner): require both robotic-subject and scene anchors Strengthen the reasoner_text content check into a fuller "complete description" gate: the generated text must reference BOTH the dominant robotic subject (_SUBJECT_PATTERN) AND the table/lab setting (_SCENE_PATTERN), not just one. Rejects "subject-only" / "scene-only" replies while staying phrasing-robust. Verified on 4xGB200: the real Edge description ("two robotic arms with mechanical hands ... on a table with wooden structures ... in a lab or research environment ...") passes both anchors; test green (1 passed). Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/edge_reasoner_inference_smoke_test.py | 38 +++++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/tests/edge_reasoner_inference_smoke_test.py b/tests/edge_reasoner_inference_smoke_test.py index 18bb5d8f..39a3e1f2 100644 --- a/tests/edge_reasoner_inference_smoke_test.py +++ b/tests/edge_reasoner_inference_smoke_test.py @@ -6,10 +6,11 @@ The Edge counterpart of ``nano_reasoner_inference_smoke_test.py``, but at smoke level: instead of comparing first-token logits against a committed golden (the Nano suite's tight regression), it validates the generated ``reasoner_text`` -directly -- non-empty, coherent (min length + lexical diversity), and on-topic -for the input image (references the robotic subject that dominates the frame). -This keeps the Edge case runnable in CI without a per-checkpoint golden bootstrap -while still checking the generated content, not just that something was emitted. +directly -- non-empty, coherent (min length + lexical diversity), and a complete +description of the input image (references BOTH the robotic subject that dominates +the frame AND the table/lab setting it sits in). This keeps the Edge case runnable +in CI without a per-checkpoint golden bootstrap while still checking the generated +content, not just that something was emitted. One case, image-conditioned reasoner inference (matching what the Nano CI job actually runs): a single ``cosmos_framework.scripts.inference`` torchrun over @@ -51,13 +52,23 @@ # an ego-view of two robotic hands/arms reaching over a table (red apple + lab # equipment + a person in the background). Decoding is greedy (``do_sample: false`` # in ``defaults/reasoner/sample_args.json``), so a correct description reliably -# references that dominant robotic subject. The stems below are broad enough to -# survive phrasing ("robotic hands", "mechanical arms", "grippers", ...) yet fail -# on garbage, an empty/degenerate reply, or a description of some other scene. +# references both the dominant robotic subject and the table/lab setting. +# +# A complete description must hit BOTH anchors below: the robotic subject AND a +# scene/surface word. Each stem set is broad enough to survive phrasing ("robotic +# hands"/"mechanical arms"/"grippers"; "table"/"desk"/"wooden surface"/"lab") yet +# both together fail on garbage, an empty/degenerate reply, or a description of an +# unrelated scene. A verified Edge run hit each set many times (robotic arms + +# mechanical hands; table x4, wooden, lab, environment, background). _SUBJECT_PATTERN = re.compile( r"\b(robot\w*|mechanical|gripper\w*|prosthetic|arm\w*|hand\w*|finger\w*)\b", re.IGNORECASE, ) +_SCENE_PATTERN = re.compile( + r"\b(table\w*|desk\w*|surface\w*|counter\w*|platform\w*|wood\w*|floor\w*|" + r"lab|labs|laboratory|room\w*|workshop\w*|office\w*|environment\w*|setting\w*|background\w*)\b", + re.IGNORECASE, +) # A real one-sentence description clears these easily; the floors only catch a # collapsed/degenerate generation (empty-ish, or one token repeated). _MIN_TEXT_CHARS = 15 @@ -102,8 +113,9 @@ def _assert_reasoner_text(out_dir: Path) -> None: Three gates: (1) a non-empty string was generated; (2) it is coherent, not a collapsed/degenerate reply (min length + lexical diversity); (3) its content - is correct for the input image -- it references the robotic subject that - dominates ``robot_153.jpg`` (see ``_SUBJECT_PATTERN``). + is a complete description of the input image -- it references BOTH the robotic + subject that dominates ``robot_153.jpg`` (``_SUBJECT_PATTERN``) AND the + table/lab setting it sits in (``_SCENE_PATTERN``). """ results = sorted(out_dir.rglob("sample_outputs.json")) assert len(results) == 1, f"expected one sample_outputs.json, found {[str(p) for p in results]}" @@ -123,12 +135,16 @@ def _assert_reasoner_text(out_dir: Path) -> None: f"reasoner_text is degenerate/repetitive ({len(set(words))} unique words): {stripped!r}" ) - # (3) content correctness: the reply describes the actual image -- it must - # reference the robotic hands/arms that dominate the frame. + # (3) content correctness: a complete description references BOTH the robotic + # subject that dominates the frame AND the table/lab setting it sits in. assert _SUBJECT_PATTERN.search(stripped), ( f"reasoner_text does not describe the image's robotic subject " f"(expected a term like robot/robotic/arm/hand/gripper): {stripped!r}" ) + assert _SCENE_PATTERN.search(stripped), ( + f"reasoner_text does not describe the image's setting " + f"(expected a term like table/desk/surface/wood/lab/room): {stripped!r}" + ) @pytest.fixture(scope="module", autouse=True) From 647f73c46e7653c691d5bcdbfbbddeef75279f82 Mon Sep 17 00:00:00 2001 From: "liang.feng" Date: Tue, 21 Jul 2026 00:03:36 -0700 Subject: [PATCH 3/3] fix(inference): scope Edge num_frames default out of reasoner mode + regression tests The Edge 121 default guard keyed on vision_mode==VIDEO, but the reasoner reports VIDEO vision_mode (it is neither an image-output nor an action mode), so an Edge reasoner sample was getting num_frames rewritten to 121. Harmless at runtime (the reasoner treats num_frames as an inert 1) but incorrect; exclude reasoner from the guard alongside action/image modes. Extend test_edge_num_frames_default with regression cases proving the default is scoped to plain video generation only: Edge text2image -> 1, Edge policy -> 189 (action, unchanged), Edge reasoner -> 1. All build on CPU (no GPU/assets). Co-Authored-By: Claude Opus 4.8 (1M context) --- cosmos_framework/inference/args.py | 6 ++++-- cosmos_framework/inference/args_test.py | 25 +++++++++++++++++++------ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/cosmos_framework/inference/args.py b/cosmos_framework/inference/args.py index b82be6e5..bcb71a5e 100644 --- a/cosmos_framework/inference/args.py +++ b/cosmos_framework/inference/args.py @@ -1091,12 +1091,14 @@ def build_sample(self, *, model_config: Any) -> OmniSampleArgs: # Model-specific num_frames default for video generation (e.g. # Cosmos3-Edge -> 121). Applies only when the user did not request a - # frame count and the mode produces a plain video; image and action - # modes keep their own num_frames handling. + # frame count and the mode produces a plain video; image, action, and + # reasoner modes keep their own num_frames handling (reasoner reports + # VIDEO vision_mode but uses num_frames as an inert 1). if ( "num_frames" not in user_fields and sample_meta.vision_mode == VisionMode.VIDEO and not sample_meta.model_mode.is_action + and not sample_meta.model_mode.is_reasoner ): num_frames_default = self._NUM_FRAMES_DEFAULTS.get(model_config.vlm_config.model_name) if num_frames_default is not None: diff --git a/cosmos_framework/inference/args_test.py b/cosmos_framework/inference/args_test.py index 66b4ad84..6b61d5b1 100644 --- a/cosmos_framework/inference/args_test.py +++ b/cosmos_framework/inference/args_test.py @@ -256,7 +256,12 @@ def test_sample_args(tmp_path: Path): def test_edge_num_frames_default(tmp_path: Path): - def _t2v_num_frames(checkpoint: str, label: str, **overrides: object) -> int: + def _num_frames( + checkpoint: str, + label: str, + model_mode: ModelMode = ModelMode.TEXT2VIDEO, + **overrides: object, + ) -> int: setup_args = OmniSetupOverrides( checkpoint_path=checkpoint, output_dir=tmp_path / f"outputs_{label}", @@ -268,17 +273,25 @@ def _t2v_num_frames(checkpoint: str, label: str, **overrides: object) -> int: args = OmniSampleOverrides( name=label, output_dir=tmp_path / label, - model_mode=ModelMode.TEXT2VIDEO, + model_mode=model_mode, **overrides, ).build_sample(model_config=model_dict.config) return args.num_frames - # Cosmos3-Edge defaults to a shorter 121-frame clip for video generation. - assert _t2v_num_frames("Cosmos3-Edge", "edge_default") == 121 + # Video generation: Cosmos3-Edge defaults to a shorter 121-frame clip. + assert _num_frames("Cosmos3-Edge", "edge_default") == 121 # Other models keep the per-modality JSON default (189). - assert _t2v_num_frames("Cosmos3-Nano", "nano_default") == 189 + assert _num_frames("Cosmos3-Nano", "nano_default") == 189 # An explicit user value always wins over the model-specific default. - assert _t2v_num_frames("Cosmos3-Edge", "edge_override", num_frames=189) == 189 + assert _num_frames("Cosmos3-Edge", "edge_override", num_frames=189) == 189 + + # Regression: the Edge 121 default is scoped to plain video generation only. + # Image modes stay single-frame; action modes keep their own default (189); + # the reasoner (which reports VIDEO vision_mode) keeps its inert 1 -- none of + # these should be rewritten to 121. + assert _num_frames("Cosmos3-Edge", "edge_t2i", model_mode=ModelMode.TEXT2IMAGE) == 1 + assert _num_frames("Cosmos3-Edge", "edge_policy", model_mode=ModelMode.POLICY) == 189 + assert _num_frames("Cosmos3-Edge", "edge_reasoner", model_mode=ModelMode.REASONER, prompt="Describe.") == 1 def test_build_sound_data_requires_sound_path_for_a2v():