Fix the GPU CI tiers and split run_tests.sh into tests/lib

.github/scripts/refresh_spur_env.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved.
+# SPDX-License-Identifier: MIT
+# Runner listeners live for weeks and keep the scheduler env they started with; a stale
+# SPUR_CONTROLLER_ADDR makes every submission fail auth. Export the host's current values.
+set -euo pipefail
+
+for f in /etc/environment /etc/profile.d/spur.sh; do
+  [ -r "$f" ] || continue
+  sed -nE 's/^[[:space:]]*(export[[:space:]]+)?(SPUR_[A-Z_]+)=["'\'']?([^"'\'']*)["'\'']?[[:space:]]*$/\2=\3/p' "$f"
+done | awk -F= '!seen[$1]++' | tee -a "${GITHUB_ENV:?not running under GitHub Actions}"

.github/workflows/ci.yml

@@ -240,6 +240,10 @@ jobs:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
         with:
           fetch-depth: 0
+      # The runner's own env predates any scheduler change since it started; every
+      # later step (tests and reclaim) must talk to the controller the host uses now.
+      - name: refresh scheduler env
+        run: .github/scripts/refresh_spur_env.sh
       # exec, so run_tests.sh IS the step's entry process: on cancel the runner
       # signals only that one pid, and a wrapping bash would not forward it, so
       # the script's INT/TERM trap (scancel of the dispatched job) never ran.
@@ -284,6 +288,8 @@ jobs:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
         with:
           fetch-depth: 0
+      - name: refresh scheduler env
+        run: .github/scripts/refresh_spur_env.sh
       # exec: see the engine job — keeps run_tests.sh's INT/TERM trap reachable.
       - name: e2e ${{ matrix.engine }} mixed
         run: exec bash tests/run_tests.sh e2e ${{ matrix.engine }} mixed
@@ -341,6 +347,8 @@ jobs:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
         with:
           fetch-depth: 0
+      - name: refresh scheduler env
+        run: .github/scripts/refresh_spur_env.sh
       # exec: see the engine job — keeps run_tests.sh's INT/TERM trap reachable,
       # which is what releases the held node pair on cancel.
       - name: e2e ${{ matrix.engine }} disag

tests/engine/sglang/test_wait_for_decode_args.py

@@ -19,7 +19,7 @@
 from infera.engine.sglang.__main__ import (  # noqa: E402
     _maybe_wait_for_decode,
     _run_started_engine,
-    _wait_for_decode_until_stop,
+    _run_startup_barrier_until_stop,
 )
 from infera.engine.sglang.args import parse_sglang_args  # noqa: E402
 
@@ -182,29 +182,31 @@ async def test_maybe_wait_for_decode_gives_a_short_budget_to_the_probe(monkeypat
 
 
 @pytest.mark.asyncio
-async def test_wait_for_decode_until_stop_aborts_when_engine_dies(monkeypatch):
+async def test_startup_barrier_until_stop_aborts_when_engine_dies(monkeypatch):
     async def _hang(*_a, **_k):
         await asyncio.Event().wait()
 
-    monkeypatch.setattr("infera.engine.sglang.__main__._maybe_wait_for_decode", _hang)
+    monkeypatch.setattr("infera.engine.sglang.__main__._startup_barrier", _hang)
     stop = asyncio.Event()
 
     async def _trip():
         await asyncio.sleep(0.01)
         stop.set()
 
     asyncio.create_task(_trip())
-    assert await _wait_for_decode_until_stop(SimpleNamespace(), SimpleNamespace(), stop) is False
+    assert (
+        await _run_startup_barrier_until_stop(SimpleNamespace(), SimpleNamespace(), stop) is False
+    )
 
 
 @pytest.mark.asyncio
-async def test_wait_for_decode_until_stop_returns_true_on_success(monkeypatch):
+async def test_startup_barrier_until_stop_returns_true_on_success(monkeypatch):
     async def _ok(*_a, **_k):
         return None
 
-    monkeypatch.setattr("infera.engine.sglang.__main__._maybe_wait_for_decode", _ok)
+    monkeypatch.setattr("infera.engine.sglang.__main__._startup_barrier", _ok)
     assert (
-        await _wait_for_decode_until_stop(SimpleNamespace(), SimpleNamespace(), asyncio.Event())
+        await _run_startup_barrier_until_stop(SimpleNamespace(), SimpleNamespace(), asyncio.Event())
         is True
     )
 
@@ -227,10 +229,10 @@ async def _watch():
     death_task = asyncio.create_task(_watch())
     monkeypatch.setattr(
         "infera.engine.sglang.__main__._supervise_engine",
-        lambda _engine: (asyncio.Event(), EngineDeath(exit_status=9), death_task),
+        lambda _engine: (asyncio.Event(), EngineDeath(observed=True, returncode=9), death_task),
     )
     monkeypatch.setattr(
-        "infera.engine.sglang.__main__._wait_for_decode_until_stop",
+        "infera.engine.sglang.__main__._run_startup_barrier_until_stop",
         _cancelled,
     )
     monkeypatch.setattr(

tests/lib/container_pytest.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+###############################################################################
+# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved.
+#
+# SPDX-License-Identifier: MIT
+###############################################################################
+# In-container pytest; failures go to /scratch/failures.txt under <tag>.
+# Usage: container_pytest.sh [--per-file] <tag> <pytest args... | test dir>
+set -uo pipefail
+
+FAILS=/scratch/failures.txt
+OUT=/scratch/.pytest.out
+PYTEST=(python3 -m pytest -p no:cacheprovider -o addopts= -rfE)
+[ -n "${INFERA_E2E_K:-}" ] && PYTEST+=(-k "$INFERA_E2E_K")
+
+pip install -q pytest pytest-asyncio nats-py 2>/dev/null || true
+
+run_once() {
+  local tag="$1" rc
+  shift
+  "${PYTEST[@]}" "$@" 2>&1 | stdbuf -oL tee "$OUT"
+  rc=${PIPESTATUS[0]}
+  grep -aE '^(FAILED|ERROR) ' "$OUT" 2>/dev/null | sed "s|^|$tag |" >> "$FAILS"
+  return "$rc"
+}
+
+# One process per file, so a ROCm/HIP native crash costs one file, not the run.
+per_file() {
+  local tag="$1" scope="$2" files f code out line fails rc=0
+  # A scope with no tests must fail, or a rename would pass having tested nothing.
+  if ! files=$(find "$scope" -name "test_*.py") || [ -z "$files" ]; then
+    echo "$tag FATAL: scope unreadable or holds no test_*.py" >&2
+    echo "$tag scope unreadable or holds no test_*.py" >> "$FAILS"
+    return 1
+  fi
+  for f in $(printf '%s\n' "$files" | sort); do
+    echo "----- pytest $f -----"
+    "${PYTEST[@]}" -q "$f" 2>&1 | stdbuf -oL tee "$OUT"
+    code=${PIPESTATUS[0]}
+    out=$(cat "$OUT")
+    case $code in
+      0)
+        line=$(printf '%s' "$out" | grep -E "passed|failed|skipped|no tests ran" | tail -1) ;;
+      134 | 137 | 139)
+        line="CRASH(exit=$code)"
+        echo "$tag CRASH(exit=$code) $f" >> "$FAILS" ;;
+      5)
+        # Guarded files importorskip a module their image ships: nothing collected = broken image.
+        line="FAIL: no tests collected (exit=5)"
+        echo "$tag $f (exit=5, no tests collected)" >> "$FAILS" ;;
+      *)
+        line=$(printf '%s' "$out" | grep -E "passed|failed|error|skipped" | tail -1)
+        line="${line:-(exit=$code)}"
+        fails=$(printf '%s\n' "$out" | grep -aE "^(FAILED|ERROR) ")
+        if [ -n "$fails" ]; then
+          printf '%s\n' "$fails" | sed "s|^|$tag |" >> "$FAILS"
+        else
+          echo "$tag $f (exit=$code)" >> "$FAILS"
+        fi ;;
+    esac
+    [ "$code" -eq 0 ] || rc=1
+    printf "  %-56s %s\n" "$f" "$line"
+  done
+  return "$rc"
+}
+
+if [ "${1:-}" = "--per-file" ]; then
+  shift
+  per_file "$@"
+else
+  run_once "$@"
+fi