From e225d68b16056c6c61f548a997c10dc1c40faa95 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Fri, 28 Aug 2026 01:04:17 +0800 Subject: [PATCH 1/3] CollectiveX: name Slurm allocations after the GHA runner for dashboard correlation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit inferencex-dash's cluster collector samples allocation names (sacct JobName) and the dashboard joins them against the GHA job's runner_name — the convention every production runner launcher already follows with --job-name="$RUNNER_NAME". CollectiveX's salloc calls carried no job name, so its allocations were invisible to that correlation. Prepend the name at the collx_salloc_jobid seam (a caller-supplied --job-name still wins; salloc takes the last occurrence), falling back to 'collectivex' for hand-driven runs or a runner name that is not a plain Slurm-safe token. --- experimental/CollectiveX/runtime/common.sh | 14 ++++++++++ .../CollectiveX/tests/test_runtime.py | 28 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh index d786799c73..21381924b1 100644 --- a/experimental/CollectiveX/runtime/common.sh +++ b/experimental/CollectiveX/runtime/common.sh @@ -424,10 +424,24 @@ exec python3 bench/run_ep.py "$@" BASH } +# Slurm job name for dashboard correlation. inferencex-dash's cluster collector +# samples allocation names (sacct JobName) and the dashboard joins them against +# the GHA job's runner_name — the production runners' convention +# (--job-name="$RUNNER_NAME"). Fall back to a fixed label for hand-driven runs; +# reject anything that is not a plain Slurm-safe token rather than quoting it. +collx_slurm_job_name() { + local name="${RUNNER_NAME:-}" + [[ "$name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ ]] || name="collectivex" + printf '%s' "$name" +} + # Allocate via salloc's stable grant message and assign JOB_ID in this shell. # Record it so workflow cleanup can release a launcher interrupted by Actions. +# The job name is prepended so a caller-supplied --job-name still wins (salloc +# takes the last occurrence). collx_salloc_jobid() { local log_label=scheduler-allocation log job_id root="${COLLX_JOB_ROOT:-}" + set -- --job-name="$(collx_slurm_job_name)" "$@" case "${COLLX_SALLOC_ATTEMPT:-1}" in 1) ;; 2|3) log_label+="-a${COLLX_SALLOC_ATTEMPT}" ;; diff --git a/experimental/CollectiveX/tests/test_runtime.py b/experimental/CollectiveX/tests/test_runtime.py index 3120879e01..43ff1ff205 100644 --- a/experimental/CollectiveX/tests/test_runtime.py +++ b/experimental/CollectiveX/tests/test_runtime.py @@ -145,6 +145,34 @@ def test_operator_config_registry_only_emits_tracked_baseline(self) -> None: self.assertIn(b"COLLX_SQUASH_DIR\0/home/sa-shared/containers\0", payload) self.assertIn(b"COLLX_RDMA_DEVICES\0", payload) +class SlurmJobNameTests(unittest.TestCase): + # inferencex-dash correlates cluster allocations with GHA jobs by Slurm job + # name == the runner_name (the production runners' --job-name="$RUNNER_NAME" + # convention), so collx_salloc_jobid must submit under that name and fall + # back to a fixed label rather than an unset or unsafe one. + @staticmethod + def _job_name(env_pairs: str) -> str: + completed = subprocess.run( + ["bash", "-c", + f"source runtime/common.sh 2>/dev/null; {env_pairs} collx_slurm_job_name"], + cwd=RUNTIME.parent, capture_output=True, text=True, check=True, + ) + return completed.stdout + + def test_runner_name_becomes_the_job_name(self) -> None: + self.assertEqual(self._job_name("RUNNER_NAME=mi355x-amds_03"), "mi355x-amds_03") + + def test_unset_runner_name_falls_back(self) -> None: + self.assertEqual(self._job_name("RUNNER_NAME="), "collectivex") + + def test_unsafe_runner_name_falls_back(self) -> None: + self.assertEqual(self._job_name("RUNNER_NAME='bad name;rm'"), "collectivex") + + def test_salloc_submits_under_the_job_name(self) -> None: + common = (RUNTIME / "common.sh").read_text() + self.assertIn('set -- --job-name="$(collx_slurm_job_name)" "$@"', common) + + class StageTests(unittest.TestCase): def test_create_copy_and_validate_cleanup(self) -> None: with tempfile.TemporaryDirectory() as directory: From ca93f2015065b7d5f2adcb2d0989cda4838819bb Mon Sep 17 00:00:00 2001 From: Bryan Shan <58582368+Oseltamivir@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:50:39 -0700 Subject: [PATCH 2/3] remove --- .../CollectiveX/tests/test_runtime.py | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/experimental/CollectiveX/tests/test_runtime.py b/experimental/CollectiveX/tests/test_runtime.py index 43ff1ff205..7726c74703 100644 --- a/experimental/CollectiveX/tests/test_runtime.py +++ b/experimental/CollectiveX/tests/test_runtime.py @@ -145,33 +145,6 @@ def test_operator_config_registry_only_emits_tracked_baseline(self) -> None: self.assertIn(b"COLLX_SQUASH_DIR\0/home/sa-shared/containers\0", payload) self.assertIn(b"COLLX_RDMA_DEVICES\0", payload) -class SlurmJobNameTests(unittest.TestCase): - # inferencex-dash correlates cluster allocations with GHA jobs by Slurm job - # name == the runner_name (the production runners' --job-name="$RUNNER_NAME" - # convention), so collx_salloc_jobid must submit under that name and fall - # back to a fixed label rather than an unset or unsafe one. - @staticmethod - def _job_name(env_pairs: str) -> str: - completed = subprocess.run( - ["bash", "-c", - f"source runtime/common.sh 2>/dev/null; {env_pairs} collx_slurm_job_name"], - cwd=RUNTIME.parent, capture_output=True, text=True, check=True, - ) - return completed.stdout - - def test_runner_name_becomes_the_job_name(self) -> None: - self.assertEqual(self._job_name("RUNNER_NAME=mi355x-amds_03"), "mi355x-amds_03") - - def test_unset_runner_name_falls_back(self) -> None: - self.assertEqual(self._job_name("RUNNER_NAME="), "collectivex") - - def test_unsafe_runner_name_falls_back(self) -> None: - self.assertEqual(self._job_name("RUNNER_NAME='bad name;rm'"), "collectivex") - - def test_salloc_submits_under_the_job_name(self) -> None: - common = (RUNTIME / "common.sh").read_text() - self.assertIn('set -- --job-name="$(collx_slurm_job_name)" "$@"', common) - class StageTests(unittest.TestCase): def test_create_copy_and_validate_cleanup(self) -> None: From d27f05f6e54e40429b30d0309f7fad91947a666e Mon Sep 17 00:00:00 2001 From: Bryan Shan <58582368+Oseltamivir@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:51:06 -0700 Subject: [PATCH 3/3] rem --- experimental/CollectiveX/tests/test_runtime.py | 1 - 1 file changed, 1 deletion(-) diff --git a/experimental/CollectiveX/tests/test_runtime.py b/experimental/CollectiveX/tests/test_runtime.py index 7726c74703..3120879e01 100644 --- a/experimental/CollectiveX/tests/test_runtime.py +++ b/experimental/CollectiveX/tests/test_runtime.py @@ -145,7 +145,6 @@ def test_operator_config_registry_only_emits_tracked_baseline(self) -> None: self.assertIn(b"COLLX_SQUASH_DIR\0/home/sa-shared/containers\0", payload) self.assertIn(b"COLLX_RDMA_DEVICES\0", payload) - class StageTests(unittest.TestCase): def test_create_copy_and_validate_cleanup(self) -> None: with tempfile.TemporaryDirectory() as directory: