fix(orchestrator): bound LDAP job submission to prevent indefinite hang on down nodes

test/orchestrator/library/functions/_workload_helpers.py

@@ -295,3 +295,42 @@ def ensure_source_mpi(host, rows) -> Tuple[bool, str]:
 
     _source_mpi_provisioned = True
     return True, ""
+
+
+def mapped_compute_readiness(host, control, compute_rows):
+    """Return normalized scheduler readiness for every mapped compute."""
+    scheduler = remote_command(
+        host,
+        control,
+        PXEBOOT_COMMANDS["slurm_partitions"],
+    )
+    if scheduler.rc != 0:
+        raise RuntimeError("sinfo could not read mapped compute-node state")
+    node_states = {}
+    for line in scheduler.stdout.splitlines():
+        parts = [part.strip() for part in line.split("|", 3)]
+        if len(parts) != 4 or not parts[0]:
+            continue
+        node = parts[0].split(".", 1)[0]
+        state = parts[2].lower().split("+", 1)[0].rstrip("*~#$@!%^")
+        availability = parts[3].lower()
+        node_states.setdefault(node, []).append((state, availability))
+    readiness = []
+    for row in compute_rows:
+        states = node_states.get(row["HOSTNAME"], [])
+        idle = bool(states) and all(
+            state == "idle" and availability == "up" for state, availability in states
+        )
+        readiness.append(
+            {
+                "row": row,
+                "idle": idle,
+                "state": ", ".join(sorted({state for state, _availability in states}))
+                or "missing",
+                "availability": ", ".join(
+                    sorted({availability for _state, availability in states})
+                )
+                or "missing",
+            }
+        )
+    return readiness

test/orchestrator/library/functions/slurm_auth_pxeboot_func.py

@@ -39,6 +39,7 @@
     RECOVERY_POLL_SECONDS,
     SLURM_COMPILER_PREFIX,
     SLURM_CONTROL_PREFIX,
+    SLURM_JOB_READY_STATES,
     SLURM_JOB_TIMEOUT_SECONDS,
     SLURM_LOGIN_PREFIX,
 )
@@ -51,6 +52,7 @@
 )
 from ._workload_helpers import ldap_identity_skip as _ldap_identity_skip
 from ._workload_helpers import ldap_test_username as _ldap_username
+from ._workload_helpers import mapped_compute_readiness as _mapped_compute_readiness
 from ._workload_helpers import optional_skip as _skip
 from ._workload_helpers import require_functional as _require_functional
 from ._workload_helpers import slurm_compute_rows as _compute_rows
@@ -835,6 +837,19 @@ def _wait_job_state(host, control, job_id: str, expected: str) -> tuple[bool, st
     return False, last
 
 
+def _compute_not_ready_reason(readiness: dict) -> str:
+    """Return why a mapped compute node cannot run a targeted job soon ('' when it can).
+
+    A job pinned with --nodelist to a down, drained, unknown or unlisted node stays PENDING
+    (ReqNodeNotAvail), so such a node is reported instead of being submitted to.
+    """
+    states = {item.strip() for item in readiness["state"].split(",") if item.strip()}
+    availability = {item.strip() for item in readiness["availability"].split(",") if item.strip()}
+    if states <= set(SLURM_JOB_READY_STATES) and availability == {"up"}:
+        return ""
+    return f"state={readiness['state']}, availability={readiness['availability']}"
+
+
 def _check_slurm_role_pam_job_access(
     host,
     role_prefix,
@@ -1247,6 +1262,10 @@ def _check_slurm_role_ldap_jobs(
         if reason:
             return _skip(summary, reason)
         username = _ldap_username()
+        compute_readiness = {
+            item["row"]["HOSTNAME"]: item
+            for item in _mapped_compute_readiness(host, control, computes)
+        }
         workspace_info = _prepare_ldap_workspace(
             host,
             context,
@@ -1262,12 +1281,34 @@ def _check_slurm_role_ldap_jobs(
         for submit_row in submit_rows:
             target_results = []
             for compute_row in computes:
+                readiness = compute_readiness[compute_row["HOSTNAME"]]
+                not_ready = _compute_not_ready_reason(readiness)
+                if not_ready:
+                    failures.append(
+                        f"{submit_row['HOSTNAME']}->{compute_row['HOSTNAME']}"
+                    )
+                    target_results.append(
+                        {
+                            "row": compute_row,
+                            "job_id": "not submitted",
+                            "submitted": False,
+                            "state": readiness["state"],
+                            "completed": False,
+                            "allocated_node": "not allocated",
+                            "allocated_ok": False,
+                            "stdout": "none",
+                            "output_ok": False,
+                            "stderr": f"compute node is not ready: {not_ready}",
+                            "stderr_ok": False,
+                            "success": False,
+                        }
+                    )
+                    continue
                 command = _ldap_sbatch_command(
                     username,
                     workspace,
                     "hostname -s",
                     target_node=compute_row["HOSTNAME"],
-                    wait=True,
                 )
                 result = remote_command(host, submit_row, command)
                 output_lines = [

test/orchestrator/library/functions/slurm_jobs_pxeboot_func.py

@@ -44,6 +44,7 @@
     runtime_result,
 )
 from ._workload_helpers import ensure_source_mpi as _ensure_source_mpi
+from ._workload_helpers import mapped_compute_readiness as _mapped_compute_readiness
 from ._workload_helpers import optional_skip as _skip
 from ._workload_helpers import require_functional as _require_functional
 from ._workload_helpers import slurm_compute_rows as _compute_rows
@@ -98,45 +99,6 @@ def _wait_for_targeted_job_accounting(host, control, job_id):
     }
 
 
-def _mapped_compute_readiness(host, control, compute_rows):
-    """Return normalized scheduler readiness for every mapped compute."""
-    scheduler = remote_command(
-        host,
-        control,
-        PXEBOOT_COMMANDS["slurm_partitions"],
-    )
-    if scheduler.rc != 0:
-        raise RuntimeError("sinfo could not read mapped compute-node state")
-    node_states = {}
-    for line in scheduler.stdout.splitlines():
-        parts = [part.strip() for part in line.split("|", 3)]
-        if len(parts) != 4 or not parts[0]:
-            continue
-        node = parts[0].split(".", 1)[0]
-        state = parts[2].lower().split("+", 1)[0].rstrip("*~#$@!%^")
-        availability = parts[3].lower()
-        node_states.setdefault(node, []).append((state, availability))
-    readiness = []
-    for row in compute_rows:
-        states = node_states.get(row["HOSTNAME"], [])
-        idle = bool(states) and all(
-            state == "idle" and availability == "up" for state, availability in states
-        )
-        readiness.append(
-            {
-                "row": row,
-                "idle": idle,
-                "state": ", ".join(sorted({state for state, _availability in states}))
-                or "missing",
-                "availability": ", ".join(
-                    sorted({availability for _state, availability in states})
-                )
-                or "missing",
-            }
-        )
-    return readiness
-
-
 def _run_targeted_srun(host, control, submission_row, compute_row):
     """Run and attest one uniquely identified Slurm job on one compute node."""
     result = remote_command(

test/orchestrator/library/vars/pxeboot_vars.py

@@ -29,6 +29,9 @@
 RECOVERY_WAIT_TIMEOUT_SECONDS = 900
 RECOVERY_POLL_SECONDS = 15
 SLURM_JOB_TIMEOUT_SECONDS = 180
+# Node states in which a targeted job can start or queue for a bounded time; a node in any
+# other state (down, drained, unknown, missing) would keep the job pending (ReqNodeNotAvail).
+SLURM_JOB_READY_STATES = ("idle", "mix", "mixed", "alloc", "allocated", "comp", "completing")
 SLURM_ACCOUNTING_TIMEOUT_SECONDS = 30
 SLURM_ACCOUNTING_POLL_SECONDS = 2
 SLURM_TEST_JOB_COMMAND = "hostname -s"
@@ -498,7 +501,7 @@
         "sinfo --noheader --nodes=%s --format='%%T' | head -1"
     ),
     "slurm_resume_node": "scontrol update NodeName=%s State=RESUME",
-    "slurm_cancel_job": "scancel %s",
+    "slurm_cancel_job": "timeout 30 scancel %s",
     "slurm_sbatch": "sbatch --parsable --wait --wrap='hostname'",
     "slurm_job_accounting": (
         "sacct --noheader --parsable2 --jobs=%s --format=State | head -1"