fix(ci): recover relay opens and isolate VM proxy ports

Signed-off-by: Drew Newberry <anewberry@nvidia.com>

Author: drew

architecture/gateway.md

@@ -405,6 +405,8 @@ the serving gateway retries ownership lookup until the normal relay wait
 deadline. Each retry re-reads the owner record, so a supervisor reconnect or
 heartbeat can surface a new owner; if no fresh reachable owner appears before
 the deadline, the client operation fails rather than electing an owner itself.
+The owning gateway replays unclaimed relay opens when a supervisor reconnects,
+including when the previous session ended before the new one registered.
 Provider-readiness reports, endpoint-status reports, and provider-status reads
 also follow the durable owner record through unary peer RPCs. The owning replica
 validates the current supervisor session and keeps the in-memory evidence; a

crates/openshell-server/src/supervisor_session.rs

@@ -933,7 +933,7 @@ impl SupervisorSessionRegistry {
                 payload: Some(gateway_message::Payload::RelayOpen(relay_open)),
             };
             if tx.send(msg).await.is_err() {
-                warn!(sandbox_id = %sandbox_id, channel_id = %channel_id, "supervisor session: failed to replay pending relay to superseding session");
+                warn!(sandbox_id = %sandbox_id, channel_id = %channel_id, "supervisor session: failed to replay pending relay to new session");
                 break;
             }
         }
@@ -1960,12 +1960,12 @@ async fn establish_supervisor_session(
     }
     state.telemetry.sandbox_session_connected(&sandbox_id);
 
-    if superseded {
-        state
-            .supervisor_sessions
-            .replay_pending_relays(&sandbox_id, &tx)
-            .await;
-    }
+    // The previous session may already have been removed before this one
+    // registers. Pending relay opens still need to reach the new session.
+    state
+        .supervisor_sessions
+        .replay_pending_relays(&sandbox_id, &tx)
+        .await;
 
     // Step 4: Spawn the session loop that reads inbound messages.
     let state_clone = Arc::clone(&state);
@@ -2971,6 +2971,52 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn replay_pending_relays_reissues_open_after_disconnected_session() {
+        let registry = SupervisorSessionRegistry::new();
+        let (tx_old, mut rx_old) = mpsc::channel::<GatewayMessage>(4);
+        let (tx_new, mut rx_new) = mpsc::channel::<GatewayMessage>(4);
+
+        registry.register(
+            "sbx".to_string(),
+            "s-old".to_string(),
+            tx_old,
+            make_shutdown(),
+        );
+        let (channel_id, _relay_rx) = registry
+            .open_relay("sbx", Duration::from_secs(1))
+            .await
+            .expect("open_relay should succeed");
+        rx_old
+            .recv()
+            .await
+            .expect("old session should receive RelayOpen");
+
+        assert_eq!(registry.remove_if_current("sbx", "s-old"), Some(false));
+        let superseded = registry.register(
+            "sbx".to_string(),
+            "s-new".to_string(),
+            tx_new,
+            make_shutdown(),
+        );
+        assert!(!superseded, "old session was removed before reconnect");
+
+        registry
+            .replay_pending_relays("sbx", &registry.lookup_session("sbx").unwrap())
+            .await;
+
+        let replayed = rx_new
+            .recv()
+            .await
+            .expect("new session should receive RelayOpen");
+        match replayed.payload {
+            Some(gateway_message::Payload::RelayOpen(open)) => {
+                assert_eq!(open.channel_id, channel_id);
+            }
+            other => panic!("expected RelayOpen, got {other:?}"),
+        }
+    }
+
     #[tokio::test]
     async fn require_persisted_sandbox_rejects_missing_sandbox() {
         let store = test_store().await;

e2e/rust/tests/vm_corporate_proxy.rs

@@ -54,6 +54,7 @@ const BYPASS_MARKER: &str = "vm-corp-proxy-e2e-bypass-upstream";
 const READY_MARKER: &str = "vm-corp-proxy-e2e-workload-done";
 
 /// Ports the fixtures bind and the guest addresses them by.
+#[derive(Debug)]
 struct FixturePorts {
     proxy: u16,
     allowed: u16,
@@ -63,11 +64,21 @@ struct FixturePorts {
 
 impl FixturePorts {
     fn pick() -> Self {
+        // find_free_port releases its listener before returning, so repeated
+        // calls can return the same port and accidentally authorize the
+        // "denied" fixture through an allowed endpoint.
+        let mut used = std::collections::HashSet::new();
+        let mut next = || loop {
+            let port = find_free_port();
+            if used.insert(port) {
+                break port;
+            }
+        };
         Self {
-            proxy: find_free_port(),
-            allowed: find_free_port(),
-            denied: find_free_port(),
-            bypass: find_free_port(),
+            proxy: next(),
+            allowed: next(),
+            denied: next(),
+            bypass: next(),
         }
     }
 }
@@ -424,7 +435,6 @@ filesystem_policy:
   read_only:
     - /usr
     - /lib
-    - /proc
     - /dev/urandom
     - /app
     - /etc
@@ -433,6 +443,7 @@ filesystem_policy:
     - /sandbox
     - /tmp
     - /dev/null
+    - /proc
 
 landlock:
   compatibility: best_effort
@@ -822,7 +833,8 @@ async fn vm_corporate_proxy_trusts_ca_bundle_for_https_proxy() {
     );
     assert!(
         !proxy_logs.contains(&format!(":{}", ports.denied)),
-        "policy-denied destination must never reach the https proxy:\n{proxy_logs}"
+        "policy-denied destination must never reach the https proxy; ports={ports:?}, workload output:\n{}\nproxy logs:\n{proxy_logs}",
+        sandbox.create_output,
     );
 
     sandbox.cleanup().await;