fix(server): keep exposed services off infrastructure errors

Exposed services do not authenticate callers, so they now reach an Error
sandbox only when its main process failed. Exec, SSH, and forwarding keep
the debugging path into an infrastructure Error.

The CLI service forward stops again once its sandbox is stopping, stopped,
or being deleted, and the TypeScript SDK forward applies the same check
instead of requiring Ready. Exec refusals name what makes the sandbox
usable again.

supervisor_connected skips the owner index on a single replica, the
unused relay error variant is gone, and relay opening outside
sandbox_access is crate-private.

Signed-off-by: Piotr Mlocek <pmlocek@nvidia.com>

Author: pimlock

crates/openshell-cli/src/run.rs

@@ -2260,10 +2260,12 @@ fn forward_requires_session_token(status: &Status) -> bool {
         && status.message().contains("authorization_token is required")
 }
 
-/// Confirm the sandbox still exists, returning the replica that owns it.
+/// Confirm the sandbox can still serve the forward, returning the replica that
+/// owns it.
 ///
 /// The gateway decides whether each connection may reach the sandbox; this
-/// only stops a forward whose sandbox is gone or going away.
+/// only stops a forward whose sandbox is gone or is stopping, stopped, or
+/// being deleted, since no connection can succeed until a new command acts.
 async fn fetch_sandbox_for_forward(
     client: &mut crate::tls::GrpcClient,
     name: &str,
@@ -2293,15 +2295,25 @@ async fn fetch_sandbox_for_forward(
         .sandbox
         .ok_or_else(|| miette::miette!("sandbox '{name}' not found"))?;
 
-    if SandboxPhase::try_from(sandbox.phase()) == Ok(SandboxPhase::Deleting) {
+    if forward_ended_by_phase(sandbox.phase()) {
         return Err(miette::miette!(
-            "sandbox '{name}' is being deleted; stopping service forward"
+            "sandbox '{name}' is no longer available (phase: {}); stopping service forward",
+            phase_name(sandbox.phase())
         ));
     }
 
     Ok(replica)
 }
 
+/// Phases in which no forwarded connection can succeed until another command
+/// starts or recreates the sandbox.
+fn forward_ended_by_phase(phase: i32) -> bool {
+    matches!(
+        SandboxPhase::try_from(phase),
+        Ok(SandboxPhase::Stopping | SandboxPhase::Stopped | SandboxPhase::Deleting)
+    )
+}
+
 #[derive(Debug)]
 struct ForwardTcpConnectionError {
     message: String,
@@ -2569,12 +2581,21 @@ impl Drop for TaskGuard {
 }
 
 /// The gateway decides whether a sandbox can be used. When it refuses, name the
-/// phase this command saw so the user knows what to wait for.
+/// phase this command saw and what would make the sandbox usable again.
 fn exec_start_error(status: Status, sandbox: &Sandbox) -> miette::Report {
     if status.code() == Code::FailedPrecondition && status.message() == "sandbox is not ready" {
+        let name = sandbox.object_name();
+        let hint = match SandboxPhase::try_from(sandbox.phase()) {
+            Ok(SandboxPhase::Stopping | SandboxPhase::Stopped) => {
+                format!("start it with `openshell sandbox start {name}`")
+            }
+            Ok(SandboxPhase::Completed | SandboxPhase::Error) => {
+                "its main process exited and its supervisor is no longer connected".to_string()
+            }
+            _ => "wait for it to reach Ready state".to_string(),
+        };
         return miette::miette!(
-            "sandbox '{}' is not ready (phase: {}); wait for it to reach Ready state",
-            sandbox.object_name(),
+            "sandbox '{name}' is not ready (phase: {}); {hint}",
             phase_name(sandbox.phase())
         );
     }
@@ -7129,14 +7150,14 @@ fn format_endpoint(endpoint: &openshell_core::proto::NetworkEndpoint) -> String
 mod tests {
     use super::{
         ForwardTcpConnectionError, PolicyGetView, ProvisioningStep, build_sandbox_resource_limits,
-        format_endpoint, format_log_line, forward_requires_session_token, git_sync_files,
-        has_main_process_result, parse_cli_setting_value, parse_credential_expiry_cli_value,
-        parse_driver_config_json, parse_env_from_pairs, parse_secret_material_env_pairs,
-        policy_revision_list_json, policy_revision_to_json, proto_execution_timeout,
-        provisioning_timeout_message, ready_false_condition_message, relay_local_socket,
-        resolve_from, rootfs_tar_sources_supported_for_gateway, sandbox_should_persist,
-        sandbox_upload_plan, service_endpoint_to_json, service_status_error,
-        service_url_for_gateway, workspace_member_to_json,
+        format_endpoint, format_log_line, forward_ended_by_phase, forward_requires_session_token,
+        git_sync_files, has_main_process_result, parse_cli_setting_value,
+        parse_credential_expiry_cli_value, parse_driver_config_json, parse_env_from_pairs,
+        parse_secret_material_env_pairs, policy_revision_list_json, policy_revision_to_json,
+        proto_execution_timeout, provisioning_timeout_message, ready_false_condition_message,
+        relay_local_socket, resolve_from, rootfs_tar_sources_supported_for_gateway,
+        sandbox_should_persist, sandbox_upload_plan, service_endpoint_to_json,
+        service_status_error, service_url_for_gateway, workspace_member_to_json,
     };
     use openshell_core::proto::TcpForwardFrame;
 
@@ -9503,4 +9524,27 @@ mod tests {
             "authorization_token is required for ForwardTcp"
         )));
     }
+
+    #[test]
+    fn forward_leaves_reachability_after_exit_to_the_gateway() {
+        use openshell_core::proto::SandboxPhase;
+
+        for phase in [
+            SandboxPhase::Stopping,
+            SandboxPhase::Stopped,
+            SandboxPhase::Deleting,
+        ] {
+            assert!(forward_ended_by_phase(phase as i32), "{phase:?}");
+        }
+        for phase in [
+            SandboxPhase::Provisioning,
+            SandboxPhase::Starting,
+            SandboxPhase::Ready,
+            SandboxPhase::Completed,
+            SandboxPhase::Error,
+            SandboxPhase::Suspended,
+        ] {
+            assert!(!forward_ended_by_phase(phase as i32), "{phase:?}");
+        }
+    }
 }

crates/openshell-cli/tests/sandbox_exec_streaming_integration.rs

@@ -528,8 +528,16 @@ async fn exec_names_the_phase_when_the_gateway_refuses() {
     drop(child.stdin.take());
     let output = finish(child).await;
     assert!(!output.status.success());
-    let stderr = String::from_utf8_lossy(&output.stderr);
-    assert!(stderr.contains("is not ready (phase: Stopped)"), "{stderr}");
+    // The error renderer wraps long messages, so compare the words alone.
+    let stderr = String::from_utf8_lossy(&output.stderr)
+        .split_whitespace()
+        .filter(|word| *word != "│")
+        .collect::<Vec<_>>()
+        .join(" ");
+    assert!(
+        stderr.contains("is not ready (phase: Stopped); start it with `openshell sandbox start"),
+        "{stderr}"
+    );
     assert!(gateway.calls.lock().unwrap().starts.is_empty());
 }
 

crates/openshell-server/src/compute/mod.rs

@@ -6879,7 +6879,7 @@ fn terminal_stop_exit_code(sandbox: &Sandbox) -> Option<i32> {
     sandbox.status.as_ref()?.exit_code
 }
 
-fn is_failed_main_process_result(sandbox: &Sandbox) -> bool {
+pub fn is_failed_main_process_result(sandbox: &Sandbox) -> bool {
     sandbox.phase() == SandboxPhase::Error as i32
         && sandbox.status.as_ref().is_some_and(|status| {
             status.exit_code.is_some()

crates/openshell-server/src/grpc/sandbox.rs

@@ -2853,8 +2853,9 @@ pub(super) async fn handle_forward_tcp(
     .await?;
 
     // The main process may finish between minting the SSH token and opening
-    // its transport. Keep the relay reachable until terminal delivery is
-    // finalized so fast commands can attach without a readiness race.
+    // its transport. The access rules keep a Completed or Error sandbox
+    // reachable while its supervisor is connected, so fast commands attach
+    // without a readiness race.
     let access = crate::sandbox_access::acquire(state, &sandbox).await?;
 
     let connection_guard =

crates/openshell-server/src/sandbox_access.rs

@@ -17,6 +17,9 @@
 //! - A `Completed` or `Error` sandbox is usable while its supervisor is still
 //!   connected to any replica. A retained sandbox keeps its access plane after
 //!   its main process exits.
+//! - Exposed services, which do not authenticate callers, reach an `Error`
+//!   sandbox only when its main process failed. An infrastructure failure
+//!   stays reachable for debugging through the authenticated paths only.
 //! - Every other phase is refused.
 
 use std::fmt;
@@ -42,16 +45,13 @@ pub enum AccessError {
     NotReachable,
     /// The gateway could not determine whether the sandbox is reachable.
     Lookup(Status),
-    /// The supervisor did not open the relay.
-    Relay(Status),
 }
 
 impl fmt::Display for AccessError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             Self::NotReachable => f.write_str("sandbox is not ready"),
             Self::Lookup(status) => write!(f, "sandbox access check failed: {status}"),
-            Self::Relay(status) => write!(f, "supervisor relay failed: {status}"),
         }
     }
 }
@@ -61,7 +61,6 @@ impl From<AccessError> for Status {
         match err {
             AccessError::NotReachable => Self::failed_precondition(err.to_string()),
             AccessError::Lookup(status) => status,
-            AccessError::Relay(_) => Self::unavailable(err.to_string()),
         }
     }
 }
@@ -95,14 +94,28 @@ pub async fn acquire(
     })
 }
 
+/// Apply the access rules for an exposed service, which serves callers the
+/// gateway does not authenticate.
+pub async fn acquire_for_exposed_service(
+    state: &Arc<ServerState>,
+    sandbox: &Sandbox,
+) -> Result<SandboxAccess, AccessError> {
+    if sandbox.phase() == SandboxPhase::Error as i32
+        && !crate::compute::is_failed_main_process_result(sandbox)
+    {
+        return Err(AccessError::NotReachable);
+    }
+    acquire(state, sandbox).await
+}
+
 impl SandboxAccess {
     /// Ask the supervisor to open a relay to `target`, returning the channel
     /// id and a receiver that resolves once the supervisor claims it.
     pub async fn open_relay(
         &self,
         target: relay_open::Target,
         service_id: String,
-    ) -> Result<(String, oneshot::Receiver<Result<DuplexStream, Status>>), AccessError> {
+    ) -> Result<(String, oneshot::Receiver<Result<DuplexStream, Status>>), Status> {
         crate::supervisor_session::open_routed_relay_with_target(
             &self.state,
             &self.sandbox_id,
@@ -111,7 +124,7 @@ impl SandboxAccess {
             RELAY_SESSION_WAIT,
         )
         .await
-        .map_err(AccessError::Relay)
+        .map_err(|err| Status::unavailable(format!("supervisor relay failed: {err}")))
     }
 }
 
@@ -187,6 +200,54 @@ mod tests {
         }
     }
 
+    fn error_sandbox(id: &str, reason: &str, exit_code: Option<i32>) -> Sandbox {
+        let mut sandbox = sandbox_in(id, SandboxPhase::Error);
+        sandbox.status = Some(openshell_core::proto::SandboxStatus {
+            phase: SandboxPhase::Error.into(),
+            exit_code,
+            conditions: vec![openshell_core::proto::SandboxCondition {
+                r#type: "Ready".into(),
+                status: "False".into(),
+                reason: reason.into(),
+                ..Default::default()
+            }],
+            ..Default::default()
+        });
+        sandbox
+    }
+
+    #[tokio::test]
+    async fn exposed_services_skip_infrastructure_errors() {
+        let state = crate::grpc::test_support::test_server_state().await;
+        let (tx, _rx) = mpsc::channel(1);
+        let (shutdown_tx, _shutdown_rx) = oneshot::channel();
+        state.supervisor_sessions.register(
+            "access".to_string(),
+            "session".to_string(),
+            tx,
+            shutdown_tx,
+        );
+        let main_failed = error_sandbox("access", "MainProcessFailed", Some(3));
+        let timed_out = error_sandbox("access", "ProvisioningTimedOut", None);
+
+        assert!(
+            acquire_for_exposed_service(&state, &main_failed)
+                .await
+                .is_ok()
+        );
+        assert!(matches!(
+            acquire_for_exposed_service(&state, &timed_out).await,
+            Err(AccessError::NotReachable)
+        ));
+        // Authenticated paths keep the debugging shell.
+        assert!(acquire(&state, &timed_out).await.is_ok());
+        assert!(
+            acquire_for_exposed_service(&state, &sandbox_in("access", SandboxPhase::Completed))
+                .await
+                .is_ok()
+        );
+    }
+
     #[test]
     fn not_reachable_keeps_the_existing_status() {
         let status = Status::from(AccessError::NotReachable);

crates/openshell-server/src/service_routing.rs

@@ -730,12 +730,12 @@ async fn proxy_to_endpoint(
     };
     // Every request passes the access gate, including ones served from a
     // pooled upstream connection.
-    let access = match sandbox_access::acquire(&state, &sandbox).await {
+    let access = match sandbox_access::acquire_for_exposed_service(&state, &sandbox).await {
         Ok(access) => access,
         Err(err) => {
             let route_err = match err {
                 AccessError::NotReachable => ServiceRouteError::sandbox_not_ready(),
-                AccessError::Lookup(err) | AccessError::Relay(err) => {
+                AccessError::Lookup(err) => {
                     warn!(error = %err, sandbox_id = %endpoint.sandbox_id, "sandbox service routing: failed to check sandbox access");
                     ServiceRouteError::internal_error()
                 }