fix(helm)!: grant operator-mode Secret permissions through the workspace chart

The operator-mode gateway ClusterRole grants no Secret permissions.
The openshell-workspace chart Role, installed in each operator-managed
namespace, grants the gateway create and delete on Secrets for sandbox
runtime generations. Operator-managed namespaces require the workspace
chart.

- Fail the chart tests on any ClusterRole rule that includes Secrets in
  operator and shared modes.
- Install the workspace chart when the operator e2e provisions a
  namespace, and assert that the gateway has no Secret permissions in a
  namespace without it.
- Update the Kubernetes setup docs, 0.1.0 upgrade guide, compute-runtime
  architecture, and cluster debugging skill.

Signed-off-by: Kris Hicks <khicks@nvidia.com>

Author: krishicks

architecture/compute-runtimes.md

@@ -522,6 +522,9 @@ the gateway ServiceAccount must not be shared with unrelated workloads.
 Operator mode does not create NetworkPolicies or copy image-pull Secrets.
 Platform teams must apply the gateway ingress boundary and provision configured
 image-pull Secrets in every operator-managed namespace.
+The gateway ClusterRole grants no Secret permissions in operator mode. The
+`openshell-workspace` chart Role installed in each operator-managed namespace
+grants bootstrap Secret `create` and `delete`.
 
 **Operator** uses pre-provisioned namespaces discovered through two optional
 sources: a K8s label selector (`operator_namespace_label`) and a drop-in

deploy/helm/openshell-workspace/templates/role.yaml

@@ -43,3 +43,11 @@ rules:
       - pods
     verbs:
       - get
+  # Generation-scoped sandbox bootstrap Secrets.
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - create
+      - delete

deploy/helm/openshell-workspace/tests/workspace_test.yaml

@@ -51,6 +51,18 @@ tests:
             resources: ["persistentvolumeclaims"]
             verbs: ["get"]
 
+  - it: grants generation Secret lifecycle and names no Secrets
+    template: templates/role.yaml
+    asserts:
+      - contains:
+          path: rules
+          content:
+            apiGroups: [""]
+            resources: ["secrets"]
+            verbs: ["create", "delete"]
+      - notExists:
+          path: rules[*].resourceNames
+
   - it: binds the shared gateway service account
     template: templates/rolebinding.yaml
     set:

deploy/helm/openshell/templates/clusterrole.yaml

@@ -93,14 +93,18 @@ rules:
   - apiGroups: [""]
     resources: ["services"]
     verbs: ["create", "get"]
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["networkpolicies"]
+    verbs: ["create", "get"]
+  {{- end }}
+  {{- if eq $workspaceMode "managed" }}
+  # Operator-mode namespaces receive Secret permissions from the openshell-workspace
+  # chart Role instead.
   - apiGroups: [""]
     resources: ["secrets"]
     # Bootstrap and image-pull Secrets are generation-scoped. Recovery deletes
     # them by exact name.
     verbs: ["create", "delete"]
-  - apiGroups: ["networking.k8s.io"]
-    resources: ["networkpolicies"]
-    verbs: ["create", "get"]
   {{- end }}
   {{- if eq $workspaceMode "managed" }}
   # ServiceAccount creation in managed namespaces.

deploy/helm/openshell/tests/clusterrole_test.yaml

@@ -91,43 +91,14 @@ tests:
             resources: ["secrets"]
             verbs: ["get", "create", "patch", "delete"]
 
-  - it: omits credential Secret access in operator mode
-    set:
-      server.drivers.kubernetes.workspaceMode: operator
-      server.credentialDrivers.kubernetesSecrets.enabled: true
-    asserts:
-      - notContains:
-          path: rules
-          content:
-            apiGroups: [""]
-            resources: ["secrets"]
-            verbs: ["get", "create", "patch", "delete"]
-
-  - it: names no Secrets in operator mode
+  - it: omits all Secret rules in operator mode
     set:
       server.drivers.kubernetes.workspaceMode: operator
       server.tls.clientTlsSecretName: custom-client-tls
+      server.credentialDrivers.kubernetesSecrets.enabled: true
     asserts:
       - notExists:
-          path: rules[*].resourceNames
-
-  - it: grants operator bootstrap secret lifecycle permissions when TLS and credential storage are disabled
-    set:
-      server.drivers.kubernetes.workspaceMode: operator
-      server.disableTls: true
-    asserts:
-      - contains:
-          path: rules
-          content:
-            apiGroups: [""]
-            resources: ["secrets"]
-            verbs: ["create", "delete"]
-      - notContains:
-          path: rules
-          content:
-            apiGroups: [""]
-            resources: ["secrets"]
-            verbs: ["get", "create", "patch", "delete"]
+          path: rules[*].resources[?(@ == "secrets")]
 
   - it: names no Secrets in managed mode
     set:
@@ -150,12 +121,8 @@ tests:
     set:
       server.drivers.kubernetes.workspaceMode: shared
     asserts:
-      - notContains:
-          path: rules
-          content:
-            apiGroups: [""]
-            resources: ["secrets"]
-          any: true
+      - notExists:
+          path: rules[*].resources[?(@ == "secrets")]
 
   - it: grants managed sandbox-runtime companion permissions
     set:
@@ -196,9 +163,3 @@ tests:
             apiGroups: [""]
             resources: ["services"]
             verbs: ["create", "get"]
-      - contains:
-          path: rules
-          content:
-            apiGroups: [""]
-            resources: ["secrets"]
-            verbs: ["create", "delete"]

docs/kubernetes/setup.mdx

@@ -130,7 +130,8 @@ release namespace. For one pre-provisioned namespace, keep
 `server.sandboxNamespace=app-a`. To map multiple workspaces to separately
 provisioned namespaces, use `workspaceMode=operator`, configure exactly one of
 `operatorNamespaceLabel` or `operatorNamespaceFile`, and install the workspace
-chart in every allowlisted namespace.
+chart in every allowlisted namespace. In operator mode, the workspace chart
+Role is the gateway's only Secret permission grant in that namespace.
 
 If you enable `server.drivers.kubernetes.allowDriverConfig` on the gateway,
 also set `gateway.allowDriverConfig=true` on every workspace release. This

docs/upgrade/0-1-0.mdx

@@ -59,6 +59,8 @@ If you run OpenShell for a team, start here.
 
 - **Recreate Kubernetes Secrets provider credentials.** The Kubernetes Secrets credential driver stores every provider credential in its configured `namespace` in every workspace mode. Remove the `workspace_mode`, `gateway_id`, and `allow_reference_namespace` driver settings; a `gateway.toml` that sets them is rejected at startup. Credentials stored by the driver are not migrated. Deploy a fresh gateway, then create providers and refresh grants again ([PR #3616](https://github.com/NVIDIA/OpenShell/pull/3616)).
 
+- **Install the workspace chart in operator-managed namespaces.** In operator workspace mode, the gateway receives Secret permissions only from the `openshell-workspace` chart. Install the matching chart release in every operator-managed namespace; without it, sandbox bootstrap fails ([PR #3616](https://github.com/NVIDIA/OpenShell/pull/3616)).
+
 - **Implement extension protocol negotiation.** Custom compute drivers, credential drivers, gateway interceptors, and middleware must exchange `PeerMetadata`, use protocol `1.0`, and advertise their family base capability. Upgrade both peers together. See [Extension Protocol Negotiation](/extensibility/extension-negotiation) and [PR #3352](https://github.com/NVIDIA/OpenShell/pull/3352).
 
 - **Remove compute-driver callback-listener negotiation.** Regenerate custom compute-driver bindings and connect supervisors to the operator-configured primary gateway endpoint ([PR #3365](https://github.com/NVIDIA/OpenShell/pull/3365)).

e2e/rust/tests/workspace_namespace_operator.rs

@@ -88,8 +88,37 @@ async fn provision_operator_namespace(name: &str) {
     let (ok, out) = kubectl(&["label", "namespace", name, OPERATOR_LABEL]).await;
     assert!(ok, "failed to label namespace {name}: {out}");
 
-    let (ok, out) = kubectl(&["create", "serviceaccount", SA_NAME, "-n", name]).await;
-    assert!(ok, "failed to create SA in {name}: {out}");
+    let (ok, out) = install_workspace_chart(name).await;
+    assert!(ok, "failed to install workspace chart in {name}: {out}");
+}
+
+async fn install_workspace_chart(namespace: &str) -> (bool, String) {
+    let chart = concat!(
+        env!("CARGO_MANIFEST_DIR"),
+        "/../../deploy/helm/openshell-workspace"
+    );
+    let output = tokio::process::Command::new("helm")
+        .args([
+            "--kube-context",
+            &kube_context(),
+            "install",
+            "openshell-workspace",
+            chart,
+            "--namespace",
+            namespace,
+            "--wait",
+        ])
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .output()
+        .await
+        .expect("failed to spawn helm");
+    let combined = format!(
+        "{}{}",
+        String::from_utf8_lossy(&output.stdout),
+        String::from_utf8_lossy(&output.stderr),
+    );
+    (output.status.success(), combined)
 }
 
 async fn delete_namespace(name: &str) {
@@ -143,6 +172,43 @@ impl Drop for OperatorCleanup {
     }
 }
 
+const GATEWAY_SERVICE_ACCOUNT: &str = "system:serviceaccount:openshell:openshell";
+
+async fn gateway_can(verb: &str, resource: &str, namespace: &str) -> bool {
+    let (_, out) = kubectl(&[
+        "auth",
+        "can-i",
+        verb,
+        resource,
+        "-n",
+        namespace,
+        "--as",
+        GATEWAY_SERVICE_ACCOUNT,
+    ])
+    .await;
+    match out.trim() {
+        "yes" => true,
+        "no" => false,
+        other => panic!("unexpected kubectl auth can-i output for {verb} {resource}: {other}"),
+    }
+}
+
+#[tokio::test]
+async fn operator_gateway_has_no_secret_access_without_workspace_chart() {
+    let ns = unique_namespace("oprbac");
+
+    for verb in ["get", "list", "create", "patch", "delete"] {
+        assert!(
+            !gateway_can(verb, "secrets", &ns).await,
+            "gateway must not {verb} Secrets in namespace {ns} without the workspace chart"
+        );
+    }
+    assert!(
+        !gateway_can("patch", "secrets/openshell-client-tls", &ns).await,
+        "gateway must not patch the client TLS Secret in namespace {ns} without the workspace chart"
+    );
+}
+
 #[tokio::test]
 async fn operator_sandbox_in_labeled_namespace() {
     let ns = unique_namespace("op");

skills/debug-openshell-cluster/SKILL.md

@@ -767,8 +767,10 @@ workload.
 
 If a Sandbox remains in the `suspending` bootstrap phase, verify that the
 gateway ServiceAccount can create and delete Secrets in the sandbox
-namespace. Recovery deletes the recorded generation's bootstrap Secrets by name
-before clearing the suspension annotations:
+namespace. In operator mode, those permissions come only from the
+`openshell-workspace` chart installed in the namespace. Recovery deletes the
+recorded generation's bootstrap Secrets by name before clearing the suspension
+annotations:
 
 ```bash
 for verb in create delete; do