ateom gvisor: drop privileged: true

cmd/atecontroller/internal/controllers/workerpool_apply.go

@@ -34,10 +34,7 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool) *appsv1ac.Deployment
 		WithArgs(
 			"--pod-uid=$(POD_UID)",
 		).
-		WithSecurityContext(corev1ac.SecurityContext().
-			WithPrivileged(true).
-			WithRunAsUser(0).
-			WithRunAsGroup(0)).
+		WithSecurityContext(ateomSecurityContext(wp.Spec.SandboxClass)).
 		WithEnv(
 			corev1ac.EnvVar().
 				WithName("POD_UID").
@@ -82,6 +79,46 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool) *appsv1ac.Deployment
 				WithSpec(podSpecAC)))
 }
 
+// ateomGvisorCapabilities is the capability set an unprivileged gVisor worker
+// needs. runsc's gofer maps a full-range identity in a user namespace
+// (SETUID/SETGID/SETPCAP/SETFCAP), the sandbox pivots root and traces the
+// application (SYS_ADMIN/SYS_CHROOT/SYS_PTRACE), actor networking programs the
+// veth and nftables rules (NET_ADMIN/NET_RAW), and the OCI rootfs is unpacked
+// and device nodes created as root over image-owned trees
+// (DAC_OVERRIDE/FOWNER/CHOWN/MKNOD). This replaces the former privileged worker;
+// the default seccomp and AppArmor profiles are sufficient (no Unconfined).
+var ateomGvisorCapabilities = []corev1.Capability{
+	"NET_ADMIN", "SYS_ADMIN", "SYS_CHROOT", "SYS_PTRACE",
+	"SETUID", "SETGID", "SETPCAP", "DAC_OVERRIDE",
+	"FOWNER", "CHOWN", "MKNOD", "NET_RAW", "SETFCAP",
+}
+
+// ateomSecurityContext returns the ateom container security context for a sandbox
+// class. The gVisor worker runs unprivileged with an explicit capability set; the
+// micro-VM worker stays privileged because kata + cloud-hypervisor needs broad
+// host access (vhost devices, mounts). An empty class defaults to gVisor.
+func ateomSecurityContext(class atev1alpha1.SandboxClass) *corev1ac.SecurityContextApplyConfiguration {
+	sc := corev1ac.SecurityContext().
+		WithRunAsUser(0).
+		WithRunAsGroup(0)
+	if class == atev1alpha1.SandboxClassMicroVM {
+		return sc.WithPrivileged(true)
+	}
+	// runsc mounts and pivots root inside the sandbox, and the worker remounts
+	// /sys/fs/cgroup read-write to nest per-actor cgroups; the default AppArmor
+	// profile denies those mounts (enforced on GKE COS, a no-op on nodes that do
+	// not load AppArmor). A privileged worker got this implicitly; unprivileged
+	// must request it. seccomp stays at the runtime default. Confining runsc with
+	// a tailored AppArmor profile instead of Unconfined is a follow-up.
+	return sc.
+		WithPrivileged(false).
+		WithCapabilities(corev1ac.Capabilities().
+			WithDrop("ALL").
+			WithAdd(ateomGvisorCapabilities...)).
+		WithAppArmorProfile(corev1ac.AppArmorProfile().
+			WithType(corev1.AppArmorProfileTypeUnconfined))
+}
+
 // maybeApplyMicroVMPodShape adds the /dev/kvm device and node placement a
 // micro-VM (kata + cloud-hypervisor) worker pool needs, on top of any
 // pod-template settings. No-op unless sandboxClass is the micro-VM class.

cmd/atecontroller/internal/controllers/workerpool_apply_test.go

@@ -261,6 +261,54 @@ func TestMicroVMPodShape(t *testing.T) {
 	}
 }
 
+// TestAteomSecurityContextByClass asserts the gVisor worker runs unprivileged
+// with the explicit capability set while the micro-VM worker stays privileged,
+// and that an empty class defaults to gVisor.
+func TestAteomSecurityContextByClass(t *testing.T) {
+	tests := []struct {
+		name           string
+		class          atev1alpha1.SandboxClass
+		wantPrivileged bool
+		wantCaps       bool
+	}{
+		{"gvisor default", "", false, true},
+		{"gvisor explicit", atev1alpha1.SandboxClassGvisor, false, true},
+		{"microvm", atev1alpha1.SandboxClassMicroVM, true, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sc := ateomSecurityContext(tt.class)
+			if sc.Privileged == nil || *sc.Privileged != tt.wantPrivileged {
+				t.Errorf("Privileged = %v, want %v", sc.Privileged, tt.wantPrivileged)
+			}
+			if sc.RunAsUser == nil || *sc.RunAsUser != 0 || sc.RunAsGroup == nil || *sc.RunAsGroup != 0 {
+				t.Errorf("RunAsUser/Group = %v/%v, want 0/0", sc.RunAsUser, sc.RunAsGroup)
+			}
+			hasCaps := sc.Capabilities != nil && len(sc.Capabilities.Add) > 0
+			if hasCaps != tt.wantCaps {
+				t.Errorf("has capabilities = %v, want %v", hasCaps, tt.wantCaps)
+			}
+			if tt.wantCaps {
+				if len(sc.Capabilities.Drop) != 1 || sc.Capabilities.Drop[0] != "ALL" {
+					t.Errorf("capabilities drop = %v, want [ALL]", sc.Capabilities.Drop)
+				}
+				if diff := cmp.Diff(ateomGvisorCapabilities, sc.Capabilities.Add); diff != "" {
+					t.Errorf("capabilities add mismatch (-want +got):\n%s", diff)
+				}
+			}
+			// The gVisor worker runs AppArmor-unconfined (runsc + cgroup remount
+			// need mount); the privileged micro-VM worker leaves it unset.
+			wantAppArmor := tt.wantCaps
+			hasAppArmor := sc.AppArmorProfile != nil &&
+				sc.AppArmorProfile.Type != nil &&
+				*sc.AppArmorProfile.Type == corev1.AppArmorProfileTypeUnconfined
+			if hasAppArmor != wantAppArmor {
+				t.Errorf("AppArmor Unconfined = %v, want %v", hasAppArmor, wantAppArmor)
+			}
+		})
+	}
+}
+
 func testWorkerPoolApplyConfig(tmpl *atev1alpha1.WorkerPoolPodTemplate) *atev1alpha1.WorkerPool {
 	return &atev1alpha1.WorkerPool{
 		ObjectMeta: metav1.ObjectMeta{Name: "pool", Namespace: "default", UID: "uid"},
@@ -289,9 +337,14 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf
 			WithImage(wp.Spec.AteomImage).
 			WithArgs("--pod-uid=$(POD_UID)").
 			WithSecurityContext(corev1ac.SecurityContext().
-				WithPrivileged(true).
 				WithRunAsUser(0).
-				WithRunAsGroup(0)).
+				WithRunAsGroup(0).
+				WithPrivileged(false).
+				WithCapabilities(corev1ac.Capabilities().
+					WithDrop("ALL").
+					WithAdd(ateomGvisorCapabilities...)).
+				WithAppArmorProfile(corev1ac.AppArmorProfile().
+					WithType(corev1.AppArmorProfileTypeUnconfined))).
 			WithEnv(corev1ac.EnvVar().
 				WithName("POD_UID").
 				WithValueFrom(corev1ac.EnvVarSource().

cmd/ateom-gvisor/main.go

@@ -25,6 +25,7 @@ import (
 	"os"
 	"runtime"
 	"sort"
+	"strings"
 	"sync"
 
 	"cloud.google.com/go/compute/metadata"
@@ -109,6 +110,12 @@ func do(ctx context.Context) error {
 		return fmt.Errorf("in os.MkdirAll(%q): %w", ateomDir, err)
 	}
 
+	// Prepare the pod cgroup so runsc can create per-actor-container leaves under
+	// it with real accounting, instead of ignoring cgroups entirely.
+	if err := setupCgroupDelegation(ctx); err != nil {
+		return fmt.Errorf("while setting up cgroup delegation: %w", err)
+	}
+
 	// TODO: Consider whether we want to fork, so that we have an "init" process
 	// as PID 1 that does nothing but reap processes that get reparented to it.
 	// Then we won't have to mess about with locking the reaper while we do our
@@ -663,12 +670,150 @@ func enableIPv4Forwarding() error {
 	// the host-side veth and then leave through the pod's eth0. Without this, the
 	// kernel would not route traffic between those interfaces even though both
 	// live in the worker pod network namespace.
-	if err := os.WriteFile("/proc/sys/net/ipv4/ip_forward", []byte("1\n"), 0o644); err != nil {
+	//
+	// Without privileged, the container runtime bind-mounts /proc/sys read-only.
+	// The worker holds CAP_SYS_ADMIN and uses no user namespace, so the ro flag
+	// is not locked: clear it, write the sysctl, restore ro.
+	const path = "/proc/sys/net/ipv4/ip_forward"
+	if b, err := os.ReadFile(path); err == nil && len(b) > 0 && b[0] == '1' {
+		return nil
+	}
+	if err := os.WriteFile(path, []byte("1\n"), 0o644); err == nil {
+		return nil
+	}
+	if err := unix.Mount("none", "/proc/sys", "", unix.MS_BIND|unix.MS_REMOUNT, ""); err != nil {
+		return fmt.Errorf("while remounting /proc/sys read-write to enable IPv4 forwarding: %w", err)
+	}
+	defer func() {
+		_ = unix.Mount("none", "/proc/sys", "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY, "")
+	}()
+	if err := os.WriteFile(path, []byte("1\n"), 0o644); err != nil {
 		return fmt.Errorf("while enabling IPv4 forwarding in worker pod netns: %w", err)
 	}
 	return nil
 }
 
+// setupCgroupDelegation prepares the worker pod's cgroup so runsc can create a
+// per-actor-container leaf under it with real cpu/memory/pids accounting.
+//
+// The unprivileged worker runs in a private cgroup namespace, so /sys/fs/cgroup
+// is the pod's own cgroup scope rather than the host root. Two things must be
+// arranged before runsc can nest container cgroups here:
+//
+//   - The cgroup v2 "no internal processes" rule forbids a cgroup from holding
+//     processes directly while also delegating controllers to children. The pod
+//     scope is not the true cgroup root, so the exemption does not apply: we move
+//     the worker's own processes into a dedicated "ateom" leaf.
+//   - Controllers are only available to children if enabled in the scope's
+//     cgroup.subtree_control. We enable everything the parent delegated to us.
+//
+// The runtime bind-mounts /sys/fs/cgroup read-only for unprivileged pods. The
+// worker holds CAP_SYS_ADMIN with no user namespace, so the ro flag is not
+// locked: clear it and leave it writable (runsc writes here on every
+// create/restore).
+func setupCgroupDelegation(ctx context.Context) error {
+	const root = "/sys/fs/cgroup"
+	const leaf = root + "/ateom"
+
+	// Delegation only makes sense inside a private cgroup namespace, where
+	// /sys/fs/cgroup is the pod's own scope. A privileged worker instead inherits
+	// the host cgroup namespace, so /sys/fs/cgroup is the true host root: it holds
+	// unmovable kernel threads (cgroup.procs would never drain) and must not be
+	// carved up. Detect the namespace via /proc/self/cgroup, which reads "0::/"
+	// only at a cgroup-namespace root, and skip delegation otherwise (runsc then
+	// falls back to its own cgroup handling).
+	if private, err := inPrivateCgroupNamespace(); err != nil {
+		return fmt.Errorf("while detecting cgroup namespace: %w", err)
+	} else if !private {
+		slog.InfoContext(ctx, "not in a private cgroup namespace; skipping cgroup delegation (worker is likely privileged)")
+		return nil
+	}
+
+	if err := os.Mkdir(leaf, 0o755); err != nil && !os.IsExist(err) {
+		// The runtime bind-mounts /sys/fs/cgroup read-only; clear the flag with a
+		// bind-remount. This needs CAP_SYS_ADMIN (held) and an AppArmor profile
+		// that permits mount. The gVisor worker runs AppArmor-unconfined, which
+		// runsc's own mounts require anyway; on nodes that do enforce the default
+		// profile (GKE COS) this mount is otherwise denied with EPERM.
+		if err := unix.Mount("none", root, "", unix.MS_BIND|unix.MS_REMOUNT, ""); err != nil {
+			return fmt.Errorf("while remounting %q read-write: %w", root, err)
+		}
+		if err := os.Mkdir(leaf, 0o755); err != nil && !os.IsExist(err) {
+			return fmt.Errorf("while creating cgroup leaf %q: %w", leaf, err)
+		}
+	}
+
+	if err := moveProcs(ctx, root+"/cgroup.procs", leaf+"/cgroup.procs"); err != nil {
+		return fmt.Errorf("while moving worker processes into %q: %w", leaf, err)
+	}
+
+	avail, err := os.ReadFile(root + "/cgroup.controllers")
+	if err != nil {
+		return fmt.Errorf("while reading available cgroup controllers: %w", err)
+	}
+	// Enable controllers one at a time so a single controller the node cannot
+	// delegate (for example cpuset without an assigned cpu set) does not prevent
+	// the others from being enabled.
+	var enabled []string
+	for _, c := range strings.Fields(string(avail)) {
+		if err := os.WriteFile(root+"/cgroup.subtree_control", []byte("+"+c), 0o644); err != nil {
+			slog.WarnContext(ctx, "could not enable cgroup controller for delegation", slog.String("controller", c), slog.Any("err", err))
+			continue
+		}
+		enabled = append(enabled, c)
+	}
+	slog.InfoContext(ctx, "cgroup delegation ready", slog.Any("controllers", enabled))
+	return nil
+}
+
+// inPrivateCgroupNamespace reports whether the process sits at the root of its
+// own cgroup namespace. The cgroup v2 line of /proc/self/cgroup ("0::<path>")
+// reports the path relative to the namespace root, so it reads exactly "/" only
+// when /sys/fs/cgroup is the namespace's own (pod-scoped) cgroup. A privileged
+// worker inheriting the host cgroup namespace instead sees its full host path
+// (for example "/kubepods.slice/.../cri-containerd-<id>.scope").
+func inPrivateCgroupNamespace() (bool, error) {
+	b, err := os.ReadFile("/proc/self/cgroup")
+	if err != nil {
+		return false, fmt.Errorf("while reading /proc/self/cgroup: %w", err)
+	}
+	for _, line := range strings.Split(strings.TrimSpace(string(b)), "\n") {
+		if path, ok := strings.CutPrefix(line, "0::"); ok {
+			return path == "/", nil
+		}
+	}
+	return false, fmt.Errorf("no cgroup v2 (0::) entry in /proc/self/cgroup")
+}
+
+// moveProcs relocates every process listed in srcProcs into dstProcs. cgroup.procs
+// only ever lists processes that are not already in a child cgroup, and the list
+// shrinks as we drain it, so loop until the source is empty.
+func moveProcs(ctx context.Context, srcProcs, dstProcs string) error {
+	// One pass moves everything it saw, but a process can fork between the read
+	// and the writes, so re-read until the source reads empty. 100 is an
+	// arbitrary generous bound (one or two passes suffice in practice) so a
+	// process that can never be moved fails startup with a clear error instead
+	// of looping forever.
+	for range 100 {
+		b, err := os.ReadFile(srcProcs)
+		if err != nil {
+			return fmt.Errorf("while reading %q: %w", srcProcs, err)
+		}
+		pids := strings.Fields(string(b))
+		if len(pids) == 0 {
+			return nil
+		}
+		for _, pid := range pids {
+			// Writing a TGID moves the whole thread group. A process can exit
+			// between the read and the write, so a failure here is not fatal.
+			if err := os.WriteFile(dstProcs, []byte(pid), 0o644); err != nil {
+				slog.WarnContext(ctx, "could not move process into cgroup leaf", slog.String("pid", pid), slog.Any("err", err))
+			}
+		}
+	}
+	return fmt.Errorf("%q did not drain after 100 iterations", srcProcs)
+}
+
 func installActorNftablesRules(podIP net.IP) error {
 	// Install a dedicated nftables table for the active actor. Keeping all
 	// rules in an ateom-owned table makes cleanup simple and avoids mutating

cmd/ateom-gvisor/runsc.go

@@ -18,11 +18,15 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"log/slog"
 	"os"
 	"os/exec"
+	"path/filepath"
+
+	specs "github.com/opencontainers/runtime-spec/specs-go"
 
 	"github.com/agent-substrate/substrate/internal/ateompath"
 )
@@ -32,12 +36,51 @@ type runsc struct {
 	actorUID string
 }
 
+// ensureContainerCgroupsPath sets the OCI spec's cgroupsPath so runsc creates a
+// per-container cgroup leaf under the worker pod's own cgroup (see
+// setupCgroupDelegation). atelet emits a runtime-agnostic spec with no
+// cgroupsPath; the gVisor ateom fills in its own convention here, mirroring how
+// the micro-VM ateom assigns /ateomchv/<id> in ensureKataCompatibleSpec. The
+// path is colon-free (so runsc uses the cgroupfs driver, not systemd) and
+// absolute, so it resolves under the pod scope in the worker's private cgroup
+// namespace.
+func (r *runsc) ensureContainerCgroupsPath(containerName string) error {
+	specPath := filepath.Join(ateompath.OCIBundlePath(r.actorUID, containerName), "config.json")
+	b, err := os.ReadFile(specPath)
+	if err != nil {
+		return fmt.Errorf("reading %q: %w", specPath, err)
+	}
+	var spec specs.Spec
+	if err := json.Unmarshal(b, &spec); err != nil {
+		return fmt.Errorf("parsing %q: %w", specPath, err)
+	}
+	if spec.Linux == nil {
+		spec.Linux = &specs.Linux{}
+	}
+	if spec.Linux.CgroupsPath != "" {
+		return nil
+	}
+	spec.Linux.CgroupsPath = "/" + containerName
+	out, err := json.MarshalIndent(&spec, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshaling %q: %w", specPath, err)
+	}
+	if err := os.WriteFile(specPath, out, 0o600); err != nil {
+		return fmt.Errorf("writing %q: %w", specPath, err)
+	}
+	return nil
+}
+
 func (r *runsc) cmdCreate(ctx context.Context, out io.Writer, containerName string, additionalArgs []string) error {
 	reapLock.RLock()
 	defer reapLock.RUnlock()
 
 	slog.InfoContext(ctx, "About to run runsc create", slog.String("container", containerName))
 
+	if err := r.ensureContainerCgroupsPath(containerName); err != nil {
+		return fmt.Errorf("while setting cgroups path for %q: %w", containerName, err)
+	}
+
 	args := []string{
 		"-log-format", "json",
 		"--alsologtostderr",
@@ -179,6 +222,10 @@ func (r *runsc) cmdRestore(ctx context.Context, out io.Writer, containerName, ch
 
 	slog.InfoContext(ctx, "About to run runsc restore", slog.String("container", containerName))
 
+	if err := r.ensureContainerCgroupsPath(containerName); err != nil {
+		return fmt.Errorf("while setting cgroups path for %q: %w", containerName, err)
+	}
+
 	cmd := exec.CommandContext(
 		ctx,
 		r.path,