diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index 5b35a291a..dd4633bb3 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -34,10 +34,7 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool) *appsv1ac.Deployment WithArgs( "--pod-uid=$(POD_UID)", ). - WithSecurityContext(corev1ac.SecurityContext(). - WithPrivileged(true). - WithRunAsUser(0). - WithRunAsGroup(0)). + WithSecurityContext(ateomSecurityContext(wp.Spec.SandboxClass)). WithEnv( corev1ac.EnvVar(). WithName("POD_UID"). @@ -82,6 +79,46 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool) *appsv1ac.Deployment WithSpec(podSpecAC))) } +// ateomGvisorCapabilities is the capability set an unprivileged gVisor worker +// needs. runsc's gofer maps a full-range identity in a user namespace +// (SETUID/SETGID/SETPCAP/SETFCAP), the sandbox pivots root and traces the +// application (SYS_ADMIN/SYS_CHROOT/SYS_PTRACE), actor networking programs the +// veth and nftables rules (NET_ADMIN/NET_RAW), and the OCI rootfs is unpacked +// and device nodes created as root over image-owned trees +// (DAC_OVERRIDE/FOWNER/CHOWN/MKNOD). This replaces the former privileged worker; +// the default seccomp and AppArmor profiles are sufficient (no Unconfined). +var ateomGvisorCapabilities = []corev1.Capability{ + "NET_ADMIN", "SYS_ADMIN", "SYS_CHROOT", "SYS_PTRACE", + "SETUID", "SETGID", "SETPCAP", "DAC_OVERRIDE", + "FOWNER", "CHOWN", "MKNOD", "NET_RAW", "SETFCAP", +} + +// ateomSecurityContext returns the ateom container security context for a sandbox +// class. The gVisor worker runs unprivileged with an explicit capability set; the +// micro-VM worker stays privileged because kata + cloud-hypervisor needs broad +// host access (vhost devices, mounts). An empty class defaults to gVisor. +func ateomSecurityContext(class atev1alpha1.SandboxClass) *corev1ac.SecurityContextApplyConfiguration { + sc := corev1ac.SecurityContext(). + WithRunAsUser(0). + WithRunAsGroup(0) + if class == atev1alpha1.SandboxClassMicroVM { + return sc.WithPrivileged(true) + } + // runsc mounts and pivots root inside the sandbox, and the worker remounts + // /sys/fs/cgroup read-write to nest per-actor cgroups; the default AppArmor + // profile denies those mounts (enforced on GKE COS, a no-op on nodes that do + // not load AppArmor). A privileged worker got this implicitly; unprivileged + // must request it. seccomp stays at the runtime default. Confining runsc with + // a tailored AppArmor profile instead of Unconfined is a follow-up. + return sc. + WithPrivileged(false). + WithCapabilities(corev1ac.Capabilities(). + WithDrop("ALL"). + WithAdd(ateomGvisorCapabilities...)). + WithAppArmorProfile(corev1ac.AppArmorProfile(). + WithType(corev1.AppArmorProfileTypeUnconfined)) +} + // maybeApplyMicroVMPodShape adds the /dev/kvm device and node placement a // micro-VM (kata + cloud-hypervisor) worker pool needs, on top of any // pod-template settings. No-op unless sandboxClass is the micro-VM class. diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index 19f6d44d9..124a8fb82 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -261,6 +261,54 @@ func TestMicroVMPodShape(t *testing.T) { } } +// TestAteomSecurityContextByClass asserts the gVisor worker runs unprivileged +// with the explicit capability set while the micro-VM worker stays privileged, +// and that an empty class defaults to gVisor. +func TestAteomSecurityContextByClass(t *testing.T) { + tests := []struct { + name string + class atev1alpha1.SandboxClass + wantPrivileged bool + wantCaps bool + }{ + {"gvisor default", "", false, true}, + {"gvisor explicit", atev1alpha1.SandboxClassGvisor, false, true}, + {"microvm", atev1alpha1.SandboxClassMicroVM, true, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sc := ateomSecurityContext(tt.class) + if sc.Privileged == nil || *sc.Privileged != tt.wantPrivileged { + t.Errorf("Privileged = %v, want %v", sc.Privileged, tt.wantPrivileged) + } + if sc.RunAsUser == nil || *sc.RunAsUser != 0 || sc.RunAsGroup == nil || *sc.RunAsGroup != 0 { + t.Errorf("RunAsUser/Group = %v/%v, want 0/0", sc.RunAsUser, sc.RunAsGroup) + } + hasCaps := sc.Capabilities != nil && len(sc.Capabilities.Add) > 0 + if hasCaps != tt.wantCaps { + t.Errorf("has capabilities = %v, want %v", hasCaps, tt.wantCaps) + } + if tt.wantCaps { + if len(sc.Capabilities.Drop) != 1 || sc.Capabilities.Drop[0] != "ALL" { + t.Errorf("capabilities drop = %v, want [ALL]", sc.Capabilities.Drop) + } + if diff := cmp.Diff(ateomGvisorCapabilities, sc.Capabilities.Add); diff != "" { + t.Errorf("capabilities add mismatch (-want +got):\n%s", diff) + } + } + // The gVisor worker runs AppArmor-unconfined (runsc + cgroup remount + // need mount); the privileged micro-VM worker leaves it unset. + wantAppArmor := tt.wantCaps + hasAppArmor := sc.AppArmorProfile != nil && + sc.AppArmorProfile.Type != nil && + *sc.AppArmorProfile.Type == corev1.AppArmorProfileTypeUnconfined + if hasAppArmor != wantAppArmor { + t.Errorf("AppArmor Unconfined = %v, want %v", hasAppArmor, wantAppArmor) + } + }) + } +} + func testWorkerPoolApplyConfig(tmpl *atev1alpha1.WorkerPoolPodTemplate) *atev1alpha1.WorkerPool { return &atev1alpha1.WorkerPool{ ObjectMeta: metav1.ObjectMeta{Name: "pool", Namespace: "default", UID: "uid"}, @@ -289,9 +337,14 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf WithImage(wp.Spec.AteomImage). WithArgs("--pod-uid=$(POD_UID)"). WithSecurityContext(corev1ac.SecurityContext(). - WithPrivileged(true). WithRunAsUser(0). - WithRunAsGroup(0)). + WithRunAsGroup(0). + WithPrivileged(false). + WithCapabilities(corev1ac.Capabilities(). + WithDrop("ALL"). + WithAdd(ateomGvisorCapabilities...)). + WithAppArmorProfile(corev1ac.AppArmorProfile(). + WithType(corev1.AppArmorProfileTypeUnconfined))). WithEnv(corev1ac.EnvVar(). WithName("POD_UID"). WithValueFrom(corev1ac.EnvVarSource(). diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 6d0129452..bae20edf7 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -25,6 +25,7 @@ import ( "os" "runtime" "sort" + "strings" "sync" "cloud.google.com/go/compute/metadata" @@ -109,6 +110,12 @@ func do(ctx context.Context) error { return fmt.Errorf("in os.MkdirAll(%q): %w", ateomDir, err) } + // Prepare the pod cgroup so runsc can create per-actor-container leaves under + // it with real accounting, instead of ignoring cgroups entirely. + if err := setupCgroupDelegation(ctx); err != nil { + return fmt.Errorf("while setting up cgroup delegation: %w", err) + } + // TODO: Consider whether we want to fork, so that we have an "init" process // as PID 1 that does nothing but reap processes that get reparented to it. // Then we won't have to mess about with locking the reaper while we do our @@ -663,12 +670,150 @@ func enableIPv4Forwarding() error { // the host-side veth and then leave through the pod's eth0. Without this, the // kernel would not route traffic between those interfaces even though both // live in the worker pod network namespace. - if err := os.WriteFile("/proc/sys/net/ipv4/ip_forward", []byte("1\n"), 0o644); err != nil { + // + // Without privileged, the container runtime bind-mounts /proc/sys read-only. + // The worker holds CAP_SYS_ADMIN and uses no user namespace, so the ro flag + // is not locked: clear it, write the sysctl, restore ro. + const path = "/proc/sys/net/ipv4/ip_forward" + if b, err := os.ReadFile(path); err == nil && len(b) > 0 && b[0] == '1' { + return nil + } + if err := os.WriteFile(path, []byte("1\n"), 0o644); err == nil { + return nil + } + if err := unix.Mount("none", "/proc/sys", "", unix.MS_BIND|unix.MS_REMOUNT, ""); err != nil { + return fmt.Errorf("while remounting /proc/sys read-write to enable IPv4 forwarding: %w", err) + } + defer func() { + _ = unix.Mount("none", "/proc/sys", "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY, "") + }() + if err := os.WriteFile(path, []byte("1\n"), 0o644); err != nil { return fmt.Errorf("while enabling IPv4 forwarding in worker pod netns: %w", err) } return nil } +// setupCgroupDelegation prepares the worker pod's cgroup so runsc can create a +// per-actor-container leaf under it with real cpu/memory/pids accounting. +// +// The unprivileged worker runs in a private cgroup namespace, so /sys/fs/cgroup +// is the pod's own cgroup scope rather than the host root. Two things must be +// arranged before runsc can nest container cgroups here: +// +// - The cgroup v2 "no internal processes" rule forbids a cgroup from holding +// processes directly while also delegating controllers to children. The pod +// scope is not the true cgroup root, so the exemption does not apply: we move +// the worker's own processes into a dedicated "ateom" leaf. +// - Controllers are only available to children if enabled in the scope's +// cgroup.subtree_control. We enable everything the parent delegated to us. +// +// The runtime bind-mounts /sys/fs/cgroup read-only for unprivileged pods. The +// worker holds CAP_SYS_ADMIN with no user namespace, so the ro flag is not +// locked: clear it and leave it writable (runsc writes here on every +// create/restore). +func setupCgroupDelegation(ctx context.Context) error { + const root = "/sys/fs/cgroup" + const leaf = root + "/ateom" + + // Delegation only makes sense inside a private cgroup namespace, where + // /sys/fs/cgroup is the pod's own scope. A privileged worker instead inherits + // the host cgroup namespace, so /sys/fs/cgroup is the true host root: it holds + // unmovable kernel threads (cgroup.procs would never drain) and must not be + // carved up. Detect the namespace via /proc/self/cgroup, which reads "0::/" + // only at a cgroup-namespace root, and skip delegation otherwise (runsc then + // falls back to its own cgroup handling). + if private, err := inPrivateCgroupNamespace(); err != nil { + return fmt.Errorf("while detecting cgroup namespace: %w", err) + } else if !private { + slog.InfoContext(ctx, "not in a private cgroup namespace; skipping cgroup delegation (worker is likely privileged)") + return nil + } + + if err := os.Mkdir(leaf, 0o755); err != nil && !os.IsExist(err) { + // The runtime bind-mounts /sys/fs/cgroup read-only; clear the flag with a + // bind-remount. This needs CAP_SYS_ADMIN (held) and an AppArmor profile + // that permits mount. The gVisor worker runs AppArmor-unconfined, which + // runsc's own mounts require anyway; on nodes that do enforce the default + // profile (GKE COS) this mount is otherwise denied with EPERM. + if err := unix.Mount("none", root, "", unix.MS_BIND|unix.MS_REMOUNT, ""); err != nil { + return fmt.Errorf("while remounting %q read-write: %w", root, err) + } + if err := os.Mkdir(leaf, 0o755); err != nil && !os.IsExist(err) { + return fmt.Errorf("while creating cgroup leaf %q: %w", leaf, err) + } + } + + if err := moveProcs(ctx, root+"/cgroup.procs", leaf+"/cgroup.procs"); err != nil { + return fmt.Errorf("while moving worker processes into %q: %w", leaf, err) + } + + avail, err := os.ReadFile(root + "/cgroup.controllers") + if err != nil { + return fmt.Errorf("while reading available cgroup controllers: %w", err) + } + // Enable controllers one at a time so a single controller the node cannot + // delegate (for example cpuset without an assigned cpu set) does not prevent + // the others from being enabled. + var enabled []string + for _, c := range strings.Fields(string(avail)) { + if err := os.WriteFile(root+"/cgroup.subtree_control", []byte("+"+c), 0o644); err != nil { + slog.WarnContext(ctx, "could not enable cgroup controller for delegation", slog.String("controller", c), slog.Any("err", err)) + continue + } + enabled = append(enabled, c) + } + slog.InfoContext(ctx, "cgroup delegation ready", slog.Any("controllers", enabled)) + return nil +} + +// inPrivateCgroupNamespace reports whether the process sits at the root of its +// own cgroup namespace. The cgroup v2 line of /proc/self/cgroup ("0::") +// reports the path relative to the namespace root, so it reads exactly "/" only +// when /sys/fs/cgroup is the namespace's own (pod-scoped) cgroup. A privileged +// worker inheriting the host cgroup namespace instead sees its full host path +// (for example "/kubepods.slice/.../cri-containerd-.scope"). +func inPrivateCgroupNamespace() (bool, error) { + b, err := os.ReadFile("/proc/self/cgroup") + if err != nil { + return false, fmt.Errorf("while reading /proc/self/cgroup: %w", err) + } + for _, line := range strings.Split(strings.TrimSpace(string(b)), "\n") { + if path, ok := strings.CutPrefix(line, "0::"); ok { + return path == "/", nil + } + } + return false, fmt.Errorf("no cgroup v2 (0::) entry in /proc/self/cgroup") +} + +// moveProcs relocates every process listed in srcProcs into dstProcs. cgroup.procs +// only ever lists processes that are not already in a child cgroup, and the list +// shrinks as we drain it, so loop until the source is empty. +func moveProcs(ctx context.Context, srcProcs, dstProcs string) error { + // One pass moves everything it saw, but a process can fork between the read + // and the writes, so re-read until the source reads empty. 100 is an + // arbitrary generous bound (one or two passes suffice in practice) so a + // process that can never be moved fails startup with a clear error instead + // of looping forever. + for range 100 { + b, err := os.ReadFile(srcProcs) + if err != nil { + return fmt.Errorf("while reading %q: %w", srcProcs, err) + } + pids := strings.Fields(string(b)) + if len(pids) == 0 { + return nil + } + for _, pid := range pids { + // Writing a TGID moves the whole thread group. A process can exit + // between the read and the write, so a failure here is not fatal. + if err := os.WriteFile(dstProcs, []byte(pid), 0o644); err != nil { + slog.WarnContext(ctx, "could not move process into cgroup leaf", slog.String("pid", pid), slog.Any("err", err)) + } + } + } + return fmt.Errorf("%q did not drain after 100 iterations", srcProcs) +} + func installActorNftablesRules(podIP net.IP) error { // Install a dedicated nftables table for the active actor. Keeping all // rules in an ateom-owned table makes cleanup simple and avoids mutating diff --git a/cmd/ateom-gvisor/runsc.go b/cmd/ateom-gvisor/runsc.go index 2901c1d2e..003bd861a 100644 --- a/cmd/ateom-gvisor/runsc.go +++ b/cmd/ateom-gvisor/runsc.go @@ -18,11 +18,15 @@ package main import ( "context" + "encoding/json" "fmt" "io" "log/slog" "os" "os/exec" + "path/filepath" + + specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/agent-substrate/substrate/internal/ateompath" ) @@ -32,12 +36,51 @@ type runsc struct { actorUID string } +// ensureContainerCgroupsPath sets the OCI spec's cgroupsPath so runsc creates a +// per-container cgroup leaf under the worker pod's own cgroup (see +// setupCgroupDelegation). atelet emits a runtime-agnostic spec with no +// cgroupsPath; the gVisor ateom fills in its own convention here, mirroring how +// the micro-VM ateom assigns /ateomchv/ in ensureKataCompatibleSpec. The +// path is colon-free (so runsc uses the cgroupfs driver, not systemd) and +// absolute, so it resolves under the pod scope in the worker's private cgroup +// namespace. +func (r *runsc) ensureContainerCgroupsPath(containerName string) error { + specPath := filepath.Join(ateompath.OCIBundlePath(r.actorUID, containerName), "config.json") + b, err := os.ReadFile(specPath) + if err != nil { + return fmt.Errorf("reading %q: %w", specPath, err) + } + var spec specs.Spec + if err := json.Unmarshal(b, &spec); err != nil { + return fmt.Errorf("parsing %q: %w", specPath, err) + } + if spec.Linux == nil { + spec.Linux = &specs.Linux{} + } + if spec.Linux.CgroupsPath != "" { + return nil + } + spec.Linux.CgroupsPath = "/" + containerName + out, err := json.MarshalIndent(&spec, "", " ") + if err != nil { + return fmt.Errorf("marshaling %q: %w", specPath, err) + } + if err := os.WriteFile(specPath, out, 0o600); err != nil { + return fmt.Errorf("writing %q: %w", specPath, err) + } + return nil +} + func (r *runsc) cmdCreate(ctx context.Context, out io.Writer, containerName string, additionalArgs []string) error { reapLock.RLock() defer reapLock.RUnlock() slog.InfoContext(ctx, "About to run runsc create", slog.String("container", containerName)) + if err := r.ensureContainerCgroupsPath(containerName); err != nil { + return fmt.Errorf("while setting cgroups path for %q: %w", containerName, err) + } + args := []string{ "-log-format", "json", "--alsologtostderr", @@ -179,6 +222,10 @@ func (r *runsc) cmdRestore(ctx context.Context, out io.Writer, containerName, ch slog.InfoContext(ctx, "About to run runsc restore", slog.String("container", containerName)) + if err := r.ensureContainerCgroupsPath(containerName); err != nil { + return fmt.Errorf("while setting cgroups path for %q: %w", containerName, err) + } + cmd := exec.CommandContext( ctx, r.path,