Speed up selfhost e2e: tunable sandbox deadline + 3-way sharding

Author: RhysSullivan

.github/workflows/ci.yml

@@ -199,7 +199,13 @@ jobs:
           - { target: cloud, shard: 2/4, shard-name: 2of4 }
           - { target: cloud, shard: 3/4, shard-name: 3of4 }
           - { target: cloud, shard: 4/4, shard-name: 4of4 }
-          - target: selfhost
+          # Selfhost shards the same way: each shard is its own runner booting
+          # its own fresh instance (own port block + data dir), so the
+          # project's shared-bootstrap-admin assumption stays intact per shard
+          # and `fileParallelism: false` still serializes within a shard.
+          - { target: selfhost, shard: 1/3, shard-name: 1of3 }
+          - { target: selfhost, shard: 2/3, shard-name: 2of3 }
+          - { target: selfhost, shard: 3/3, shard-name: 3of3 }
     runs-on: blacksmith-4vcpu-ubuntu-2404
     timeout-minutes: 30
     steps:
@@ -254,7 +260,7 @@ jobs:
 
       - name: Run selfhost scenarios
         if: matrix.target == 'selfhost'
-        run: bunx vitest run --project selfhost --retry=2
+        run: bunx vitest run --project selfhost --retry=2 ${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }}
         working-directory: e2e
 
       # Failed runs keep their trace.zip / session.mp4 / step screenshots in

apps/host-selfhost/src/config.ts

@@ -43,6 +43,14 @@ export interface SelfHostConfig {
   readonly organizationName: string;
   /** URL slug for org-prefixed console paths (`/<slug>/policies`). */
   readonly orgSlug: string;
+  /**
+   * Sandbox execution budget passed to the QuickJS runtime, or undefined for
+   * the runtime's own default (5 minutes). An operator knob in principle, but
+   * its real consumer is the e2e harness, which shrinks it to seconds so the
+   * sandbox-deadline scenario proves its race without waiting out real
+   * minutes (the same pattern as MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS on cloud).
+   */
+  readonly sandboxTimeoutMs: number | undefined;
 }
 
 export const resolveDataDir = (): string =>
@@ -148,9 +156,26 @@ export const loadConfig = (): SelfHostConfig => {
     bootstrapAdminName: process.env.EXECUTOR_BOOTSTRAP_ADMIN_NAME ?? "Admin",
     organizationName: process.env.EXECUTOR_ORG_NAME ?? "Default",
     orgSlug: resolveOrgSlug(),
+    sandboxTimeoutMs: resolveSandboxTimeoutMs(),
   };
 };
 
+// A malformed value is refused rather than silently ignored: an operator who
+// sets the knob and typos it should find out at boot, not by watching a
+// runaway execution use the 5-minute default.
+const resolveSandboxTimeoutMs = (): number | undefined => {
+  const raw = process.env.EXECUTOR_SANDBOX_TIMEOUT_MS;
+  if (!raw) return undefined;
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed) || parsed <= 0) {
+    // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: refuse to boot on a malformed operator knob
+    throw new Error(
+      `EXECUTOR_SANDBOX_TIMEOUT_MS ${JSON.stringify(raw)} is not a positive number of milliseconds`,
+    );
+  }
+  return Math.floor(parsed);
+};
+
 // The org slug doubles as a URL segment (`/<slug>/policies`), so an
 // operator-set value must fit the shared grammar and avoid reserved root
 // segments (api, mcp, login, …) — a colliding slug would shadow real routes.

apps/host-selfhost/src/execution.ts

@@ -65,7 +65,12 @@ export const SelfHostHostConfig: Layer.Layer<HostConfig> = Layer.sync(HostConfig
 
 export const SelfHostCodeExecutorProvider: Layer.Layer<CodeExecutorProvider> = Layer.sync(
   CodeExecutorProvider,
-  () => makeQuickJsExecutor(),
+  () => {
+    const { sandboxTimeoutMs } = loadConfig();
+    return makeQuickJsExecutor(
+      sandboxTimeoutMs === undefined ? {} : { timeoutMs: sandboxTimeoutMs },
+    );
+  },
 );
 
 /**

e2e/scenarios/resume-after-sandbox-deadline.test.ts

@@ -8,14 +8,18 @@
 // unknown execution.
 //
 // The journey drives exactly that shape: ONE execution with TWO approval
-// gates. The first approval is granted late in its window (~3.5 min), so the
-// second pause's window reaches well past the sandbox's 5-minute mark. The
-// second approval arrives ~5.75 min after execution start — inside its OWN
-// advertised window, but past the old absolute deadline. Deliberately slow
-// (~6 min): the elapsed time IS the subject under test. A single-pause
-// variant cannot express this cross-target — hosts that advertise a
-// 4-minute window would expire it legitimately before the sandbox clock
-// even matters.
+// gates. The first approval is granted late (70% of the sandbox budget in),
+// so the second pause's window reaches well past the budget. The second
+// approval arrives at ~115% of the budget after execution start — inside its
+// OWN advertised window, but past the old absolute deadline. The subject is
+// that RATIO, not any absolute duration, so the delays scale off the budget
+// the target was booted with: selfhost boots with a seconds-long
+// EXECUTOR_SANDBOX_TIMEOUT_MS (setup/sandbox-timeout.ts) and proves the race
+// in ~25s; a target on the production 5-minute budget runs the original
+// ~6-minute journey (the elapsed time IS the subject — nothing is mocked). A
+// single-pause variant cannot express this cross-target — hosts that
+// advertise a 4-minute window would expire it legitimately before the
+// sandbox clock even matters.
 //
 // The gate is `policies.create`'s own `requiresApproval` annotation
 // (hermetic, same device as policy-tool-approval.test.ts); both approvals
@@ -29,15 +33,23 @@ import { composePluginApi } from "@executor-js/api/server";
 import { scenario } from "../src/scenario";
 import { Api, Mcp, Target } from "../src/services";
 import { configuredMcpPausedSessionIdleTimeoutMs } from "../setup/mcp-session-timeouts";
+import { configuredSandboxTimeoutMs } from "../setup/sandbox-timeout";
 
 const coreApi = composePluginApi([] as const);
 
-// Grant the first approval at 3.5 min — late but inside its 4-minute window.
-// The second pause then opens a fresh window reaching ~7.5 min.
-const FIRST_APPROVAL_DELAY_MS = 3.5 * 60_000;
-// Grant the second approval 2.25 min later: ~5.75 min after execution start,
-// past the sandbox's 5-minute budget but inside the second window.
-const SECOND_APPROVAL_DELAY_MS = 2.25 * 60_000;
+const SANDBOX_BUDGET_MS = configuredSandboxTimeoutMs();
+
+// Grant the first approval at 70% of the budget — late but inside its window
+// (was 3.5 of 5 min). The second pause then opens a fresh window reaching
+// past the budget.
+const FIRST_APPROVAL_DELAY_MS = 0.7 * SANDBOX_BUDGET_MS;
+// Grant the second approval 45% of the budget later: ~115% of the budget
+// after execution start, past the sandbox clock but inside the second window
+// (was 2.25 of 5 min → ~5.75 min total).
+const SECOND_APPROVAL_DELAY_MS = 0.45 * SANDBOX_BUDGET_MS;
+// The whole journey plus scheduling slack, for the idle-window guard and the
+// vitest timeout.
+const JOURNEY_MS = FIRST_APPROVAL_DELAY_MS + SECOND_APPROVAL_DELAY_MS;
 
 /** Sandbox code that creates two policies through the approval-gated core
  *  tool. Patterns are unique-per-run and match no real tool, so the rules are
@@ -56,19 +68,22 @@ const second = await tools.executor.coreTools.policies.create({
 return JSON.stringify({ first: first.ok, second: second.ok });
 `;
 
-// The journey spans ~6 real minutes of paused waiting, so the host must keep
-// the paused session alive that long. The suite's default e2e override shrinks
+// The journey spans the whole paused waiting time, so the host must keep the
+// paused session alive that long. The suite's default e2e override shrinks
 // the paused-session idle teardown to seconds (to keep teardown tests fast),
 // which would evict the session mid-scenario for reasons unrelated to the
-// clock under test — require the production-like window instead.
+// clock under test — require a window that outlasts the journey instead.
+// With a shrunken sandbox budget the journey shrinks too, so even the short
+// e2e idle window can suffice; the guard compares the two rather than
+// hardcoding either.
 const PAUSED_IDLE_WINDOW_TOO_SHORT =
-  configuredMcpPausedSessionIdleTimeoutMs() < 8 * 60_000
-    ? `the target's paused-session idle teardown (${configuredMcpPausedSessionIdleTimeoutMs()}ms) evicts the session before this ~6-minute journey completes; boot the target with MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS >= 480000 to run it`
+  configuredMcpPausedSessionIdleTimeoutMs() < JOURNEY_MS + 60_000
+    ? `the target's paused-session idle teardown (${configuredMcpPausedSessionIdleTimeoutMs()}ms) evicts the session before this ${Math.round(JOURNEY_MS / 1000)}s journey completes; boot the target with MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS >= ${JOURNEY_MS + 60_000} or a smaller E2E_SANDBOX_TIMEOUT_MS to run it`
     : undefined;
 
 scenario(
   "MCP · chained approvals granted within their windows survive the sandbox clock",
-  { timeout: 480_000, skip: PAUSED_IDLE_WINDOW_TOO_SHORT },
+  { timeout: Math.max(120_000, JOURNEY_MS + 120_000), skip: PAUSED_IDLE_WINDOW_TOO_SHORT },
   Effect.gen(function* () {
     const target = yield* Target;
     const apiSurface = yield* Api;

e2e/setup/sandbox-timeout.ts

@@ -0,0 +1,28 @@
+// The sandbox execution budget shared between a target's boot env and the
+// sandbox-deadline scenario, so they cannot drift apart (same pattern as
+// execution-limits.ts). The scenario proves a RATIO — approvals granted
+// inside their own windows survive an execution that outlives the sandbox's
+// absolute budget — so the budget's magnitude is free to shrink: on selfhost
+// the boot recipe passes E2E_SANDBOX_TIMEOUT_MS through to the server as
+// EXECUTOR_SANDBOX_TIMEOUT_MS and the scenario scales its approval delays to
+// match, turning a ~6-minute real-time wait into seconds. Targets that cannot
+// shrink the budget (cloud's dynamic-worker deadline is not env-tunable) run
+// against the production default and skip via their paused-session window
+// guard instead.
+export const E2E_SANDBOX_TIMEOUT_MS = 20_000;
+
+export const SANDBOX_TIMEOUT_ENV = "E2E_SANDBOX_TIMEOUT_MS";
+
+const PRODUCTION_SANDBOX_TIMEOUT_MS = 5 * 60_000;
+
+const positiveMilliseconds = (raw: string | undefined): number | undefined => {
+  if (!raw) return undefined;
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed) || parsed <= 0) return undefined;
+  return Math.floor(parsed);
+};
+
+/** The sandbox budget the current target enforces: the harness override when
+ *  the target was booted with one, else the production default. */
+export const configuredSandboxTimeoutMs = (): number =>
+  positiveMilliseconds(process.env[SANDBOX_TIMEOUT_ENV]) ?? PRODUCTION_SANDBOX_TIMEOUT_MS;

e2e/setup/selfhost.boot.ts

@@ -21,6 +21,9 @@ export interface SelfhostBootOptions {
   /** vite --host (e.g. "0.0.0.0" to be tailnet-reachable). */
   readonly host?: string;
   readonly logFile?: string;
+  /** Shrink the sandbox execution budget (EXECUTOR_SANDBOX_TIMEOUT_MS) so
+   *  deadline scenarios prove their race in seconds. Omit for production. */
+  readonly sandboxTimeoutMs?: number;
 }
 
 export const bootSelfhost = async (options: SelfhostBootOptions): Promise<BootedProcesses> => {
@@ -51,6 +54,9 @@ export const bootSelfhost = async (options: SelfhostBootOptions): Promise<Booted
           // instance at them; the hosted SSRF guard would otherwise block
           // outbound probes/dials to localhost. Hermetic test instance only.
           EXECUTOR_ALLOW_LOCAL_NETWORK: "true",
+          ...(options.sandboxTimeoutMs !== undefined
+            ? { EXECUTOR_SANDBOX_TIMEOUT_MS: String(options.sandboxTimeoutMs) }
+            : {}),
         },
         logFile: options.logFile,
       },

e2e/setup/selfhost.globalsetup.ts

@@ -8,6 +8,7 @@ import { resolve } from "node:path";
 import { claimAndBoot } from "../src/ports";
 import { SELFHOST_ADMIN } from "../targets/selfhost";
 import { waitForHttp } from "./boot";
+import { E2E_SANDBOX_TIMEOUT_MS, SANDBOX_TIMEOUT_ENV } from "./sandbox-timeout";
 import { bootSelfhost } from "./selfhost.boot";
 import { RUNS_DIR } from "../src/scenario";
 
@@ -37,13 +38,18 @@ export default async function setup(): Promise<(() => Promise<void>) | void> {
     [{ envVar: "E2E_SELFHOST_PORT", offset: 4, label: "selfhost vite dev" }],
     async (ports) => {
       const port = ports.E2E_SELFHOST_PORT!;
+      // Shrink the sandbox execution budget and publish the value to the test
+      // workers (spawned after this globalsetup, so they inherit the env): the
+      // sandbox-deadline scenario reads it to scale its approval delays.
+      process.env[SANDBOX_TIMEOUT_ENV] = String(E2E_SANDBOX_TIMEOUT_MS);
       // Fresh data dir per suite run — hermetic; in-suite isolation comes from
       // fresh identities, not resets (bootSelfhost wipes it).
       const procs = await bootSelfhost({
         port,
         webBaseUrl: `http://localhost:${port}`,
         admin: SELFHOST_ADMIN,
         logFile: bootLogFile,
+        sandboxTimeoutMs: E2E_SANDBOX_TIMEOUT_MS,
       });
       return { teardown: procs.teardown, value: procs };
     },