feat(e2e): add basic e2e test suite for runtime templates

.github/workflows/ci.yml

@@ -21,3 +21,13 @@ jobs:
       ref: ${{ github.event.pull_request.head.sha || github.sha }}
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  e2e:
+    if: github.event_name == 'push'
+    uses: ./.github/workflows/e2e-test.yml
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      ref: ${{ github.sha }}
+    secrets: inherit

.github/workflows/e2e-test.yml

@@ -0,0 +1,121 @@
+name: e2e-test
+on:
+  workflow_call:
+    inputs:
+      ref:
+        description: git reference (commit, branch, or tag) to build and test
+        required: false
+        type: string
+        default: ""
+      tags:
+        description: Vitest tag expression, for example runtime || canary
+        required: false
+        type: string
+        default: ""
+    secrets:
+      WORKFLOW_SECRETS_READER_ROLE_ARN:
+        required: true
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: git reference (commit, branch, or tag) to build and test
+        type: string
+        default: refactor
+      tags:
+        description: Vitest tag expression, for example runtime || canary
+        type: string
+        default: ""
+
+concurrency:
+  group: e2e-test-${{ inputs.ref || github.ref }}-${{ inputs.tags || 'all' }}
+  cancel-in-progress: false
+
+env:
+  AGENTCORE_TELEMETRY_DISABLED: "1"
+
+jobs:
+  authorize:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    outputs:
+      is-authorized: ${{ steps.check.outputs.is_authorized }}
+    steps:
+      - name: Fetch secrets from Secrets Manager
+        uses: aws/agentcore-devx-devtools/.github/actions/fetch-secrets@31aa3b031a86664e29861d68956e44b07cf21a74
+        with:
+          role-arn: ${{ secrets.WORKFLOW_SECRETS_READER_ROLE_ARN }}
+          repo: AUTHORIZED_USERS
+      - name: Check authorization
+        id: authz
+        uses: aws/agentcore-devx-devtools/.github/actions/check-authorized-user@31aa3b031a86664e29861d68956e44b07cf21a74
+        with:
+          subject: ${{ github.actor }}
+          authorized-users: ${{ env.AUTHORIZED_USERS }}
+      - name: Determine authorization
+        id: check
+        env:
+          IS_AUTHORIZED: ${{ steps.authz.outputs.is-authorized }}
+          ACTOR: ${{ github.actor }}
+        run: |
+          if [[ "$IS_AUTHORIZED" == "true" ]]; then
+            echo "Actor ${ACTOR} is authorized"
+            echo "is_authorized=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "Actor ${ACTOR} is not in AUTHORIZED_USERS, skipping"
+            echo "is_authorized=false" >> "$GITHUB_OUTPUT"
+          fi
+
+  e2e:
+    name: e2e (${{ matrix.name }})
+    needs: authorize
+    if: needs.authorize.outputs.is-authorized == 'true'
+    runs-on: ${{ fromJSON(matrix.runner) }}
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Per-job labels stop GitHub from routing a job to the runner CodeBuild created for another job.
+          # https://docs.aws.amazon.com/codebuild/latest/userguide/sample-github-action-runners-update-labels.html
+          - name: Linux
+            runner: '["codebuild-agentcore-e2e-${{ github.run_id }}-${{ github.run_attempt }}", "e2e-linux"]'
+          - name: Windows
+            runner: '["codebuild-agentcore-e2e-${{ github.run_id }}-${{ github.run_attempt }}", "image:windows-1.0", "e2e-windows"]'
+          # CodeBuild does not support macOS.
+          # https://docs.aws.amazon.com/codebuild/latest/userguide/action-runner-questions.html#action-runner-platform
+          - name: macOS
+            runner: '["macos-latest"]'
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ inputs.ref || github.sha }}
+          persist-credentials: false
+      - uses: oven-sh/setup-bun@v2
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "24"
+      - uses: astral-sh/setup-uv@v6
+      - run: bun install --frozen-lockfile
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ vars.E2E_ROLE_ARN }}
+          aws-region: ${{ vars.E2E_REGION || 'us-east-1' }}
+      - name: Build CLI
+        run: bun run build
+      - name: Run all E2E tests
+        if: inputs.tags == ''
+        env:
+          AGENTCORE_CLI_PATH: node ${{ github.workspace }}/dist/index.js
+          AWS_REGION: ${{ vars.E2E_REGION || 'us-east-1' }}
+        run: bun run test:e2e
+      - name: Run tagged E2E tests
+        if: inputs.tags != ''
+        env:
+          AGENTCORE_CLI_PATH: node ${{ github.workspace }}/dist/index.js
+          AWS_REGION: ${{ vars.E2E_REGION || 'us-east-1' }}
+        run: bun run test:e2e -- --tagsFilter="${{ inputs.tags }}"

bunfig.toml

@@ -1,4 +1,4 @@
 [test]
 preload = ["./src/testing/setup.ts"]
-pathIgnorePatterns = ["src/assets/**", "out/**", "dist/**"]
+pathIgnorePatterns = ["src/assets/**", "out/**", "dist/**", "e2eTest/**"]
 coveragePathIgnorePatterns = ["src/testing/**", "src/assets/**"]

e2eTest/README.md

@@ -0,0 +1,21 @@
+# End-to-end tests
+
+The e2e suite deploys and invokes real AgentCore resources. Run it with AWS credentials:
+
+```sh
+bun run build
+export AGENTCORE_CLI_PATH="node $PWD/dist/index.js"
+bun run test:e2e
+```
+
+To run tagged tests:
+
+```sh
+bun run test:e2e -- --tagsFilter='runtime || canary'
+```
+
+Set `AGENTCORE_CLI_PATH` to use a different executable:
+
+```sh
+AGENTCORE_CLI_PATH=/path/to/agentcore bun run test:e2e
+```

e2eTest/constants.ts

@@ -0,0 +1,5 @@
+export const E2E_PREFIX = "e2e";
+
+export const TAGS = {
+  RUNTIME: "runtime",
+} as const;

e2eTest/helpers/logger.ts

@@ -0,0 +1,22 @@
+type LogMethod = (...messages: unknown[]) => void;
+
+export type TestLogger = {
+  debug: LogMethod;
+  info: LogMethod;
+  warn: LogMethod;
+  error: LogMethod;
+};
+
+/** Given a scope, creates a thin console-backed logger for E2E diagnostics. */
+export function createLogger(scope: string): TestLogger {
+  const write = (method: (...messages: unknown[]) => void): LogMethod => {
+    return (...messages) => method(`[${scope}]`, ...messages);
+  };
+
+  return {
+    debug: write(console.debug),
+    info: write(console.info),
+    warn: write(console.warn),
+    error: write(console.error),
+  };
+}

e2eTest/helpers/retry.ts

@@ -0,0 +1,20 @@
+/** Given an async operation, retries it until success or the timeout expires. */
+export async function retry<T>(
+  operation: () => Promise<T>,
+  timeoutMs = 10_000,
+  intervalMs = 250,
+): Promise<T> {
+  const deadline = Date.now() + timeoutMs;
+  let lastError: unknown;
+
+  while (true) {
+    try {
+      return await operation();
+    } catch (error) {
+      lastError = error;
+      const remainingMs = deadline - Date.now();
+      if (remainingMs <= 0) throw lastError;
+      await new Promise((resolve) => setTimeout(resolve, Math.min(intervalMs, remainingMs)));
+    }
+  }
+}

e2eTest/helpers/run.ts

@@ -0,0 +1,76 @@
+import { spawn } from "node:child_process";
+import z from "zod";
+
+/** Given a CLI process, captures its standard output, error output, and exit code. */
+export type RunResult = {
+  stdout: string;
+  stderr: string;
+  exitCode: number;
+};
+
+/** Given an environment variable name, returns its required non-empty value. */
+function requireEnv(key: string): string {
+  const value = process.env[key];
+  if (!value) throw new Error(`missing environment variable for ${key}`);
+  return value;
+}
+
+/** Given a command argument, returns a shell-safe representation for the current platform. */
+function quoteShellArg(value: string): string {
+  if (process.platform === "win32") {
+    return `"${value.replaceAll('"', '\\"')}"`;
+  }
+  return `'${value.replaceAll("'", "'\\''")}'`;
+}
+
+/** Minimal abstraction to handle the running of CLI commands **/
+export class CliRunner {
+  private readonly command = requireEnv("AGENTCORE_CLI_PATH");
+
+  /** Given arguments and a working directory, runs the CLI and captures its result. */
+  run(args: string[], cwd: string): Promise<RunResult> {
+    return new Promise((resolve, reject) => {
+      const child = this.start(args, cwd);
+      let stdout = "";
+      let stderr = "";
+      child.stdout.on("data", (chunk) => (stdout += chunk));
+      child.stderr.on("data", (chunk) => (stderr += chunk));
+      child.on("error", reject);
+      child.on("close", (exitCode) => resolve({ stdout, stderr, exitCode: exitCode ?? -1 }));
+    });
+  }
+
+  /** Given arguments and a working directory, starts the CLI and returns its child process. */
+  start(args: string[], cwd: string) {
+    const command = [this.command, ...args.map(quoteShellArg)].join(" ");
+    return spawn(command, {
+      cwd,
+      env: { ...process.env, AGENTCORE_TELEMETRY_DISABLED: "1", FORCE_COLOR: "0" },
+      shell: true,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+  }
+}
+
+/** Given a Zod schema and CLI result, returns typed output or throws a diagnostic error. */
+export function parseResult<TSchema extends z.ZodType>(
+  schema: TSchema,
+  result: RunResult,
+): z.infer<TSchema> {
+  if (result.exitCode !== 0) {
+    throw new Error(
+      `CLI exited ${result.exitCode}\nstdout: ${result.stdout}\nstderr: ${result.stderr}`,
+    );
+  }
+
+  const parseResult = schema.safeParse(JSON.parse(result.stdout));
+
+  if (!parseResult.success) {
+    throw new Error(
+      `CLI output did not match expected. stdout: ${result.stdout}\nstderr: ${result.stderr}\n` +
+        `error: ${z.prettifyError(parseResult.error)}`,
+    );
+  }
+
+  return parseResult.data;
+}