ci: add live test suite workflow

.github/actions/setup/action.yml

@@ -6,28 +6,107 @@ inputs:
     description: "Skip Compact compiler installation"
     required: false
     default: "false"
+  turbo-cache-key:
+    description: "Cache-key prefix for the turbo build cache, after the runner OS. Set an empty string to skip the cache entirely, for a job that compiles nothing."
+    required: false
+    default: "turbo"
 
 runs:
   using: "composite"
   steps:
+    # Two jobs that share one key are a write race, since both save at the end:
+    # one wins and the other's slice is silently lost to whoever restores next.
+    # A job that compiles per target therefore passes a per-target key. A job
+    # that compiles nothing passes an empty one and skips the step: saving an
+    # empty cache under a key later jobs restore would leave them with nothing
+    # instead of an earlier commit's populated fallback.
+    #
+    # Keyed on the contract sources and the pinned compiler, NOT on the commit.
+    # A commit-keyed entry is written on every push whether or not anything it
+    # caches changed, and these entries are large — `token`'s slice is ~275 MB of
+    # ZK keys, so five commits had left five copies of it, and the repo was at
+    # 6.5 GB of a 10 GB budget. Past that ceiling GitHub evicts least-recently
+    # used, which would silently drop the dependency-tree cache below and undo
+    # what it saves. Content keys mean a docs-only or test-only commit reuses the
+    # existing entry rather than duplicating it.
     - name: Cache turbo build setup
+      if: ${{ inputs.turbo-cache-key != '' }}
       uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
       with:
         path: .turbo
-        key: ${{ runner.os }}-turbo-${{ github.sha }}
+        key: ${{ runner.os }}-${{ inputs.turbo-cache-key }}-${{ hashFiles('contracts/src/**/*.compact', 'contracts/test/integration/**/*.compact', '.github/actions/setup/action.yml') }}
+        # Kept, so a source change still starts from the previous build rather
+        # than from nothing: turbo's store is content-addressed, so the tasks
+        # whose inputs did not move stay hits.
         restore-keys: |
-          ${{ runner.os }}-turbo-
+          ${{ runner.os }}-${{ inputs.turbo-cache-key }}-
+
+    # The materialised dependency tree, not just the package zips.
+    #
+    # `setup-node`'s `cache: yarn` restores `.yarn/cache` alone, so every job
+    # unpacked it into `node_modules` and rebuilt the native modules from source
+    # — `ssh2`, `classic-level`, `cpu-features`, `msgpackr-extract`,
+    # `protobufjs`, all reported as "must be built because it never has been
+    # before". That install was 24s of a 38s setup, in every job of a fan-out
+    # that is now one job per spec file (measured on run 32815780786).
+    #
+    # `.yarn/install-state.gz` is in the entry for the same reason: it is where
+    # yarn records which packages have been built, and without it a job with a
+    # fully restored `node_modules` still reports every native module as "must be
+    # built because it never has been before" and recompiles it. Restoring the
+    # tree alone took the install from 24s to 9s; the state file is what removes
+    # the rest.
+    #
+    # Caching the tree makes the install a no-op instead. It costs transfer:
+    # `node_modules` is 137 MB zstd-compressed against `.yarn/cache`'s 138 MB, so
+    # keeping both roughly doubles the entry — worth it, since a 275 MB entry
+    # restores in seconds (the turbo cache is that size and takes ~3s) and buys
+    # back 24s of CPU. The zips stay in the entry because they are what a miss
+    # needs: without them a cache miss fetches the whole tree over the network.
+    #
+    # Exact key only, no `restore-keys`: a tree restored from a different
+    # lockfile is worse than no tree at all, since `--immutable` would have to
+    # undo it. A miss then behaves exactly as it did before this step existed.
+    #
+    # The `v1` token is the manual version of the key. actions/cache never
+    # overwrites an existing key, so a change to the `path` list above would
+    # silently keep restoring the old, incomplete entry until the lockfile
+    # happens to move — bump the token together with any path-list change.
+    - name: Cache the dependency tree
+      id: deps-cache
+      uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+      with:
+        path: |
+          node_modules
+          contracts/node_modules
+          .yarn/cache
+          .yarn/install-state.gz
+        key: ${{ runner.os }}-deps-v1-${{ hashFiles('yarn.lock', '.yarnrc.yml', '.nvmrc') }}
+
+    # A miss is expected only after a lockfile (or key-version) change; on any
+    # other run it means the entry was evicted or the key list drifted, and the
+    # job silently pays the full install. Make it visible in the annotations.
+    - name: Warn on a dependency cache miss
+      if: steps.deps-cache.outputs.cache-hit != 'true'
+      shell: bash
+      run: echo "::warning::deps cache miss — full install; expected only after a lockfile change"
 
     - name: Enable corepack
       shell: bash
       run: corepack enable
 
+    # `package-manager-cache: false` because the step above already owns the
+    # dependency cache; leaving `setup-node`'s own enabled would restore
+    # `.yarn/cache` a second time.
     - name: Setup Node.js
       uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
       with:
         node-version-file: ".nvmrc"
-        cache: "yarn"
+        package-manager-cache: false
 
+    # Unconditional even on a cache hit: with the tree and yarn's install state
+    # restored this is a fast no-op, and it is the check that a partial restore
+    # fails here rather than inside a spec.
     - name: Install dependencies
       shell: bash
       run: yarn install --immutable

.github/workflows/live-compile.yml

@@ -0,0 +1,96 @@
+name: Live compile
+
+# Builds one live target's contract slice with real ZK keys and uploads it for
+# that target's suite jobs to download.
+#
+# A reusable workflow rather than a matrix job, because `needs` in Actions is
+# job-level: a single compile matrix feeding a single suite matrix makes every
+# spec file wait for the slowest compile of ALL targets (`utils` compiles in 41s
+# and would sit behind `integration`). One compile job per target, each with its
+# own suite matrix downstream, is what lets a target's specs start as soon as
+# that target is built. `live.yml` pairs them.
+on:
+  workflow_call:
+    inputs:
+      target:
+        description: "The live target to compile: a src/ category or `integration`"
+        required: true
+        type: string
+
+# Not inherited from the calling workflow: a reusable workflow gets its own env.
+env:
+  TURBO_TELEMETRY_DISABLED: 1
+
+jobs:
+  compile:
+    name: compile
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+    env:
+      # `setup-compact-action` runs `compact update` on every job, cache hit or
+      # miss, and that call queries the GitHub API. The action passes
+      # `github.token` to its install step but NOT to its update step, so the
+      # query goes out unauthenticated: 60 requests an hour per runner egress IP,
+      # shared by every job in the fan-out. Run 32812144831 hit the ceiling and
+      # `compile-integration` died in setup with
+      #
+      #   API rate limit exceeded for 20.25.10.69
+      #   Error: Failed to update ... Error: Failed to run compactc
+      #   Caused by: No default compiler set
+      #
+      # A cache hit is the fragile path, not the safe one: it skips the install
+      # step, so nothing has set the default compiler, and `compact update` is
+      # what sets it. Its failure is swallowed by `|| true` and the next line
+      # (`compact compile --version`) is what actually fails the job.
+      #
+      # A composite action's steps inherit the calling job's environment, so
+      # exporting the token here authenticates that query (1000 requests an hour)
+      # without patching the action. The upstream fix is for its update step to
+      # take `github.token` the way its install step already does.
+      GITHUB_TOKEN: ${{ github.token }}
+    # Real ZK key generation for one slice, plus the self-heal path's serial
+    # recompile if the tree comes back poisoned (#675).
+    timeout-minutes: 90
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
+        with:
+          egress-policy: audit
+
+      - name: Check out code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 2 # Recommended by turbo team
+
+      # Per-target turbo cache. One key shared by every compile job is a write
+      # race: they all save at the end, one wins, and the rest of the run
+      # restores a slice that is not theirs. Keyed on the commit with a
+      # target-scoped prefix to fall back on, so a night whose sources did not
+      # change restores instead of rebuilding. `live-` rather than `turbo-` so
+      # the regular checks' fallback prefix can never match these entries: they
+      # carry full ZK keys, hundreds of MB a PR check would fetch for nothing.
+      - name: Setup Environment
+        uses: ./.github/actions/setup
+        with:
+          turbo-cache-key: live-${{ inputs.target }}
+
+      # Compiles and verifies the slice, then stops: no stack, no specs. Exits 2
+      # if the tree is still poisoned after the serial recompile.
+      - name: Compile the target
+        env:
+          TARGET: ${{ inputs.target }}
+        run: yarn test:live "$TARGET" --compile-only
+
+      # What this target's suite jobs consume. Retained a day: it is scratch for
+      # this run, not a release artifact. No compression, since ZK keys are
+      # high-entropy and gzip only costs time on them.
+      - name: Upload the compiled artifacts
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: live-artifacts-${{ inputs.target }}
+          path: contracts/artifacts
+          retention-days: 1
+          compression-level: 0
+          if-no-files-found: error

.github/workflows/live-suite.yml

@@ -0,0 +1,128 @@
+name: Live suite
+
+# Runs ONE live spec file against its own local stack, using the artifact tree
+# its target's compile job built. `live.yml` calls this once per spec file,
+# through a matrix per target.
+#
+# One job per file rather than per target because a target is one long serial run
+# (the 2026-08-24 nightly spent 5.75h in `live-token`) while the files are
+# independent: every job resets its own stack, so nothing one spec leaves on the
+# node can fail another, and a red job names the file that caused it.
+on:
+  workflow_call:
+    inputs:
+      target:
+        description: "The live target this spec belongs to"
+        required: true
+        type: string
+      file:
+        description: "Spec path relative to `contracts/`, matching exactly one file"
+        required: true
+        type: string
+      name:
+        description: "The spec's name within its target, for the job and its artifacts"
+        required: true
+        type: string
+      test-filter:
+        description: >-
+          vitest -t regex scoping the run to one leg of a split file
+          (empty = the whole file). The plan job budgets legs at MAX_LEG_MS
+          of estimated duration (see scripts/ci/split.ts).
+        required: false
+        default: ""
+        type: string
+
+# Not inherited from the calling workflow: a reusable workflow gets its own env.
+env:
+  TURBO_TELEMETRY_DISABLED: 1
+
+jobs:
+  suite:
+    name: ${{ inputs.name }}
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+    # One spec file: reset the stack, run it, and verify its failures with a
+    # second round. Far under the old per-target budget, but kept generous — a
+    # single multisig file has run over an hour.
+    timeout-minutes: 350
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
+        with:
+          egress-policy: audit
+
+      - name: Check out code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 2 # Recommended by turbo team
+
+      # No Compact compiler and no turbo cache: this job compiles nothing, and
+      # restoring a cache of build outputs it is about to download would only
+      # duplicate the transfer.
+      - name: Setup Environment
+        uses: ./.github/actions/setup
+        with:
+          skip-compact: "true"
+          turbo-cache-key: ""
+
+      # Built once by this target's compile job. `--prebuilt` below refuses to
+      # rebuild, so a tree that arrives incomplete or with a truncated key aborts
+      # the job (exit 2) instead of reaching the specs, where a failed deploy
+      # would show up as a silently skipped suite.
+      - name: Download the compiled artifacts
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
+        with:
+          name: live-artifacts-${{ inputs.target }}
+          path: contracts/artifacts
+
+      # The same entry point a local run uses, so the two-round flake semantics,
+      # the stack lifecycle, and the verdict are identical — the file is passed
+      # as a full path, which matches exactly one spec. The test-name pattern
+      # travels as an env var, not an argument: the runner appends it to every
+      # vitest spawn itself, so the round-2 flake re-run keeps the same slice,
+      # and its regex metacharacters never meet the shell. Empty means no
+      # pattern, which is what an unsplit leg passes.
+      - name: Run live suite
+        env:
+          TARGET: ${{ inputs.target }}
+          FILE: ${{ inputs.file }}
+          MIDNIGHT_LIVE_TEST_PATTERN: ${{ inputs.test-filter }}
+        run: yarn test:live "$TARGET" "$FILE" --prebuilt
+
+      # The runner tears the stack down itself on every exit path. This is the
+      # safety net for the one path it cannot cover: a job timeout or a cancel
+      # that kills it outright, leaving containers streaming into the logs about
+      # to be uploaded.
+      - name: Stop local stack
+        if: always()
+        run: make env-down
+
+      # The JSON verdict reports are tiny and useful on green runs too (flake
+      # trends). The service and worker logs can reach hundreds of MB after a
+      # multi-hour run, so those upload only when something failed.
+      - name: Upload live reports
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: live-reports-${{ inputs.target }}-${{ inputs.name }}
+          if-no-files-found: ignore
+          retention-days: 14
+          path: logs/live-*.json
+
+      # Two roots on purpose: the stack's container logs land in the repo-root
+      # `logs/` (the Makefile's), and the per-worker harness logs in
+      # `contracts/logs/` (vitest runs with `contracts/` as its cwd).
+      - name: Upload live logs
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: live-logs-${{ inputs.target }}-${{ inputs.name }}
+          if-no-files-found: ignore
+          retention-days: 7
+          path: |
+            contracts/logs/live-harness*.log
+            logs/node.log
+            logs/indexer.log
+            logs/proof-server.log