From 9d03ad445d90b1cd4224de8cb8dfe6ebd95a8e6e Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Thu, 8 Oct 2026 17:01:05 +0100 Subject: [PATCH 1/2] feat(evals): accept any npm dist-tag for cliVersion and drop CLI version workflow inputs localStackRuntime({ cliVersion }) now takes an exact version or any npm dist-tag of supabase instead of a closed stable/beta/next list. Run-level pins move to a single SUPABASE_CLI_DIST_TAG_PINS JSON env var, and the cli_*_version dispatch inputs are removed from eval-refresh.yml. --- .github/workflows/eval-refresh.yml | 33 -- CONTRIBUTING.md | 4 +- README.md | 2 +- .../scripts/run-vercel-evals.test.ts | 193 ++++------- apps/framework/scripts/run-vercel-evals.ts | 86 +++-- .../codex-gpt-6-luna-cli-stable.experiment.ts | 2 +- packages/core/src/index.ts | 8 +- packages/sandbox/src/cli-channel.ts | 153 ++++---- packages/sandbox/src/index.ts | 3 +- packages/sandbox/src/local-stack-runtime.ts | 27 +- packages/sandbox/test/cli-channel.test.ts | 328 +++++++++++------- .../sandbox/test/local-stack-docker.test.ts | 27 +- 12 files changed, 451 insertions(+), 415 deletions(-) diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 7dc78e7a..4ec7e213 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -41,18 +41,6 @@ on: type: boolean required: false default: false - cli_stable_version: - description: "Pin the cli suite's stable CLI version instead of resolving npm's latest dist-tag (blank to resolve)" - required: false - default: "" - cli_beta_version: - description: "Pin the cli suite's beta CLI version instead of resolving npm's beta dist-tag (blank to resolve)" - required: false - default: "" - cli_next_version: - description: "Pin the cli suite's next CLI version instead of resolving npm's next dist-tag (blank to resolve)" - required: false - default: "" schedule: - cron: '15 6 * * *' pull_request: @@ -94,9 +82,6 @@ jobs: sandbox_concurrency: ${{ steps.inputs.outputs.sandbox_concurrency }} filter_changed: ${{ steps.inputs.outputs.filter_changed }} do_merge: ${{ steps.inputs.outputs.do_merge }} - cli_stable_version: ${{ steps.inputs.outputs.cli_stable_version }} - cli_beta_version: ${{ steps.inputs.outputs.cli_beta_version }} - cli_next_version: ${{ steps.inputs.outputs.cli_next_version }} steps: - name: Prepare inputs id: inputs @@ -112,9 +97,6 @@ jobs: runs="${{ inputs.runs }}" timeout_sec="${{ inputs.timeout_sec }}" sandbox_concurrency="${{ inputs.sandbox_concurrency }}" - cli_stable_version="${{ inputs.cli_stable_version }}" - cli_beta_version="${{ inputs.cli_beta_version }}" - cli_next_version="${{ inputs.cli_next_version }}" if [ -z "$suite" ] && [ -z "$eval_id" ]; then echo "::error::Set suite or eval to choose which evals to run." @@ -132,9 +114,6 @@ jobs: runs="3" timeout_sec="720" sandbox_concurrency="250" - cli_stable_version="" - cli_beta_version="" - cli_next_version="" else experiments_override="" eval_id="" @@ -143,9 +122,6 @@ jobs: runs="3" timeout_sec="720" sandbox_concurrency="250" - cli_stable_version="" - cli_beta_version="" - cli_next_version="" fi suite_json="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$suite")" @@ -177,9 +153,6 @@ jobs: echo "sandbox_concurrency=$sandbox_concurrency" echo "filter_changed=$filter_changed" echo "do_merge=$do_merge" - echo "cli_stable_version=$cli_stable_version" - echo "cli_beta_version=$cli_beta_version" - echo "cli_next_version=$cli_next_version" } >> "$GITHUB_OUTPUT" - name: Checkout @@ -319,9 +292,6 @@ jobs: VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} - SUPABASE_CLI_STABLE_VERSION: ${{ needs.prepare.outputs.cli_stable_version }} - SUPABASE_CLI_BETA_VERSION: ${{ needs.prepare.outputs.cli_beta_version }} - SUPABASE_CLI_NEXT_VERSION: ${{ needs.prepare.outputs.cli_next_version }} steps: - name: Checkout uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 @@ -350,9 +320,6 @@ jobs: echo "OPENAI_API_KEY=${OPENAI_API_KEY}" echo "AI_GATEWAY_API_KEY=${AI_GATEWAY_API_KEY}" echo "XAI_API_KEY=${XAI_API_KEY}" - echo "SUPABASE_CLI_STABLE_VERSION=${SUPABASE_CLI_STABLE_VERSION}" - echo "SUPABASE_CLI_BETA_VERSION=${SUPABASE_CLI_BETA_VERSION}" - echo "SUPABASE_CLI_NEXT_VERSION=${SUPABASE_CLI_NEXT_VERSION}" } > .env - name: Run evals diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f3228ba5..6b525d5c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -84,7 +84,7 @@ Common workflows: ## CLI evals -The CLI team owns `evals/cli/` and its results. CLI evals run on `codex-gpt-6-luna-cli-{pinned,stable,beta,next,nodaemon,absent}` under `experiments/cli/`: `pinned` runs the repo's pinned CLI version, `stable`/`beta`/`next` install the latest stable, beta or next CLI, and `nodaemon`/`absent` additionally force Docker-less sandboxes — comparing the same scenario across CLI environments. +The CLI team owns `evals/cli/` and its results. CLI evals run on `codex-gpt-6-luna-cli-{pinned,stable,beta,next,nodaemon,absent}` under `experiments/cli/`: `pinned` runs the repo's pinned CLI version, `stable`/`beta`/`next` install the CLI that npm's `latest`, `beta` or `next` dist-tag points at, and `nodaemon`/`absent` additionally force Docker-less sandboxes — comparing the same scenario across CLI environments. Which evals each arm picks up: @@ -94,6 +94,6 @@ Which evals each arm picks up: Common workflows: - **Add or change a CLI eval.** Add the scenario under `evals/cli//` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview. -- **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli` and `experiment_suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. Leave `cli_stable_version`/`cli_beta_version`/`cli_next_version` blank to resolve npm's latest dist-tags, or pin them to reproduce a specific run. +- **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli` and `experiment_suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. To reproduce a specific run, set an exact `cliVersion` in the experiment's `localStackRuntime` instead of a dist-tag. - **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories. - **Run the unit tests.** `pnpm --filter @supabase-evals/framework test:cli-lib` (the CLI skip predicates in `experiments/cli/lib/` plus every CLI eval's scorer tests) — also part of `pnpm test`. diff --git a/README.md b/README.md index e3c6365e..7b2b52f6 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ An eval's optional `local/` directory is copied into the sandbox workspace befor Set `cliVersion: 2.109.1` in an eval's frontmatter when it requires a specific Supabase CLI release. This overrides an experiment's `localStackRuntime({ cliVersion })` setting; otherwise the runtime setting or repository-wide default applies. -An experiment can instead pass `localStackRuntime({ cliVersion: 'stable' })`, `'beta'` or `'next'` to track npm's dist-tag for the `supabase` package rather than an exact version, resolved once at session start. An eval's own `cliVersion:` pin still wins over either form. +An experiment's `localStackRuntime({ cliVersion })` takes an exact version or any npm dist-tag for the `supabase` package (e.g. `'latest'`, `'beta'`, `'next'`), resolved once per run. An eval's own `cliVersion:` pin still wins over either form. An experiment can pass `localStackRuntime({ docker: 'no-daemon' })` or `'absent'` to stage a sandbox where the Docker daemon is unreachable or the `docker` binary is missing entirely, instead of the default `'available'`. `needsDocker` defaults to `true`; set it `false` in an eval's frontmatter when the scenario can run, and is meaningful, without a Docker daemon (e.g. starting the stack is the agent's own job) — that's what lets a Docker-less experiment pick the eval up. diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index e6478311..ecc25b8c 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -1,5 +1,5 @@ import { APIError } from '@vercel/sandbox'; -import { resolveCliVersion, type CliChannel } from '@supabase-evals/sandbox'; +import { resolveCliDistTag } from '@supabase-evals/sandbox'; import { execFileSync } from 'node:child_process'; import { copyFileSync, @@ -21,20 +21,21 @@ import { cleanupSandbox, downloadResults, finalizeResult, + distTagPinsEnv, FORWARDED_ENV_NAMES, isRetryableSandboxCreateError, isTerminalSandboxCreateError, packWorkspaceScript, parsePairs, - requiredCliChannels, - resolveChannelPins, + requiredCliDistTags, + resolveDistTagPins, runBounded, tagValue, expandJobs, type EvalPair, } from './run-vercel-evals.js'; -vi.mock('@supabase-evals/sandbox', () => ({ resolveCliVersion: vi.fn() })); +vi.mock('@supabase-evals/sandbox', () => ({ resolveCliDistTag: vi.fn() })); const BROKERED_NAMES = BROKERED_KEYS.map(({ name }) => name); @@ -71,41 +72,51 @@ describe('agentEnvironment', () => { } }); - it('omits the CLI channel pins when unset', () => { + it('omits the CLI dist-tag pins when unset', () => { process.env.ANTHROPIC_API_KEY = 'anthropic-value'; const env = agentEnvironment(); expect(env).toBe(`ANTHROPIC_API_KEY=${BROKERED_KEY_PLACEHOLDER}`); - expect(env).not.toContain('SUPABASE_CLI_STABLE_VERSION'); - expect(env).not.toContain('SUPABASE_CLI_BETA_VERSION'); - expect(env).not.toContain('SUPABASE_CLI_NEXT_VERSION'); + expect(env).not.toContain('SUPABASE_CLI_DIST_TAG_PINS'); }); it('prefers an explicit pin over the same-named process.env value', () => { - process.env.SUPABASE_CLI_STABLE_VERSION = 'env-value'; + process.env.SUPABASE_CLI_DIST_TAG_PINS = '{"latest":"1.0.0"}'; const env = agentEnvironment({ - SUPABASE_CLI_STABLE_VERSION: 'pinned-value', + SUPABASE_CLI_DIST_TAG_PINS: '{"latest":"2.117.0"}', }); - expect(env).toContain('SUPABASE_CLI_STABLE_VERSION=pinned-value'); - expect(env).not.toContain('env-value'); + expect(env).toContain('SUPABASE_CLI_DIST_TAG_PINS={"latest":"2.117.0"}'); + expect(env).not.toContain('1.0.0'); }); it('shares one pin value across multiple .env writes, simulating a two-job fan-out', () => { - const pins = { - SUPABASE_CLI_STABLE_VERSION: '2.117.0', - SUPABASE_CLI_BETA_VERSION: '2.118.0-beta.5', - SUPABASE_CLI_NEXT_VERSION: '3.0.0-next.2', - }; + const pins = distTagPinsEnv({ + latest: '2.117.0', + beta: '2.118.0-beta.5', + next: '3.0.0-next.2', + }); const jobOneEnv = agentEnvironment(pins); const jobTwoEnv = agentEnvironment(pins); expect(jobOneEnv).toBe(jobTwoEnv); - expect(jobOneEnv).toContain('SUPABASE_CLI_STABLE_VERSION=2.117.0'); - expect(jobOneEnv).toContain('SUPABASE_CLI_BETA_VERSION=2.118.0-beta.5'); - expect(jobOneEnv).toContain('SUPABASE_CLI_NEXT_VERSION=3.0.0-next.2'); + expect(jobOneEnv).toBe( + 'SUPABASE_CLI_DIST_TAG_PINS={"latest":"2.117.0","beta":"2.118.0-beta.5","next":"3.0.0-next.2"}' + ); + }); +}); + +describe('distTagPinsEnv', () => { + it('forwards a single JSON env var for the resolved pins', () => { + expect(distTagPinsEnv({ canary: '1.4.0-canary.7' })).toEqual({ + SUPABASE_CLI_DIST_TAG_PINS: '{"canary":"1.4.0-canary.7"}', + }); + }); + + it('forwards nothing when no dist-tag was resolved', () => { + expect(distTagPinsEnv({})).toEqual({}); }); }); @@ -166,111 +177,61 @@ describe('brokeredNetworkPolicy', () => { }); }); -describe('resolveChannelPins', () => { - const STABLE_ENV = 'SUPABASE_CLI_STABLE_VERSION'; - const BETA_ENV = 'SUPABASE_CLI_BETA_VERSION'; - const NEXT_ENV = 'SUPABASE_CLI_NEXT_VERSION'; - const BOTH_CHANNELS = new Set(['stable', 'beta']); - const originalValues = new Map(); - +describe('resolveDistTagPins', () => { beforeEach(() => { - for (const name of [STABLE_ENV, BETA_ENV, NEXT_ENV]) { - originalValues.set(name, process.env[name]); - delete process.env[name]; - } - vi.mocked(resolveCliVersion).mockReset(); + vi.mocked(resolveCliDistTag).mockReset(); }); - afterEach(() => { - for (const [name, value] of originalValues) { - if (value === undefined) delete process.env[name]; - else process.env[name] = value; - } - }); - - it('resolves only the requested channels and returns a pin every job can share', async () => { - vi.mocked(resolveCliVersion).mockImplementation(async (channel) => - channel === 'stable' ? '2.117.0' : '2.118.0-beta.5' + it('resolves only the requested dist-tags into one pin map every job can share', async () => { + vi.mocked(resolveCliDistTag).mockImplementation(async (tag) => + tag === 'latest' ? '2.117.0' : '2.118.0-beta.5' ); - const pins = await resolveChannelPins(BOTH_CHANNELS); - - expect(resolveCliVersion).toHaveBeenCalledTimes(2); - expect(resolveCliVersion).toHaveBeenCalledWith('stable'); - expect(resolveCliVersion).toHaveBeenCalledWith('beta'); + const pins = await resolveDistTagPins(new Set(['latest', 'beta'])); - // Two fanned-out jobs writing their own .env from the same pins object - // must get the identical value the resolver was called once for. - const jobOneEnv = agentEnvironment(pins); - const jobTwoEnv = agentEnvironment(pins); - expect(jobOneEnv).toBe(jobTwoEnv); - expect(jobOneEnv).toContain(`${STABLE_ENV}=2.117.0`); - expect(jobOneEnv).toContain(`${BETA_ENV}=2.118.0-beta.5`); - }); + expect(pins).toEqual({ latest: '2.117.0', beta: '2.118.0-beta.5' }); + expect(resolveCliDistTag).toHaveBeenCalledTimes(2); + expect(resolveCliDistTag).toHaveBeenCalledWith('latest'); + expect(resolveCliDistTag).toHaveBeenCalledWith('beta'); - it('resolves only stable when that is the only requested channel, never calling resolveCliVersion with beta', async () => { - vi.mocked(resolveCliVersion).mockImplementation(async (channel) => - channel === 'stable' ? '2.117.0' : '2.118.0-beta.5' + // Two fanned-out jobs writing their own .env from the same pins must get + // the identical value the resolver was called once for. + const forwarded = distTagPinsEnv(pins); + expect(agentEnvironment(forwarded)).toBe(agentEnvironment(forwarded)); + expect(agentEnvironment(forwarded)).toContain( + 'SUPABASE_CLI_DIST_TAG_PINS={"latest":"2.117.0","beta":"2.118.0-beta.5"}' ); - - const pins = await resolveChannelPins(new Set(['stable'])); - - expect(pins).toEqual({ [STABLE_ENV]: '2.117.0' }); - expect(resolveCliVersion).toHaveBeenCalledTimes(1); - expect(resolveCliVersion).not.toHaveBeenCalledWith('beta'); }); - it('resolves the next channel into its own env pin', async () => { - vi.mocked(resolveCliVersion).mockResolvedValue('3.0.0-next.2'); + it('resolves an arbitrary dist-tag without calling the resolver for others', async () => { + vi.mocked(resolveCliDistTag).mockResolvedValue('1.4.0-canary.7'); - const pins = await resolveChannelPins(new Set(['next'])); + const pins = await resolveDistTagPins(new Set(['canary'])); - expect(pins).toEqual({ [NEXT_ENV]: '3.0.0-next.2' }); - expect(resolveCliVersion).toHaveBeenCalledWith('next'); + expect(pins).toEqual({ canary: '1.4.0-canary.7' }); + expect(resolveCliDistTag).toHaveBeenCalledTimes(1); + expect(resolveCliDistTag).toHaveBeenCalledWith('canary'); }); - it('does no network work for an empty channel set', async () => { - const pins = await resolveChannelPins(new Set()); + it('does no network work for an empty dist-tag set', async () => { + const pins = await resolveDistTagPins(new Set()); expect(pins).toEqual({}); - expect(resolveCliVersion).not.toHaveBeenCalled(); - }); - - it('uses an already-set env var verbatim without calling the resolver', async () => { - process.env[STABLE_ENV] = '9.9.9'; - vi.mocked(resolveCliVersion).mockImplementation( - async () => '2.118.0-beta.5' - ); - - const pins = await resolveChannelPins(BOTH_CHANNELS); - - expect(pins[STABLE_ENV]).toBe('9.9.9'); - expect(resolveCliVersion).toHaveBeenCalledTimes(1); - expect(resolveCliVersion).toHaveBeenCalledWith('beta'); + expect(resolveCliDistTag).not.toHaveBeenCalled(); }); it('propagates a resolution failure rather than swallowing it', async () => { - vi.mocked(resolveCliVersion).mockRejectedValue( + vi.mocked(resolveCliDistTag).mockRejectedValue( new Error('npm unreachable') ); - await expect( - resolveChannelPins(new Set(['stable'])) - ).rejects.toThrow('npm unreachable'); - }); - - it('treats a blank or whitespace-only env var as unset, resolving it instead', async () => { - process.env[STABLE_ENV] = ' '; - vi.mocked(resolveCliVersion).mockImplementation(async () => '2.117.0'); - - const pins = await resolveChannelPins(new Set(['stable'])); - - expect(pins[STABLE_ENV]).toBe('2.117.0'); - expect(resolveCliVersion).toHaveBeenCalledWith('stable'); + await expect(resolveDistTagPins(new Set(['latest']))).rejects.toThrow( + 'npm unreachable' + ); }); }); -describe('requiredCliChannels', () => { +describe('requiredCliDistTags', () => { const pair = (overrides: Partial = {}): EvalPair => ({ eval_id: 'eval-1', experiment: 'experiment-1', @@ -279,23 +240,23 @@ describe('requiredCliChannels', () => { ...overrides, }); - it('resolves only the channel a stable-tagged experiment declares', async () => { + it('resolves only the dist-tag a latest-tagged experiment declares', async () => { const loadExperimentConfig = vi.fn(async () => ({ - localStack: { cliChannel: 'stable' as const }, + localStack: { cliDistTag: 'latest' }, })); - const channels = await requiredCliChannels([pair()], { + const distTags = await requiredCliDistTags([pair()], { loadEvalMetadata: () => ({ cliVersion: undefined }), loadExperimentConfig, }); - expect(channels).toEqual(new Set(['stable'])); + expect(distTags).toEqual(new Set(['latest'])); }); - it('resolves nothing when no experiment in the pair set declares a channel', async () => { + it('resolves nothing when no experiment in the pair set declares a dist-tag', async () => { const loadExperimentConfig = vi.fn(async () => ({})); - const channels = await requiredCliChannels( + const distTags = await requiredCliDistTags( [pair(), pair({ eval_id: 'eval-2', experiment: 'experiment-2' })], { loadEvalMetadata: () => ({ cliVersion: undefined }), @@ -303,29 +264,29 @@ describe('requiredCliChannels', () => { } ); - expect(channels.size).toBe(0); + expect(distTags.size).toBe(0); }); - it("an eval's pinned cliVersion contributes no channel, even when its experiment declares one", async () => { + it("an eval's pinned cliVersion contributes no dist-tag, even when its experiment declares one", async () => { const loadExperimentConfig = vi.fn(async () => ({ - localStack: { cliChannel: 'beta' as const }, + localStack: { cliDistTag: 'beta' }, })); - const channels = await requiredCliChannels([pair()], { + const distTags = await requiredCliDistTags([pair()], { loadEvalMetadata: () => ({ cliVersion: '2.109.1' }), loadExperimentConfig, }); - expect(channels.size).toBe(0); + expect(distTags.size).toBe(0); expect(loadExperimentConfig).not.toHaveBeenCalled(); }); - it('unions channels across pairs without resolving an experiment config twice', async () => { + it('unions dist-tags across pairs without resolving an experiment config twice', async () => { const loadExperimentConfig = vi.fn(async () => ({ - localStack: { cliChannel: 'beta' as const }, + localStack: { cliDistTag: 'beta' }, })); - const channels = await requiredCliChannels( + const distTags = await requiredCliDistTags( [pair(), pair({ eval_id: 'eval-2' })], { loadEvalMetadata: () => ({ cliVersion: undefined }), @@ -333,13 +294,13 @@ describe('requiredCliChannels', () => { } ); - expect(channels).toEqual(new Set(['beta'])); + expect(distTags).toEqual(new Set(['beta'])); expect(loadExperimentConfig).toHaveBeenCalledTimes(1); }); it('throws naming an experiment that cannot be resolved to a config', async () => { await expect( - requiredCliChannels([pair({ experiment: 'ghost' })], { + requiredCliDistTags([pair({ experiment: 'ghost' })], { loadEvalMetadata: () => ({ cliVersion: undefined }), loadExperimentConfig: async () => { throw new Error('no experiment config found for "ghost"'); diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index 60a376b0..540fa3db 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -13,7 +13,7 @@ import { sandboxUsageSchema, type SandboxUsage, } from '@supabase-evals/core/eval-metadata'; -import { resolveCliVersion, type CliChannel } from '@supabase-evals/sandbox'; +import { resolveCliDistTag } from '@supabase-evals/sandbox'; import { execFileSync } from 'node:child_process'; import { readFileSync, renameSync, writeFileSync } from 'node:fs'; import { join, relative, resolve } from 'node:path'; @@ -61,19 +61,10 @@ export const BROKERED_KEYS: { ]; export const BROKERED_KEY_PLACEHOLDER = 'injected-by-sandbox-firewall'; /** - * Pins the CLI channel version resolved for this run across sandbox jobs. + * Pins the CLI dist-tag versions resolved for this run across sandbox jobs. * Provider keys belong in `BROKERED_KEYS` so the sandbox never sees them. */ -export const FORWARDED_ENV_NAMES = [ - 'SUPABASE_CLI_STABLE_VERSION', - 'SUPABASE_CLI_BETA_VERSION', - 'SUPABASE_CLI_NEXT_VERSION', -]; -const CLI_CHANNEL_ENV: Record = { - stable: 'SUPABASE_CLI_STABLE_VERSION', - beta: 'SUPABASE_CLI_BETA_VERSION', - next: 'SUPABASE_CLI_NEXT_VERSION', -}; +export const FORWARDED_ENV_NAMES = ['SUPABASE_CLI_DIST_TAG_PINS']; /** * Slack for the non-agent work inside `pnpm eval` (supabase start, resets, * scoring, export). Cold image pulls alone can take ~10 min. @@ -125,46 +116,51 @@ interface PairOptions extends RunnerOptions { } /** - * Resolves each requested CLI channel's pin once, so every sandbox job in a + * Resolves each requested CLI dist-tag's pin once, so every sandbox job in a * fan-out runs against the same concrete version rather than each resolving - * independently and risking a mid-run release landing between them. An - * already-set env var is used verbatim, matching the workflow/manual - * override path in resolveCliVersion. An empty set does no network work. + * independently and risking a mid-run release landing between them. A tag + * already pinned in SUPABASE_CLI_DIST_TAG_PINS is used verbatim by + * resolveCliDistTag. An empty set does no network work. */ -export async function resolveChannelPins( - channels: ReadonlySet +export async function resolveDistTagPins( + distTags: ReadonlySet ): Promise> { const resolved = await Promise.all( - [...channels].map(async (channel) => { - const envVar = CLI_CHANNEL_ENV[channel]; - // A workflow that exports a blank input still sets the env var, so - // blank/whitespace must be treated as unset rather than as a pin of ''. - const override = process.env[envVar]?.trim(); - return [envVar, override || (await resolveCliVersion(channel))] as const; - }) + [...distTags].map( + async (distTag) => [distTag, await resolveCliDistTag(distTag)] as const + ) ); return Object.fromEntries(resolved); } -export interface RequiredCliChannelsDeps { +/** The `.env` pin forwarded to every sandbox job, or none when no dist-tag was resolved. */ +export function distTagPinsEnv( + pins: Readonly> +): Record { + return Object.keys(pins).length > 0 + ? { SUPABASE_CLI_DIST_TAG_PINS: JSON.stringify(pins) } + : {}; +} + +export interface RequiredCliDistTagsDeps { loadEvalMetadata: (pair: EvalPair) => Pick; loadExperimentConfig: ( experiment: string - ) => Promise<{ localStack?: { cliChannel?: CliChannel } }>; + ) => Promise<{ localStack?: { cliDistTag?: string } }>; } /** - * Maps a pair set to the CLI channels at least one pair needs, so - * resolveChannelPins only resolves those. An eval's own `cliVersion` - * frontmatter is an exact pin that wins over its experiment's channel and - * needs no resolution; a pair whose experiment has no `localStack.cliChannel` + * Maps a pair set to the CLI dist-tags at least one pair needs, so + * resolveDistTagPins only resolves those. An eval's own `cliVersion` + * frontmatter is an exact pin that wins over its experiment's dist-tag and + * needs no resolution; a pair whose experiment has no `localStack.cliDistTag` * needs none either. */ -export async function requiredCliChannels( +export async function requiredCliDistTags( pairs: readonly EvalPair[], - { loadEvalMetadata, loadExperimentConfig }: RequiredCliChannelsDeps -): Promise> { - const channels = new Set(); + { loadEvalMetadata, loadExperimentConfig }: RequiredCliDistTagsDeps +): Promise> { + const distTags = new Set(); const configs = new Map>(); for (const pair of pairs) { @@ -175,11 +171,11 @@ export async function requiredCliChannels( config = loadExperimentConfig(pair.experiment); configs.set(pair.experiment, config); } - const channel = (await config).localStack?.cliChannel; - if (channel) channels.add(channel); + const distTag = (await config).localStack?.cliDistTag; + if (distTag) distTags.add(distTag); } - return channels; + return distTags; } /** Mirrors run-eval.ts's evals///PROMPT.md convention. */ @@ -207,8 +203,8 @@ function experimentPathsByName(): Promise> { } /** - * Throws rather than treating a config it can't find as needing no channel, - * which would silently resolve every channel per-sandbox again. + * Throws rather than treating a config it can't find as needing no dist-tag, + * which would silently resolve every dist-tag per-sandbox again. */ async function loadExperimentConfig( experiment: string @@ -259,14 +255,14 @@ async function runPairs(options: RunnerOptions): Promise { `max ${options.concurrency} at a time` ); - // Resolved once, for only the channels this run's pairs actually need, so + // Resolved once, for only the dist-tags this run's pairs actually need, so // every job below writes the same pin rather than each sandbox resolving - // its own channel version independently. - const channels = await requiredCliChannels(options.pairs, { + // its own dist-tag version independently. + const distTags = await requiredCliDistTags(options.pairs, { loadEvalMetadata, loadExperimentConfig, }); - const pins = await resolveChannelPins(channels); + const pins = distTagPinsEnv(await resolveDistTagPins(distTags)); const results = await runBounded( jobs, @@ -825,7 +821,7 @@ export function brokeredNetworkPolicy(): NetworkPolicy { return { allow }; } -/** Builds the sandbox `.env` from key placeholders and CLI channel pins. An explicit pin wins over process.env. */ +/** Builds the sandbox `.env` from key placeholders and CLI dist-tag pins. An explicit pin wins over process.env. */ export function agentEnvironment(pins: Record = {}): string { const lines: string[] = []; for (const { name } of BROKERED_KEYS) { diff --git a/experiments/cli/codex-gpt-6-luna-cli-stable.experiment.ts b/experiments/cli/codex-gpt-6-luna-cli-stable.experiment.ts index fe68bc7c..12d330fd 100644 --- a/experiments/cli/codex-gpt-6-luna-cli-stable.experiment.ts +++ b/experiments/cli/codex-gpt-6-luna-cli-stable.experiment.ts @@ -8,6 +8,6 @@ export default defineExperiment({ suite: ['cli'], // Currently equal to the pin (npm `latest` == SUPABASE_CLI_VERSION); kept // as drift insurance between pin bumps. - localStack: localStackRuntime({ cliVersion: 'stable' }), + localStack: localStackRuntime({ cliVersion: 'latest' }), skipEval: skipUnlessCli, }); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d931b73f..08082d8a 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -333,8 +333,8 @@ export interface LocalStackStatus { */ export interface LocalStackEnvironmentMarker { runtime: 'local-stack'; - /** The channel (`stable`/`beta`/`next`) the session's CLI version resolved from, when the runtime named one. */ - channel?: 'stable' | 'beta' | 'next'; + /** The npm dist-tag the session's CLI version resolved from. */ + channel?: string; cliVersion: string; docker: 'available' | 'no-daemon' | 'absent'; sessionStartedMs: number; @@ -635,8 +635,8 @@ export type LocalStackSession = { export type LocalStackRuntime = { id: string; startSession(args: LocalStackSessionArgs): Promise; - /** Channel (e.g. `beta`) this runtime resolves its CLI version against, unset when it's pinned to an exact version. */ - cliChannel?: 'stable' | 'beta' | 'next'; + /** npm dist-tag (e.g. `beta`) this runtime resolves its CLI version against, unset when it's pinned to an exact version. */ + cliDistTag?: string; }; export type ExperimentConfig = { diff --git a/packages/sandbox/src/cli-channel.ts b/packages/sandbox/src/cli-channel.ts index a57890eb..3ec0dc25 100644 --- a/packages/sandbox/src/cli-channel.ts +++ b/packages/sandbox/src/cli-channel.ts @@ -1,13 +1,11 @@ /** - * Resolves "latest stable"/"latest beta"/"latest next" Supabase CLI versions from npm's - * dist-tags. npm can point at a still-draft release with no downloadable - * asset, so the resolved version's `.deb` is verified first. + * Resolves a Supabase CLI version from an exact version or any npm dist-tag of + * the `supabase` package. npm can point a tag at a still-draft release with no + * downloadable asset, so the resolved version's `.deb` is verified first. */ import { isRecord } from '@supabase-evals/core/json'; -export type CliChannel = 'stable' | 'beta' | 'next'; - const NPM_DIST_TAGS_URL = 'https://registry.npmjs.org/-/package/supabase/dist-tags'; @@ -15,9 +13,11 @@ const NPM_DIST_TAGS_URL = const NPM_PACKUMENT_URL = 'https://registry.npmjs.org/supabase'; const NPM_INSTALL_V1_ACCEPT = 'application/vnd.npm.install-v1+json'; -// npm's beta dist-tag can carry a prerelease suffix like -rc.1, not just -beta.N. +// A dist-tag can carry any prerelease suffix (-rc.1, -next.2), not just -beta.N. const VERSION_RE = /^\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$/; +const DIST_TAG_RE = /^[A-Za-z][A-Za-z0-9._-]*$/; + // Only the -beta.N shape has a defined ordering the walk-back below can search. const BETA_SUFFIX_RE = /^\d+\.\d+\.\d+-beta\.\d+$/; @@ -27,25 +27,18 @@ const BETA_VERSION_RE = /^(\d+)\.(\d+)\.(\d+)-beta\.(\d+)$/; // Caps the walk-back so a broken release can't chain into unbounded GitHub requests. const MAX_BETA_FALLBACK_CANDIDATES = 5; -const ENV_OVERRIDE: Record = { - stable: 'SUPABASE_CLI_STABLE_VERSION', - beta: 'SUPABASE_CLI_BETA_VERSION', - next: 'SUPABASE_CLI_NEXT_VERSION', -}; +/** JSON object mapping dist-tag names to exact versions, pinning a run's resolution. */ +const DIST_TAG_PINS_ENV = 'SUPABASE_CLI_DIST_TAG_PINS'; -const DIST_TAG: Record = { - stable: 'latest', - beta: 'beta', - next: 'next', -}; +const versionCache = new Map>(); -const versionCache = new Map>(); - -const CLI_CHANNELS = new Set(['stable', 'beta', 'next']); +/** Whether `value` has the shape of an exact CLI version (optionally `v`-prefixed) rather than a dist-tag name. */ +export function isExactCliVersion(value: string): boolean { + return VERSION_RE.test(stripVersionPrefix(value)); +} -/** Whether `value` names a channel (`'stable'` | `'beta'` | `'next'`) rather than an exact version. */ -export function isCliChannel(value: string): value is CliChannel { - return CLI_CHANNELS.has(value as CliChannel); +function stripVersionPrefix(value: string): string { + return value.replace(/^v/, ''); } /** Must match the download URL installSupabaseCli() uses in supabase.ts. */ @@ -80,71 +73,111 @@ async function debAssetExists(version: string): Promise { } /** - * Resolves a channel to a concrete CLI version, memoised per channel for the - * process lifetime. A rejection clears the cache entry so the next call - * retries; never falls back to the pinned SUPABASE_CLI_VERSION, since that - * would silently mislabel data. + * Resolves an npm dist-tag of `supabase` to a concrete CLI version, memoised + * per tag for the process lifetime. A rejection clears the cache entry so the + * next call retries; never falls back to the pinned SUPABASE_CLI_VERSION, + * since that would silently mislabel data. */ -export async function resolveCliVersion(channel: CliChannel): Promise { - const cached = versionCache.get(channel); +export async function resolveCliDistTag(tag: string): Promise { + const cached = versionCache.get(tag); if (cached) return cached; - const promise = resolveCliVersionUncached(channel); - versionCache.set(channel, promise); + const promise = resolveCliDistTagUncached(tag); + versionCache.set(tag, promise); promise.catch(() => { - if (versionCache.get(channel) === promise) versionCache.delete(channel); + if (versionCache.get(tag) === promise) versionCache.delete(tag); }); return promise; } -/** Resolves `value`: a channel tag (`'stable'` | `'beta'` | `'next'`) resolves against npm; an exact version or `undefined` passes through unchanged. */ +/** Resolves `value`: an exact version passes through (minus any leading `v`), `undefined` stays `undefined`, anything else is an npm dist-tag. */ export async function resolveCliVersionOption( - value: string | CliChannel | undefined + value: string | undefined ): Promise { if (value === undefined) return undefined; - return isCliChannel(value) ? resolveCliVersion(value) : value; + return isExactCliVersion(value) + ? stripVersionPrefix(value) + : resolveCliDistTag(value); } -async function resolveCliVersionUncached(channel: CliChannel): Promise { - const envVar = ENV_OVERRIDE[channel]; - const override = process.env[envVar]?.trim().replace(/^v/, ''); - if (override) { - if (!VERSION_RE.test(override)) { +function readDistTagPins(): Record { + const raw = process.env[DIST_TAG_PINS_ENV]?.trim(); + if (!raw) return {}; + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error(`${DIST_TAG_PINS_ENV} is not valid JSON: ${raw}`); + } + if (!isRecord(parsed)) { + throw new Error( + `${DIST_TAG_PINS_ENV} must be a JSON object mapping dist-tags to versions` + ); + } + + const pins: Record = {}; + for (const [tag, value] of Object.entries(parsed)) { + const version = + typeof value === 'string' ? stripVersionPrefix(value.trim()) : ''; + if (!VERSION_RE.test(version)) { throw new Error( - `${envVar}=${JSON.stringify(override)} is not a valid Supabase CLI version` + `${DIST_TAG_PINS_ENV} pins "${tag}" to ${JSON.stringify(value)}, which is not a valid Supabase CLI version` ); } - // An explicit pin is trusted as-is — no asset check, network or otherwise. - return override; + pins[tag] = version; + } + return pins; +} + +async function resolveCliDistTagUncached(distTag: string): Promise { + if (!DIST_TAG_RE.test(distTag)) { + throw new Error( + `${JSON.stringify(distTag)} is neither an exact Supabase CLI version nor a valid npm dist-tag name` + ); } + // An explicit pin is trusted as-is — no asset check, network or otherwise. + const pinned = readDistTagPins()[distTag]; + if (pinned) return pinned; + const response = await fetch(NPM_DIST_TAGS_URL, { signal: AbortSignal.timeout(15_000), }); if (!response.ok) { throw new Error( - `failed to resolve the latest ${channel} Supabase CLI version: ` + + `failed to resolve the "${distTag}" Supabase CLI dist-tag: ` + `GET ${NPM_DIST_TAGS_URL} -> ${response.status} ${response.statusText}` ); } const tags = await response.json(); - const distTag = DIST_TAG[channel]; - const version = isRecord(tags) ? tags[distTag] : undefined; + if (!isRecord(tags)) { + throw new Error( + `npm dist-tags for "supabase" were not a JSON object: ${JSON.stringify(tags)}` + ); + } + const version = tags[distTag]; + if (version === undefined) { + throw new Error( + `npm has no "${distTag}" dist-tag for "supabase"; available dist-tags: ` + + Object.keys(tags).join(', ') + ); + } if (typeof version !== 'string' || !VERSION_RE.test(version)) { throw new Error( - `npm dist-tags for "supabase" did not have a valid "${distTag}" version ` + - `for the ${channel} channel: ${JSON.stringify(version)}` + `npm dist-tags for "supabase" did not have a valid "${distTag}" version: ` + + JSON.stringify(version) ); } // A transient failure here throws rather than silently walking back to an - // older beta (or never walking back, for stable and next). + // older beta (or never walking back, for other shapes). if (await debAssetExists(version)) return version; // npm's dist-tag can point at a version whose GitHub release is still a - // draft with no downloadable .deb. Only beta walks back to an older - // published version; stable and next are never guessed at. - if (channel !== 'beta') { + // draft with no downloadable .deb. Only -beta.N versions walk back to an + // older published version; other shapes are never guessed at. + if (!BETA_SUFFIX_RE.test(version)) { throw new Error( `npm's "${distTag}" dist-tag for "supabase" points at ${version}, but its ` + `release asset is missing (checked amd64 and arm64 .deb assets at ` + @@ -152,7 +185,7 @@ async function resolveCliVersionUncached(channel: CliChannel): Promise { ); } - return resolveFallbackBetaVersion(version); + return resolveFallbackBetaVersion(version, distTag); } /** @@ -164,24 +197,16 @@ async function resolveCliVersionUncached(channel: CliChannel): Promise { * candidate is absent. */ async function resolveFallbackBetaVersion( - unpublishedVersion: string + unpublishedVersion: string, + distTag: string ): Promise { - if (!BETA_SUFFIX_RE.test(unpublishedVersion)) { - throw new Error( - `npm's "beta" dist-tag for "supabase" points at ${unpublishedVersion}, ` + - `whose release asset is missing (checked amd64 and arm64 .deb assets ` + - `at ${releaseTagUrl(unpublishedVersion)}), and its version does not ` + - 'match the X.Y.Z-beta.N shape this fallback can walk back through' - ); - } - const response = await fetch(NPM_PACKUMENT_URL, { headers: { Accept: NPM_INSTALL_V1_ACCEPT }, signal: AbortSignal.timeout(15_000), }); if (!response.ok) { throw new Error( - `failed to look up published beta versions for "supabase": ` + + `failed to look up published beta versions for the "${distTag}" dist-tag of "supabase": ` + `GET ${NPM_PACKUMENT_URL} -> ${response.status} ${response.statusText}` ); } @@ -206,7 +231,7 @@ async function resolveFallbackBetaVersion( } throw new Error( - `no published beta Supabase CLI release has a downloadable .deb asset; ` + + `no published beta Supabase CLI release has a downloadable .deb asset for the "${distTag}" dist-tag; ` + `checked ${[unpublishedVersion, ...candidates].join(', ')} via ` + `${NPM_PACKUMENT_URL}` ); diff --git a/packages/sandbox/src/index.ts b/packages/sandbox/src/index.ts index 8cfc4fb7..d20c442b 100644 --- a/packages/sandbox/src/index.ts +++ b/packages/sandbox/src/index.ts @@ -1,5 +1,4 @@ -export { resolveCliVersion } from './cli-channel.js'; -export type { CliChannel } from './cli-channel.js'; +export { resolveCliDistTag } from './cli-channel.js'; export { DockerSandbox, dockerCli } from './docker-sandbox.js'; export type { DockerSandboxOptions, diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index ae1439cc..e1f4efb7 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -24,11 +24,7 @@ import { teardownSupabaseProject, } from './supabase.js'; import { buildSkillsPrompt, installSkills } from './skills.js'; -import { - isCliChannel, - resolveCliVersionOption, - type CliChannel, -} from './cli-channel.js'; +import { isExactCliVersion, resolveCliVersionOption } from './cli-channel.js'; import type { SupabaseService } from './types.js'; const DEFAULT_BASH_TIMEOUT_SEC = 240; @@ -53,11 +49,11 @@ const STACK_CONFIG_RETRY_MS = 2_000; export interface LocalStackRuntimeOptions { /** * Supabase CLI version baked into the sandbox image: an exact version - * (e.g. `2.109.1`) or a channel tag (`'stable'` | `'beta'` | `'next'`) resolved - * against npm's dist-tag and memoised per process. An eval's own + * (e.g. `2.109.1`) or any npm dist-tag of `supabase` (e.g. `'latest'`, + * `'beta'`, `'next'`), resolved once per process. An eval's own * `cliVersion:` frontmatter pin always wins over this option. */ - cliVersion?: CliChannel | (string & {}); + cliVersion?: string; /** * Supabase MCP feature groups to expose to the agent when the eval links to * a hosted project (`hostedProject: true`). The MCP server runs host-side and @@ -130,8 +126,8 @@ export function localStackRuntime( ): LocalStackRuntime { return { id: buildRuntimeId(options), - cliChannel: - options.cliVersion !== undefined && isCliChannel(options.cliVersion) + cliDistTag: + options.cliVersion !== undefined && !isExactCliVersion(options.cliVersion) ? options.cliVersion : undefined, async startSession({ @@ -148,11 +144,11 @@ export function localStackRuntime( // Stamped before setup so it's comparable with the scorer's PID-1 fallback. const sessionStartedMs = Date.now(); const docker = options.docker ?? 'available'; - // Only an unpinned eval inherits a channel; an eval's own pin always wins. + // Only an unpinned eval inherits a dist-tag; an eval's own pin always wins. const channel = cliVersion === undefined && options.cliVersion !== undefined && - isCliChannel(options.cliVersion) + !isExactCliVersion(options.cliVersion) ? options.cliVersion : undefined; const version = @@ -369,7 +365,7 @@ function buildLocalStackMarker( docker: DockerState, cliVersion: string, sessionStartedMs: number, - channel?: CliChannel + channel?: string ): LocalStackEnvironmentMarker { return { runtime: 'local-stack', @@ -836,10 +832,7 @@ function isLocalStackEnvironmentMarker( value.docker === 'no-daemon' || value.docker === 'absent') && typeof value.sessionStartedMs === 'number' && - (value.channel === undefined || - value.channel === 'stable' || - value.channel === 'beta' || - value.channel === 'next') + (value.channel === undefined || typeof value.channel === 'string') ); } diff --git a/packages/sandbox/test/cli-channel.test.ts b/packages/sandbox/test/cli-channel.test.ts index ece36e9f..2fc68231 100644 --- a/packages/sandbox/test/cli-channel.test.ts +++ b/packages/sandbox/test/cli-channel.test.ts @@ -1,9 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { cliDebUrl } from '../src/cli-channel.js'; -const STABLE_ENV = 'SUPABASE_CLI_STABLE_VERSION'; -const BETA_ENV = 'SUPABASE_CLI_BETA_VERSION'; -const NEXT_ENV = 'SUPABASE_CLI_NEXT_VERSION'; +const PINS_ENV = 'SUPABASE_CLI_DIST_TAG_PINS'; const DIST_TAGS_URL = 'https://registry.npmjs.org/-/package/supabase/dist-tags'; const PACKUMENT_URL = 'https://registry.npmjs.org/supabase'; const INSTALL_V1_ACCEPT = 'application/vnd.npm.install-v1+json'; @@ -31,7 +29,7 @@ function headResponse(ok: boolean, status?: number, statusText?: string) { /** * Routes a single stubbed `fetch` by method + URL, mirroring the real * mixture of GET (dist-tags, packument) and HEAD (amd64 + arm64 asset check) - * calls resolveCliVersion makes. + * calls resolveCliDistTag makes. */ function routedFetchMock(routes: { distTags?: unknown; @@ -62,28 +60,24 @@ function routedFetchMock(routes: { beforeEach(() => { vi.resetModules(); - delete process.env[STABLE_ENV]; - delete process.env[BETA_ENV]; - delete process.env[NEXT_ENV]; + delete process.env[PINS_ENV]; }); afterEach(() => { vi.unstubAllGlobals(); - delete process.env[STABLE_ENV]; - delete process.env[BETA_ENV]; - delete process.env[NEXT_ENV]; + delete process.env[PINS_ENV]; }); -describe('resolveCliVersion', () => { - it('resolves the stable channel from the "latest" dist-tag when its asset exists', async () => { +describe('resolveCliDistTag', () => { + it('resolves the "latest" dist-tag when its asset exists', async () => { const fetchMock = routedFetchMock({ distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, assetOk: () => true, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + await expect(resolveCliDistTag('latest')).resolves.toBe('1.2.3'); const headCalls = fetchMock.mock.calls.filter( ([, init]) => init?.method === 'HEAD' @@ -94,64 +88,26 @@ describe('resolveCliVersion', () => { ]); }); - it('resolves the beta channel from the "beta" dist-tag when its asset exists', async () => { + it('resolves the "beta" dist-tag when its asset exists', async () => { const fetchMock = routedFetchMock({ distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, assetOk: () => true, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('beta')).resolves.toBe('1.3.0-beta.1'); + await expect(resolveCliDistTag('beta')).resolves.toBe('1.3.0-beta.1'); }); - it('resolves the next channel from the "next" dist-tag when its asset exists', async () => { + it('resolves the "next" dist-tag when its asset exists', async () => { const fetchMock = routedFetchMock({ distTags: { latest: '1.2.3', beta: '1.3.0-beta.1', next: '3.0.0-next.2' }, assetOk: () => true, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('next')).resolves.toBe('3.0.0-next.2'); - }); - - it('honours the next env override without touching the network', async () => { - process.env[NEXT_ENV] = 'v3.0.0-next.3'; - const fetchMock = vi.fn(); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); - - await expect(resolveCliVersion('next')).resolves.toBe('3.0.0-next.3'); - expect(fetchMock).not.toHaveBeenCalled(); - }); - - it('prefers the env override over the network and strips a leading v', async () => { - process.env[STABLE_ENV] = 'v9.9.9'; - const fetchMock = vi.fn(); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); - - await expect(resolveCliVersion('stable')).resolves.toBe('9.9.9'); - expect(fetchMock).not.toHaveBeenCalled(); - }); - - it('trims surrounding whitespace from the env override', async () => { - process.env[BETA_ENV] = ' 1.4.0-rc.2 '; - vi.stubGlobal('fetch', vi.fn()); - const { resolveCliVersion } = await import('../src/cli-channel.js'); - - await expect(resolveCliVersion('beta')).resolves.toBe('1.4.0-rc.2'); - }); - - it('throws naming the env var when the override is not a valid version', async () => { - process.env[BETA_ENV] = 'not-a-version'; - vi.stubGlobal('fetch', vi.fn()); - const { resolveCliVersion } = await import('../src/cli-channel.js'); - - await expect(resolveCliVersion('beta')).rejects.toThrow( - `${BETA_ENV}="not-a-version" is not a valid Supabase CLI version` - ); + await expect(resolveCliDistTag('next')).resolves.toBe('3.0.0-next.2'); }); it('throws with the HTTP status when the registry request fails', async () => { @@ -164,20 +120,20 @@ describe('resolveCliVersion', () => { }, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('stable')).rejects.toThrow( + await expect(resolveCliDistTag('latest')).rejects.toThrow( `${DIST_TAGS_URL} -> 500 Internal Server Error` ); }); - it('throws when the requested dist-tag is missing from the response', async () => { + it('throws listing the available dist-tags when the requested one is missing', async () => { const fetchMock = routedFetchMock({ distTags: { beta: '1.0.0-beta.1' } }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('stable')).rejects.toThrow( - /did not have a valid "latest" version for the stable channel/ + await expect(resolveCliDistTag('latest')).rejects.toThrow( + 'npm has no "latest" dist-tag for "supabase"; available dist-tags: beta' ); }); @@ -186,20 +142,20 @@ describe('resolveCliVersion', () => { distTags: { latest: 'not-a-version' }, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('stable')).rejects.toThrow( - /did not have a valid "latest" version for the stable channel/ + await expect(resolveCliDistTag('latest')).rejects.toThrow( + /did not have a valid "latest" version/ ); }); it('treats an array dist-tags response as invalid rather than a record', async () => { const fetchMock = routedFetchMock({ distTags: ['not', 'a', 'record'] }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('stable')).rejects.toThrow( - /did not have a valid "latest" version for the stable channel/ + await expect(resolveCliDistTag('latest')).rejects.toThrow( + /were not a JSON object/ ); }); @@ -209,10 +165,10 @@ describe('resolveCliVersion', () => { assetOk: () => true, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); - await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + await expect(resolveCliDistTag('latest')).resolves.toBe('1.2.3'); + await expect(resolveCliDistTag('latest')).resolves.toBe('1.2.3'); // One GET (dist-tags) + two HEAD (amd64+arm64 asset check) — the second // call hits the cache. @@ -237,18 +193,18 @@ describe('resolveCliVersion', () => { throw new Error(`unexpected fetch: ${url}`); }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('stable')).rejects.toThrow('500'); - await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + await expect(resolveCliDistTag('latest')).rejects.toThrow('500'); + await expect(resolveCliDistTag('latest')).resolves.toBe('1.2.3'); // A third call must still hit the cache populated by the successful // retry — the rejected first promise's cleanup must not evict it. - await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + await expect(resolveCliDistTag('latest')).resolves.toBe('1.2.3'); expect(fetchMock).toHaveBeenCalledTimes(4); }); - it('does not let a beta rejection clear the stable cache entry', async () => { + it('does not let a beta rejection clear the latest cache entry', async () => { const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { if (init?.method === 'HEAD') return headResponse(true); if (url === DIST_TAGS_URL) { @@ -257,15 +213,15 @@ describe('resolveCliVersion', () => { throw new Error(`unexpected fetch: ${url}`); }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); - await expect(resolveCliVersion('beta')).rejects.toThrow( - /did not have a valid "beta" version for the beta channel/ + await expect(resolveCliDistTag('latest')).resolves.toBe('1.2.3'); + await expect(resolveCliDistTag('beta')).rejects.toThrow( + /did not have a valid "beta" version/ ); - await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + await expect(resolveCliDistTag('latest')).resolves.toBe('1.2.3'); - // Two dist-tags GETs (stable, beta) + two HEAD (stable's amd64+arm64 + // Two dist-tags GETs (latest, beta) + two HEAD (latest's amd64+arm64 // asset check only — beta never gets that far). expect(fetchMock).toHaveBeenCalledTimes(4); }); @@ -284,9 +240,9 @@ describe('resolveCliVersion', () => { }, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.51'); + await expect(resolveCliDistTag('beta')).resolves.toBe('2.118.0-beta.51'); const packumentCall = fetchMock.mock.calls.find( ([url]) => url === PACKUMENT_URL @@ -320,9 +276,9 @@ describe('resolveCliVersion', () => { }, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.60'); + await expect(resolveCliDistTag('beta')).resolves.toBe('2.118.0-beta.60'); }); it('orders beta.10 ahead of beta.9 during the walk-back (not a string compare)', async () => { @@ -340,9 +296,9 @@ describe('resolveCliVersion', () => { }, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.10'); + await expect(resolveCliDistTag('beta')).resolves.toBe('2.118.0-beta.10'); const headUrls = fetchMock.mock.calls .filter(([, init]) => init?.method === 'HEAD') @@ -373,9 +329,9 @@ describe('resolveCliVersion', () => { }, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.49'); + await expect(resolveCliDistTag('beta')).resolves.toBe('2.118.0-beta.49'); }); it('aborts the walk-back and propagates when a candidate probe throws, rather than continuing to the next candidate', async () => { @@ -403,9 +359,9 @@ describe('resolveCliVersion', () => { throw new Error(`unexpected fetch: ${url}`); }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('beta')).rejects.toThrow( + await expect(resolveCliDistTag('beta')).rejects.toThrow( /2\.118\.0-beta\.51.*-> 500 Internal Server Error/ ); @@ -434,9 +390,9 @@ describe('resolveCliVersion', () => { }, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.50'); + await expect(resolveCliDistTag('beta')).resolves.toBe('2.118.0-beta.50'); const headUrls = fetchMock.mock.calls .filter(([, init]) => init?.method === 'HEAD') @@ -460,9 +416,9 @@ describe('resolveCliVersion', () => { packument: { versions }, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('beta')).rejects.toThrow(); + await expect(resolveCliDistTag('beta')).rejects.toThrow(); const probedVersions = [ ...new Set( @@ -496,28 +452,28 @@ describe('resolveCliVersion', () => { }, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('beta')).rejects.toThrow( + await expect(resolveCliDistTag('beta')).rejects.toThrow( /2\.118\.0-beta\.52.*2\.118\.0-beta\.51/ ); }); - it('throws instead of guessing when the stable dist-tag asset is a 404', async () => { + it('throws instead of guessing when the latest dist-tag asset is a 404', async () => { const fetchMock = routedFetchMock({ distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, assetOk: () => false, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('stable')).rejects.toThrow( + await expect(resolveCliDistTag('latest')).rejects.toThrow( 'checked amd64 and arm64 .deb assets at ' + 'https://github.com/supabase/cli/releases/tag/v1.2.3' ); }); - it('throws instead of walking back when the next dist-tag asset is a 404', async () => { + it('throws instead of walking back when a non-beta-shaped dist-tag asset is a 404', async () => { const fetchMock = routedFetchMock({ distTags: { latest: '1.2.3', next: '3.0.0-next.2' }, assetOk: () => false, @@ -526,9 +482,9 @@ describe('resolveCliVersion', () => { }, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('next')).rejects.toThrow( + await expect(resolveCliDistTag('next')).rejects.toThrow( `npm's "next" dist-tag for "supabase" points at 3.0.0-next.2, but its ` + 'release asset is missing (checked amd64 and arm64 .deb assets at ' + 'https://github.com/supabase/cli/releases/tag/v3.0.0-next.2)' @@ -555,9 +511,9 @@ describe('resolveCliVersion', () => { throw new Error(`unexpected fetch: ${url}`); }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('beta')).rejects.toThrow( + await expect(resolveCliDistTag('beta')).rejects.toThrow( `${cliDebUrl('2.118.0-beta.52', 'amd64')} -> 500 Internal Server Error` ); @@ -566,17 +522,111 @@ describe('resolveCliVersion', () => { ); }); - it('does not hit the network at all for an env override', async () => { - process.env[STABLE_ENV] = '9.9.9'; - process.env[BETA_ENV] = '9.9.9-beta.1'; + it('resolves an arbitrary dist-tag name', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', canary: '1.4.0-canary.7' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); + + await expect(resolveCliDistTag('canary')).resolves.toBe('1.4.0-canary.7'); + }); + + it('throws without fetching when the dist-tag name is invalid', async () => { const fetchMock = vi.fn(); vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersion } = await import('../src/cli-channel.js'); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('stable')).resolves.toBe('9.9.9'); - await expect(resolveCliVersion('beta')).resolves.toBe('9.9.9-beta.1'); + await expect(resolveCliDistTag('1bad tag')).rejects.toThrow( + /neither an exact Supabase CLI version nor a valid npm dist-tag name/ + ); expect(fetchMock).not.toHaveBeenCalled(); }); + + it('walks back a beta-shaped version even when the dist-tag is not named beta', async () => { + const fetchMock = routedFetchMock({ + distTags: { canary: '2.118.0-beta.52' }, + assetOk: (version) => version === '2.118.0-beta.51', + packument: { + versions: { '2.118.0-beta.52': {}, '2.118.0-beta.51': {} }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); + + await expect(resolveCliDistTag('canary')).resolves.toBe('2.118.0-beta.51'); + }); + + it('throws instead of walking back a missing asset that is not beta-shaped, even on the beta dist-tag', async () => { + const fetchMock = routedFetchMock({ + distTags: { beta: '1.4.0-rc.2' }, + assetOk: () => false, + packument: { versions: { '1.4.0-rc.2': {} } }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); + + await expect(resolveCliDistTag('beta')).rejects.toThrow( + `npm's "beta" dist-tag for "supabase" points at 1.4.0-rc.2, but its release asset is missing` + ); + expect(fetchMock.mock.calls.some(([url]) => url === PACKUMENT_URL)).toBe( + false + ); + }); + + describe(PINS_ENV, () => { + it('uses a pinned version without touching the network and strips a leading v', async () => { + process.env[PINS_ENV] = JSON.stringify({ latest: 'v9.9.9' }); + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); + + await expect(resolveCliDistTag('latest')).resolves.toBe('9.9.9'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('only pins the tags it names and resolves the rest from npm', async () => { + process.env[PINS_ENV] = JSON.stringify({ latest: '9.9.9' }); + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); + + await expect(resolveCliDistTag('latest')).resolves.toBe('9.9.9'); + await expect(resolveCliDistTag('beta')).resolves.toBe('1.3.0-beta.1'); + }); + + it('treats a blank or whitespace-only value as unset', async () => { + process.env[PINS_ENV] = ' '; + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); + + await expect(resolveCliDistTag('latest')).resolves.toBe('1.2.3'); + }); + + it.each([ + ['malformed JSON', '{not json', /is not valid JSON/], + ['a non-object', '["9.9.9"]', /must be a JSON object/], + [ + 'an invalid version', + JSON.stringify({ latest: 'not-a-version' }), + /pins "latest" to "not-a-version", which is not a valid Supabase CLI version/, + ], + ])('throws for %s', async (_name, value, message) => { + process.env[PINS_ENV] = value; + vi.stubGlobal('fetch', vi.fn()); + const { resolveCliDistTag } = await import('../src/cli-channel.js'); + + await expect(resolveCliDistTag('latest')).rejects.toThrow(message); + }); + }); }); describe('resolveCliVersionOption', () => { @@ -598,7 +648,16 @@ describe('resolveCliVersionOption', () => { expect(fetchMock).not.toHaveBeenCalled(); }); - it('resolves a "stable" channel tag against npm', async () => { + it('strips a leading v from an exact version', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionOption } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionOption('v2.109.1')).resolves.toBe('2.109.1'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('resolves a "latest" dist-tag against npm', async () => { const fetchMock = routedFetchMock({ distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, assetOk: () => true, @@ -606,10 +665,23 @@ describe('resolveCliVersionOption', () => { vi.stubGlobal('fetch', fetchMock); const { resolveCliVersionOption } = await import('../src/cli-channel.js'); - await expect(resolveCliVersionOption('stable')).resolves.toBe('1.2.3'); + await expect(resolveCliVersionOption('latest')).resolves.toBe('1.2.3'); }); - it('resolves a "beta" channel tag against npm', async () => { + it('resolves an arbitrary dist-tag against npm', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', canary: '1.4.0-canary.7' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionOption } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionOption('canary')).resolves.toBe( + '1.4.0-canary.7' + ); + }); + + it('throws listing the available dist-tags for an unknown tag', async () => { const fetchMock = routedFetchMock({ distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, assetOk: () => true, @@ -617,10 +689,30 @@ describe('resolveCliVersionOption', () => { vi.stubGlobal('fetch', fetchMock); const { resolveCliVersionOption } = await import('../src/cli-channel.js'); - await expect(resolveCliVersionOption('beta')).resolves.toBe('1.3.0-beta.1'); + await expect(resolveCliVersionOption('canary')).rejects.toThrow( + 'npm has no "canary" dist-tag for "supabase"; available dist-tags: latest, beta' + ); }); }); +describe('isExactCliVersion', () => { + it.each(['2.109.1', 'v2.109.1', '2.118.0-beta.5', '1.4.0-rc.2'])( + 'accepts %s', + async (value) => { + const { isExactCliVersion } = await import('../src/cli-channel.js'); + expect(isExactCliVersion(value)).toBe(true); + } + ); + + it.each(['latest', 'beta', 'next', 'canary', 'v1', '1.2'])( + 'rejects %s', + async (value) => { + const { isExactCliVersion } = await import('../src/cli-channel.js'); + expect(isExactCliVersion(value)).toBe(false); + } + ); +}); + describe('compareBetaVersionsDesc', () => { it('sorts newer beta numbers before older ones within the same minor', async () => { const { compareBetaVersionsDesc } = await import('../src/cli-channel.js'); diff --git a/packages/sandbox/test/local-stack-docker.test.ts b/packages/sandbox/test/local-stack-docker.test.ts index 727f34b0..992002c3 100644 --- a/packages/sandbox/test/local-stack-docker.test.ts +++ b/packages/sandbox/test/local-stack-docker.test.ts @@ -165,18 +165,21 @@ describe('buildLocalStackScoringContext environmentMarker', () => { }); } - it('includes an optional channel when present', async () => { - const marker = { - runtime: 'local-stack', - channel: 'beta', - cliVersion: '2.109.1', - docker: 'available', - sessionStartedMs: 1_700_000_000_000, - }; - const readRootFile = vi.fn().mockResolvedValue(JSON.stringify(marker)); - const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); - await expect(ctx.environmentMarker()).resolves.toEqual(marker); - }); + it.each(['beta', 'canary'])( + 'includes an optional channel %s when present', + async (channel) => { + const marker = { + runtime: 'local-stack', + channel, + cliVersion: '2.109.1', + docker: 'available', + sessionStartedMs: 1_700_000_000_000, + }; + const readRootFile = vi.fn().mockResolvedValue(JSON.stringify(marker)); + const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); + await expect(ctx.environmentMarker()).resolves.toEqual(marker); + } + ); it('reads the marker as root, not through exec or resolveSandboxPath', async () => { const readRootFile = vi.fn().mockResolvedValue( From 3dca4333f0d406b4ab60653af8a9718473663449 Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Fri, 9 Oct 2026 09:42:04 +0100 Subject: [PATCH 2/2] feat(evals): accept any npm version spec for cliVersion Resolve exact versions, dist-tags and semver ranges the way npm view supabase@ does, drop the beta walk-back, fail only the pairs whose spec cannot be resolved, and record the resolved version as cli_version in Braintrust metadata. Refs AI-1299 --- CONTRIBUTING.md | 2 +- README.md | 2 +- .../scripts/run-vercel-evals.test.ts | 240 ++++-- apps/framework/scripts/run-vercel-evals.ts | 118 +-- apps/framework/scripts/upload-braintrust.ts | 1 + packages/core/src/index.ts | 6 +- packages/sandbox/package.json | 4 +- packages/sandbox/src/cli-channel.ts | 229 +++--- packages/sandbox/src/index.ts | 2 +- packages/sandbox/src/local-stack-runtime.ts | 25 +- packages/sandbox/test/cli-channel.test.ts | 728 +++++++----------- .../sandbox/test/local-stack-docker.test.ts | 15 +- pnpm-lock.yaml | 18 + 13 files changed, 680 insertions(+), 710 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6b525d5c..050988b5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -94,6 +94,6 @@ Which evals each arm picks up: Common workflows: - **Add or change a CLI eval.** Add the scenario under `evals/cli//` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview. -- **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli` and `experiment_suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. To reproduce a specific run, set an exact `cliVersion` in the experiment's `localStackRuntime` instead of a dist-tag. +- **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli` and `experiment_suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. `cliVersion` in an experiment's `localStackRuntime` takes anything `npm view supabase@` resolves (an exact version, a dist-tag like `latest`/`beta`/`next`, or a range like `^2.120.0`), resolved once per run; a spec that fails to resolve fails only the pairs that use it. To reproduce a specific run, set an exact version. - **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories. - **Run the unit tests.** `pnpm --filter @supabase-evals/framework test:cli-lib` (the CLI skip predicates in `experiments/cli/lib/` plus every CLI eval's scorer tests) — also part of `pnpm test`. diff --git a/README.md b/README.md index 7b2b52f6..cb15aee0 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ An eval's optional `local/` directory is copied into the sandbox workspace befor Set `cliVersion: 2.109.1` in an eval's frontmatter when it requires a specific Supabase CLI release. This overrides an experiment's `localStackRuntime({ cliVersion })` setting; otherwise the runtime setting or repository-wide default applies. -An experiment's `localStackRuntime({ cliVersion })` takes an exact version or any npm dist-tag for the `supabase` package (e.g. `'latest'`, `'beta'`, `'next'`), resolved once per run. An eval's own `cliVersion:` pin still wins over either form. +An experiment's `localStackRuntime({ cliVersion })` takes anything `npm view supabase@` resolves: an exact version, a dist-tag such as `'latest'`, `'beta'` or `'next'`, or a semver range like `'^2.120.0'`. It is resolved once per run, and a spec that fails to resolve fails only the pairs that use it. An eval's own `cliVersion:` pin still wins over any of these. An experiment can pass `localStackRuntime({ docker: 'no-daemon' })` or `'absent'` to stage a sandbox where the Docker daemon is unreachable or the `docker` binary is missing entirely, instead of the default `'available'`. `needsDocker` defaults to `true`; set it `false` in an eval's frontmatter when the scenario can run, and is meaningful, without a Docker daemon (e.g. starting the stack is the agent's own job) — that's what lets a Docker-less experiment pick the eval up. diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index ecc25b8c..6e75c642 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -1,5 +1,5 @@ import { APIError } from '@vercel/sandbox'; -import { resolveCliDistTag } from '@supabase-evals/sandbox'; +import { resolveCliVersionSpec } from '@supabase-evals/sandbox'; import { execFileSync } from 'node:child_process'; import { copyFileSync, @@ -21,21 +21,24 @@ import { cleanupSandbox, downloadResults, finalizeResult, - distTagPinsEnv, + cliVersionPinsEnv, FORWARDED_ENV_NAMES, isRetryableSandboxCreateError, isTerminalSandboxCreateError, packWorkspaceScript, parsePairs, - requiredCliDistTags, - resolveDistTagPins, + requiredCliVersionSpecs, + resolveCliVersionPins, + runPairs, runBounded, tagValue, expandJobs, type EvalPair, } from './run-vercel-evals.js'; -vi.mock('@supabase-evals/sandbox', () => ({ resolveCliDistTag: vi.fn() })); +vi.mock('@supabase-evals/sandbox', () => ({ + resolveCliVersionSpec: vi.fn(), +})); const BROKERED_NAMES = BROKERED_KEYS.map(({ name }) => name); @@ -72,27 +75,27 @@ describe('agentEnvironment', () => { } }); - it('omits the CLI dist-tag pins when unset', () => { + it('omits the CLI version pins when unset', () => { process.env.ANTHROPIC_API_KEY = 'anthropic-value'; const env = agentEnvironment(); expect(env).toBe(`ANTHROPIC_API_KEY=${BROKERED_KEY_PLACEHOLDER}`); - expect(env).not.toContain('SUPABASE_CLI_DIST_TAG_PINS'); + expect(env).not.toContain('SUPABASE_CLI_VERSION_PINS'); }); it('prefers an explicit pin over the same-named process.env value', () => { - process.env.SUPABASE_CLI_DIST_TAG_PINS = '{"latest":"1.0.0"}'; + process.env.SUPABASE_CLI_VERSION_PINS = '{"latest":"1.0.0"}'; const env = agentEnvironment({ - SUPABASE_CLI_DIST_TAG_PINS: '{"latest":"2.117.0"}', + SUPABASE_CLI_VERSION_PINS: '{"latest":"2.117.0"}', }); - expect(env).toContain('SUPABASE_CLI_DIST_TAG_PINS={"latest":"2.117.0"}'); + expect(env).toContain('SUPABASE_CLI_VERSION_PINS={"latest":"2.117.0"}'); expect(env).not.toContain('1.0.0'); }); it('shares one pin value across multiple .env writes, simulating a two-job fan-out', () => { - const pins = distTagPinsEnv({ + const pins = cliVersionPinsEnv({ latest: '2.117.0', beta: '2.118.0-beta.5', next: '3.0.0-next.2', @@ -103,20 +106,20 @@ describe('agentEnvironment', () => { expect(jobOneEnv).toBe(jobTwoEnv); expect(jobOneEnv).toBe( - 'SUPABASE_CLI_DIST_TAG_PINS={"latest":"2.117.0","beta":"2.118.0-beta.5","next":"3.0.0-next.2"}' + 'SUPABASE_CLI_VERSION_PINS={"latest":"2.117.0","beta":"2.118.0-beta.5","next":"3.0.0-next.2"}' ); }); }); -describe('distTagPinsEnv', () => { +describe('cliVersionPinsEnv', () => { it('forwards a single JSON env var for the resolved pins', () => { - expect(distTagPinsEnv({ canary: '1.4.0-canary.7' })).toEqual({ - SUPABASE_CLI_DIST_TAG_PINS: '{"canary":"1.4.0-canary.7"}', + expect(cliVersionPinsEnv({ canary: '1.4.0-canary.7' })).toEqual({ + SUPABASE_CLI_VERSION_PINS: '{"canary":"1.4.0-canary.7"}', }); }); - it('forwards nothing when no dist-tag was resolved', () => { - expect(distTagPinsEnv({})).toEqual({}); + it('forwards nothing when no spec was resolved', () => { + expect(cliVersionPinsEnv({})).toEqual({}); }); }); @@ -177,61 +180,69 @@ describe('brokeredNetworkPolicy', () => { }); }); -describe('resolveDistTagPins', () => { +describe('resolveCliVersionPins', () => { beforeEach(() => { - vi.mocked(resolveCliDistTag).mockReset(); + vi.mocked(resolveCliVersionSpec).mockReset(); }); - it('resolves only the requested dist-tags into one pin map every job can share', async () => { - vi.mocked(resolveCliDistTag).mockImplementation(async (tag) => - tag === 'latest' ? '2.117.0' : '2.118.0-beta.5' + it('resolves only the requested specs into one pin map every job can share', async () => { + vi.mocked(resolveCliVersionSpec).mockImplementation(async (spec) => + spec === 'latest' ? '2.117.0' : '2.118.0-beta.5' ); - const pins = await resolveDistTagPins(new Set(['latest', 'beta'])); + const { pins, errors } = await resolveCliVersionPins( + new Set(['latest', 'beta']) + ); expect(pins).toEqual({ latest: '2.117.0', beta: '2.118.0-beta.5' }); - expect(resolveCliDistTag).toHaveBeenCalledTimes(2); - expect(resolveCliDistTag).toHaveBeenCalledWith('latest'); - expect(resolveCliDistTag).toHaveBeenCalledWith('beta'); + expect(errors.size).toBe(0); + expect(resolveCliVersionSpec).toHaveBeenCalledTimes(2); + expect(resolveCliVersionSpec).toHaveBeenCalledWith('latest'); + expect(resolveCliVersionSpec).toHaveBeenCalledWith('beta'); // Two fanned-out jobs writing their own .env from the same pins must get // the identical value the resolver was called once for. - const forwarded = distTagPinsEnv(pins); + const forwarded = cliVersionPinsEnv(pins); expect(agentEnvironment(forwarded)).toBe(agentEnvironment(forwarded)); expect(agentEnvironment(forwarded)).toContain( - 'SUPABASE_CLI_DIST_TAG_PINS={"latest":"2.117.0","beta":"2.118.0-beta.5"}' + 'SUPABASE_CLI_VERSION_PINS={"latest":"2.117.0","beta":"2.118.0-beta.5"}' ); }); - it('resolves an arbitrary dist-tag without calling the resolver for others', async () => { - vi.mocked(resolveCliDistTag).mockResolvedValue('1.4.0-canary.7'); + it('resolves a range spec', async () => { + vi.mocked(resolveCliVersionSpec).mockResolvedValue('2.121.0'); - const pins = await resolveDistTagPins(new Set(['canary'])); + const { pins } = await resolveCliVersionPins(new Set(['^2.120.0'])); - expect(pins).toEqual({ canary: '1.4.0-canary.7' }); - expect(resolveCliDistTag).toHaveBeenCalledTimes(1); - expect(resolveCliDistTag).toHaveBeenCalledWith('canary'); + expect(pins).toEqual({ '^2.120.0': '2.121.0' }); + expect(resolveCliVersionSpec).toHaveBeenCalledWith('^2.120.0'); }); - it('does no network work for an empty dist-tag set', async () => { - const pins = await resolveDistTagPins(new Set()); + it('does no network work for an empty spec set', async () => { + const { pins, errors } = await resolveCliVersionPins(new Set()); expect(pins).toEqual({}); - expect(resolveCliDistTag).not.toHaveBeenCalled(); + expect(errors.size).toBe(0); + expect(resolveCliVersionSpec).not.toHaveBeenCalled(); }); - it('propagates a resolution failure rather than swallowing it', async () => { - vi.mocked(resolveCliDistTag).mockRejectedValue( - new Error('npm unreachable') - ); + it('reports a failing spec without dropping the ones that resolved', async () => { + const failure = new Error('npm unreachable'); + vi.mocked(resolveCliVersionSpec).mockImplementation(async (spec) => { + if (spec === 'beta') throw failure; + return '2.117.0'; + }); - await expect(resolveDistTagPins(new Set(['latest']))).rejects.toThrow( - 'npm unreachable' + const { pins, errors } = await resolveCliVersionPins( + new Set(['latest', 'beta']) ); + + expect(pins).toEqual({ latest: '2.117.0' }); + expect([...errors]).toEqual([['beta', failure]]); }); }); -describe('requiredCliDistTags', () => { +describe('requiredCliVersionSpecs', () => { const pair = (overrides: Partial = {}): EvalPair => ({ eval_id: 'eval-1', experiment: 'experiment-1', @@ -240,23 +251,24 @@ describe('requiredCliDistTags', () => { ...overrides, }); - it('resolves only the dist-tag a latest-tagged experiment declares', async () => { + it('resolves only the spec a latest-tagged experiment declares', async () => { const loadExperimentConfig = vi.fn(async () => ({ - localStack: { cliDistTag: 'latest' }, + localStack: { cliVersionSpec: 'latest' }, })); + const target = pair(); - const distTags = await requiredCliDistTags([pair()], { + const specs = await requiredCliVersionSpecs([target], { loadEvalMetadata: () => ({ cliVersion: undefined }), loadExperimentConfig, }); - expect(distTags).toEqual(new Set(['latest'])); + expect([...specs]).toEqual([[target, 'latest']]); }); - it('resolves nothing when no experiment in the pair set declares a dist-tag', async () => { + it('resolves nothing when no experiment in the pair set declares a spec', async () => { const loadExperimentConfig = vi.fn(async () => ({})); - const distTags = await requiredCliDistTags( + const specs = await requiredCliVersionSpecs( [pair(), pair({ eval_id: 'eval-2', experiment: 'experiment-2' })], { loadEvalMetadata: () => ({ cliVersion: undefined }), @@ -264,29 +276,29 @@ describe('requiredCliDistTags', () => { } ); - expect(distTags.size).toBe(0); + expect(specs.size).toBe(0); }); - it("an eval's pinned cliVersion contributes no dist-tag, even when its experiment declares one", async () => { + it("an eval's pinned cliVersion contributes no spec, even when its experiment declares one", async () => { const loadExperimentConfig = vi.fn(async () => ({ - localStack: { cliDistTag: 'beta' }, + localStack: { cliVersionSpec: 'beta' }, })); - const distTags = await requiredCliDistTags([pair()], { + const specs = await requiredCliVersionSpecs([pair()], { loadEvalMetadata: () => ({ cliVersion: '2.109.1' }), loadExperimentConfig, }); - expect(distTags.size).toBe(0); + expect(specs.size).toBe(0); expect(loadExperimentConfig).not.toHaveBeenCalled(); }); - it('unions dist-tags across pairs without resolving an experiment config twice', async () => { + it('maps pairs sharing an experiment to one spec without loading its config twice', async () => { const loadExperimentConfig = vi.fn(async () => ({ - localStack: { cliDistTag: 'beta' }, + localStack: { cliVersionSpec: '^2.120.0' }, })); - const distTags = await requiredCliDistTags( + const specs = await requiredCliVersionSpecs( [pair(), pair({ eval_id: 'eval-2' })], { loadEvalMetadata: () => ({ cliVersion: undefined }), @@ -294,13 +306,14 @@ describe('requiredCliDistTags', () => { } ); - expect(distTags).toEqual(new Set(['beta'])); + expect([...new Set(specs.values())]).toEqual(['^2.120.0']); + expect(specs.size).toBe(2); expect(loadExperimentConfig).toHaveBeenCalledTimes(1); }); it('throws naming an experiment that cannot be resolved to a config', async () => { await expect( - requiredCliDistTags([pair({ experiment: 'ghost' })], { + requiredCliVersionSpecs([pair({ experiment: 'ghost' })], { loadEvalMetadata: () => ({ cliVersion: undefined }), loadExperimentConfig: async () => { throw new Error('no experiment config found for "ghost"'); @@ -310,6 +323,113 @@ describe('requiredCliDistTags', () => { }); }); +describe('runPairs', () => { + const pair = (experiment: string, evalId = 'eval-1'): EvalPair => ({ + eval_id: evalId, + experiment, + experiment_suite: 'cli', + eval_suite: 'cli', + }); + const options = (pairs: EvalPair[]) => ({ + pairs, + revision: 'main', + repoUrl: 'https://example.com/repo.git', + outputDir: '/tmp/unused', + runs: 2, + timeoutSec: 60, + concurrency: 4, + vcpus: 2, + }); + const specsByExperiment: Record = { + 'on-latest': 'latest', + 'on-broken': 'broken-tag', + }; + + beforeEach(() => { + vi.mocked(resolveCliVersionSpec).mockReset(); + vi.stubEnv('VERCEL_TOKEN', 'token'); + vi.stubEnv('VERCEL_TEAM_ID', 'team'); + vi.stubEnv('VERCEL_PROJECT_ID', 'project'); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + function deps( + cliVersions: Record = {} + ): Parameters[1] & { + runPairOnce: ReturnType; + } { + return { + runPairOnce: vi.fn(async () => {}), + loadEvalMetadata: (target) => ({ + cliVersion: cliVersions[target.eval_id], + }), + loadExperimentConfig: async (experiment) => ({ + localStack: { cliVersionSpec: specsByExperiment[experiment] }, + }), + }; + } + + it('fails only the jobs whose spec could not resolve, and forwards only resolved pins', async () => { + vi.mocked(resolveCliVersionSpec).mockImplementation(async (spec) => { + if (spec === 'broken-tag') throw new Error('asset is missing'); + return '2.117.0'; + }); + const runDeps = deps(); + + const outcome = runPairs( + options([pair('on-latest'), pair('on-broken')]), + runDeps + ); + + await expect(outcome).rejects.toThrow('2 Sandbox eval run(s) failed'); + await outcome.catch((error: AggregateError) => { + expect(error.errors.map((cause) => cause.message)).toEqual([ + '[on-broken x eval-1 run 1]: could not resolve Supabase CLI version "broken-tag": asset is missing', + '[on-broken x eval-1 run 2]: could not resolve Supabase CLI version "broken-tag": asset is missing', + ]); + }); + + const started = runDeps.runPairOnce.mock.calls.map( + ([jobOptions]) => `${jobOptions.pair.experiment}#${jobOptions.run}` + ); + expect(started.sort()).toEqual(['on-latest#1', 'on-latest#2']); + for (const [jobOptions] of runDeps.runPairOnce.mock.calls) { + expect(jobOptions.pins).toEqual({ + SUPABASE_CLI_VERSION_PINS: '{"latest":"2.117.0"}', + }); + } + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining('SANDBOX FAILED [on-broken x eval-1 run 1]') + ); + }); + + it("runs a job whose eval pins cliVersion even when its experiment's spec failed", async () => { + vi.mocked(resolveCliVersionSpec).mockRejectedValue(new Error('npm down')); + const runDeps = deps({ 'pinned-eval': '2.109.1' }); + + await runPairs(options([pair('on-broken', 'pinned-eval')]), runDeps); + + expect(runDeps.runPairOnce).toHaveBeenCalledTimes(2); + expect(resolveCliVersionSpec).not.toHaveBeenCalled(); + }); + + it('runs every job when all specs resolve', async () => { + vi.mocked(resolveCliVersionSpec).mockResolvedValue('2.117.0'); + const runDeps = deps(); + + await runPairs(options([pair('on-latest'), pair('on-broken')]), runDeps); + + expect(runDeps.runPairOnce).toHaveBeenCalledTimes(4); + expect(resolveCliVersionSpec).toHaveBeenCalledTimes(2); + }); +}); + describe('Vercel eval controller', () => { it('bounds concurrent work and lets independent failures settle', async () => { let active = 0; diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index 540fa3db..7d8741f8 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -13,7 +13,7 @@ import { sandboxUsageSchema, type SandboxUsage, } from '@supabase-evals/core/eval-metadata'; -import { resolveCliDistTag } from '@supabase-evals/sandbox'; +import { resolveCliVersionSpec } from '@supabase-evals/sandbox'; import { execFileSync } from 'node:child_process'; import { readFileSync, renameSync, writeFileSync } from 'node:fs'; import { join, relative, resolve } from 'node:path'; @@ -61,10 +61,10 @@ export const BROKERED_KEYS: { ]; export const BROKERED_KEY_PLACEHOLDER = 'injected-by-sandbox-firewall'; /** - * Pins the CLI dist-tag versions resolved for this run across sandbox jobs. + * Pins the CLI versions resolved for this run across sandbox jobs. * Provider keys belong in `BROKERED_KEYS` so the sandbox never sees them. */ -export const FORWARDED_ENV_NAMES = ['SUPABASE_CLI_DIST_TAG_PINS']; +export const FORWARDED_ENV_NAMES = ['SUPABASE_CLI_VERSION_PINS']; /** * Slack for the non-agent work inside `pnpm eval` (supabase start, resets, * scoring, export). Cold image pulls alone can take ~10 min. @@ -115,52 +115,66 @@ interface PairOptions extends RunnerOptions { pins: Record; } +export interface CliVersionResolution { + /** Spec to exact version, for the specs that resolved. */ + pins: Record; + /** Resolution failure per spec that did not. */ + errors: Map; +} + /** - * Resolves each requested CLI dist-tag's pin once, so every sandbox job in a + * Resolves each requested CLI version spec once, so every sandbox job in a * fan-out runs against the same concrete version rather than each resolving - * independently and risking a mid-run release landing between them. A tag - * already pinned in SUPABASE_CLI_DIST_TAG_PINS is used verbatim by - * resolveCliDistTag. An empty set does no network work. + * independently and risking a mid-run release landing between them. A spec + * already pinned in SUPABASE_CLI_VERSION_PINS is used verbatim by + * resolveCliVersionSpec. A spec that fails to resolve is reported without + * affecting the others. An empty set does no network work. */ -export async function resolveDistTagPins( - distTags: ReadonlySet -): Promise> { - const resolved = await Promise.all( - [...distTags].map( - async (distTag) => [distTag, await resolveCliDistTag(distTag)] as const - ) +export async function resolveCliVersionPins( + specs: ReadonlySet +): Promise { + const requested = [...specs]; + const settled = await Promise.allSettled( + requested.map((spec) => resolveCliVersionSpec(spec)) ); - return Object.fromEntries(resolved); + const pins: [string, string][] = []; + const errors = new Map(); + settled.forEach((result, index) => { + const spec = requested[index]; + if (result.status === 'fulfilled') pins.push([spec, result.value]); + else errors.set(spec, result.reason); + }); + return { pins: Object.fromEntries(pins), errors }; } -/** The `.env` pin forwarded to every sandbox job, or none when no dist-tag was resolved. */ -export function distTagPinsEnv( +/** The `.env` pin forwarded to every sandbox job, or none when no spec was resolved. */ +export function cliVersionPinsEnv( pins: Readonly> ): Record { return Object.keys(pins).length > 0 - ? { SUPABASE_CLI_DIST_TAG_PINS: JSON.stringify(pins) } + ? { SUPABASE_CLI_VERSION_PINS: JSON.stringify(pins) } : {}; } -export interface RequiredCliDistTagsDeps { +export interface RequiredCliVersionSpecsDeps { loadEvalMetadata: (pair: EvalPair) => Pick; loadExperimentConfig: ( experiment: string - ) => Promise<{ localStack?: { cliDistTag?: string } }>; + ) => Promise<{ localStack?: { cliVersionSpec?: string } }>; } /** - * Maps a pair set to the CLI dist-tags at least one pair needs, so - * resolveDistTagPins only resolves those. An eval's own `cliVersion` - * frontmatter is an exact pin that wins over its experiment's dist-tag and - * needs no resolution; a pair whose experiment has no `localStack.cliDistTag` + * Maps each pair that needs a CLI version resolved to its spec, so + * resolveCliVersionPins only resolves those. An eval's own `cliVersion` + * frontmatter is an exact pin that wins over its experiment's spec and needs + * no resolution; a pair whose experiment has no `localStack.cliVersionSpec` * needs none either. */ -export async function requiredCliDistTags( +export async function requiredCliVersionSpecs( pairs: readonly EvalPair[], - { loadEvalMetadata, loadExperimentConfig }: RequiredCliDistTagsDeps -): Promise> { - const distTags = new Set(); + { loadEvalMetadata, loadExperimentConfig }: RequiredCliVersionSpecsDeps +): Promise> { + const specs = new Map(); const configs = new Map>(); for (const pair of pairs) { @@ -171,11 +185,11 @@ export async function requiredCliDistTags( config = loadExperimentConfig(pair.experiment); configs.set(pair.experiment, config); } - const distTag = (await config).localStack?.cliDistTag; - if (distTag) distTags.add(distTag); + const spec = (await config).localStack?.cliVersionSpec; + if (spec) specs.set(pair, spec); } - return distTags; + return specs; } /** Mirrors run-eval.ts's evals///PROMPT.md convention. */ @@ -203,8 +217,8 @@ function experimentPathsByName(): Promise> { } /** - * Throws rather than treating a config it can't find as needing no dist-tag, - * which would silently resolve every dist-tag per-sandbox again. + * Throws rather than treating a config it can't find as needing no spec, + * which would silently resolve every spec per-sandbox again. */ async function loadExperimentConfig( experiment: string @@ -246,8 +260,18 @@ export async function runBounded( return Promise.allSettled(items.map((item) => limit(run, item))); } +export interface RunPairsDeps extends RequiredCliVersionSpecsDeps { + runPairOnce: ( + options: PairOptions, + credentials: VercelCredentials + ) => Promise; +} + /** Runs all pairs and reports failures only after independent work finishes. */ -async function runPairs(options: RunnerOptions): Promise { +export async function runPairs( + options: RunnerOptions, + deps: RunPairsDeps = { loadEvalMetadata, loadExperimentConfig, runPairOnce } +): Promise { const credentials = vercelCredentialsFromEnv(); const jobs = expandJobs(options.pairs, options.runs); console.log( @@ -255,29 +279,35 @@ async function runPairs(options: RunnerOptions): Promise { `max ${options.concurrency} at a time` ); - // Resolved once, for only the dist-tags this run's pairs actually need, so + // Resolved once, for only the specs this run's pairs actually need, so // every job below writes the same pin rather than each sandbox resolving - // its own dist-tag version independently. - const distTags = await requiredCliDistTags(options.pairs, { - loadEvalMetadata, - loadExperimentConfig, - }); - const pins = distTagPinsEnv(await resolveDistTagPins(distTags)); + // its own version independently. + const specsByPair = await requiredCliVersionSpecs(options.pairs, deps); + const { pins, errors: specErrors } = await resolveCliVersionPins( + new Set(specsByPair.values()) + ); + const forwardedPins = cliVersionPinsEnv(pins); const results = await runBounded( jobs, options.concurrency, async ({ pair, run }) => { try { + const spec = specsByPair.get(pair); + if (spec !== undefined && specErrors.has(spec)) { + throw new Error( + `could not resolve Supabase CLI version "${spec}": ${errorMessage(specErrors.get(spec))}` + ); + } await pRetry( (attempt) => - runPairOnce( + deps.runPairOnce( { ...options, pair, run, attempt, - pins, + pins: forwardedPins, }, credentials ), @@ -821,7 +851,7 @@ export function brokeredNetworkPolicy(): NetworkPolicy { return { allow }; } -/** Builds the sandbox `.env` from key placeholders and CLI dist-tag pins. An explicit pin wins over process.env. */ +/** Builds the sandbox `.env` from key placeholders and CLI version pins. An explicit pin wins over process.env. */ export function agentEnvironment(pins: Record = {}): string { const lines: string[] = []; for (const { name } of BROKERED_KEYS) { diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts index be1e2d17..f8dc132c 100644 --- a/apps/framework/scripts/upload-braintrust.ts +++ b/apps/framework/scripts/upload-braintrust.ts @@ -338,6 +338,7 @@ async function collectRows( eval_suite: suite, stage: promptData?.stage ?? result.stage, interface: promptData?.interface ?? result.interface, + cli_version: result.cliVersion ?? promptData?.cliVersion, skills: result.skills, docs: result.docs, source_path: sourcePath, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 08082d8a..57773906 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -333,7 +333,7 @@ export interface LocalStackStatus { */ export interface LocalStackEnvironmentMarker { runtime: 'local-stack'; - /** The npm dist-tag the session's CLI version resolved from. */ + /** The version spec (dist-tag or range) the CLI version resolved from; named `channel` for results compatibility. */ channel?: string; cliVersion: string; docker: 'available' | 'no-daemon' | 'absent'; @@ -635,8 +635,8 @@ export type LocalStackSession = { export type LocalStackRuntime = { id: string; startSession(args: LocalStackSessionArgs): Promise; - /** npm dist-tag (e.g. `beta`) this runtime resolves its CLI version against, unset when it's pinned to an exact version. */ - cliDistTag?: string; + /** Version spec (dist-tag such as `beta`, or a semver range) this runtime resolves its CLI version against, unset when it's pinned to an exact version. */ + cliVersionSpec?: string; }; export type ExperimentConfig = { diff --git a/packages/sandbox/package.json b/packages/sandbox/package.json index 9da33f75..1c043514 100644 --- a/packages/sandbox/package.json +++ b/packages/sandbox/package.json @@ -17,10 +17,12 @@ "dependencies": { "@supabase-evals/core": "workspace:*", "@supabase/supabase-js": "catalog:", - "ai": "catalog:" + "ai": "catalog:", + "semver": "^7.8.5" }, "devDependencies": { "@types/node": "catalog:", + "@types/semver": "^7.8.0", "typescript": "catalog:", "vitest": "catalog:" } diff --git a/packages/sandbox/src/cli-channel.ts b/packages/sandbox/src/cli-channel.ts index 3ec0dc25..b86a8e85 100644 --- a/packages/sandbox/src/cli-channel.ts +++ b/packages/sandbox/src/cli-channel.ts @@ -1,10 +1,11 @@ /** - * Resolves a Supabase CLI version from an exact version or any npm dist-tag of - * the `supabase` package. npm can point a tag at a still-draft release with no - * downloadable asset, so the resolved version's `.deb` is verified first. + * Resolves a Supabase CLI version spec: an exact version, an npm dist-tag of + * `supabase`, or a semver range. npm can point at a release whose GitHub draft + * has no downloadable asset, so a resolved version's `.deb` is verified first. */ import { isRecord } from '@supabase-evals/core/json'; +import { maxSatisfying, satisfies, validRange } from 'semver'; const NPM_DIST_TAGS_URL = 'https://registry.npmjs.org/-/package/supabase/dist-tags'; @@ -18,21 +19,13 @@ const VERSION_RE = /^\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$/; const DIST_TAG_RE = /^[A-Za-z][A-Za-z0-9._-]*$/; -// Only the -beta.N shape has a defined ordering the walk-back below can search. -const BETA_SUFFIX_RE = /^\d+\.\d+\.\d+-beta\.\d+$/; - -// Feeds compareBetaVersionsDesc's numeric comparison; matches any version, unlike BETA_SUFFIX_RE. -const BETA_VERSION_RE = /^(\d+)\.(\d+)\.(\d+)-beta\.(\d+)$/; - -// Caps the walk-back so a broken release can't chain into unbounded GitHub requests. -const MAX_BETA_FALLBACK_CANDIDATES = 5; - -/** JSON object mapping dist-tag names to exact versions, pinning a run's resolution. */ -const DIST_TAG_PINS_ENV = 'SUPABASE_CLI_DIST_TAG_PINS'; +/** JSON object mapping version specs to exact versions, pinning a run's resolution. */ +const VERSION_PINS_ENV = 'SUPABASE_CLI_VERSION_PINS'; const versionCache = new Map>(); +const registryDocCache = new Map>(); -/** Whether `value` has the shape of an exact CLI version (optionally `v`-prefixed) rather than a dist-tag name. */ +/** Whether `value` has the shape of an exact CLI version (optionally `v`-prefixed) rather than a dist-tag or range. */ export function isExactCliVersion(value: string): boolean { return VERSION_RE.test(stripVersionPrefix(value)); } @@ -72,199 +65,159 @@ async function debAssetExists(version: string): Promise { return amd64 && arm64; } -/** - * Resolves an npm dist-tag of `supabase` to a concrete CLI version, memoised - * per tag for the process lifetime. A rejection clears the cache entry so the - * next call retries; never falls back to the pinned SUPABASE_CLI_VERSION, - * since that would silently mislabel data. - */ -export async function resolveCliDistTag(tag: string): Promise { - const cached = versionCache.get(tag); +/** Shares one in-flight load per key; a rejection clears the entry so the next call retries. */ +function memoise( + cache: Map>, + key: string, + load: () => Promise +): Promise { + const cached = cache.get(key); if (cached) return cached; - const promise = resolveCliDistTagUncached(tag); - versionCache.set(tag, promise); + const promise = load(); + cache.set(key, promise); promise.catch(() => { - if (versionCache.get(tag) === promise) versionCache.delete(tag); + if (cache.get(key) === promise) cache.delete(key); }); return promise; } -/** Resolves `value`: an exact version passes through (minus any leading `v`), `undefined` stays `undefined`, anything else is an npm dist-tag. */ +function fetchRegistryDoc( + url: string, + headers?: Record +): Promise { + return memoise(registryDocCache, url, async () => { + const response = await fetch(url, { + headers, + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + throw new Error( + `GET ${url} -> ${response.status} ${response.statusText}` + ); + } + return response.json(); + }); +} + +/** + * Resolves an exact version, npm dist-tag or semver range of `supabase` to a + * concrete CLI version, memoised per spec for the process lifetime. A + * rejection clears the cache entry so the next call retries; never falls back + * to the pinned SUPABASE_CLI_VERSION, since that would silently mislabel data. + */ +export function resolveCliVersionSpec(spec: string): Promise { + if (isExactCliVersion(spec)) return Promise.resolve(stripVersionPrefix(spec)); + return memoise(versionCache, spec, () => resolveCliVersionSpecUncached(spec)); +} + +/** Resolves `value`: `undefined` stays `undefined`, anything else goes through {@link resolveCliVersionSpec}. */ export async function resolveCliVersionOption( value: string | undefined ): Promise { - if (value === undefined) return undefined; - return isExactCliVersion(value) - ? stripVersionPrefix(value) - : resolveCliDistTag(value); + return value === undefined ? undefined : resolveCliVersionSpec(value); } -function readDistTagPins(): Record { - const raw = process.env[DIST_TAG_PINS_ENV]?.trim(); +function readVersionPins(): Record { + const raw = process.env[VERSION_PINS_ENV]?.trim(); if (!raw) return {}; let parsed: unknown; try { parsed = JSON.parse(raw); } catch { - throw new Error(`${DIST_TAG_PINS_ENV} is not valid JSON: ${raw}`); + throw new Error(`${VERSION_PINS_ENV} is not valid JSON: ${raw}`); } if (!isRecord(parsed)) { throw new Error( - `${DIST_TAG_PINS_ENV} must be a JSON object mapping dist-tags to versions` + `${VERSION_PINS_ENV} must be a JSON object mapping version specs to versions` ); } - const pins: Record = {}; - for (const [tag, value] of Object.entries(parsed)) { + const pins: Record = Object.create(null); + for (const [spec, value] of Object.entries(parsed)) { const version = typeof value === 'string' ? stripVersionPrefix(value.trim()) : ''; if (!VERSION_RE.test(version)) { throw new Error( - `${DIST_TAG_PINS_ENV} pins "${tag}" to ${JSON.stringify(value)}, which is not a valid Supabase CLI version` + `${VERSION_PINS_ENV} pins "${spec}" to ${JSON.stringify(value)}, which is not a valid Supabase CLI version` ); } - pins[tag] = version; + pins[spec] = version; } return pins; } -async function resolveCliDistTagUncached(distTag: string): Promise { - if (!DIST_TAG_RE.test(distTag)) { +async function resolveCliVersionSpecUncached(spec: string): Promise { + // semver reads '' as '*', which would silently resolve to the newest release. + const isRange = spec.trim() !== '' && validRange(spec) !== null; + if (!isRange && !DIST_TAG_RE.test(spec)) { throw new Error( - `${JSON.stringify(distTag)} is neither an exact Supabase CLI version nor a valid npm dist-tag name` + `${JSON.stringify(spec)} is not an exact Supabase CLI version, a semver range, or a valid npm dist-tag name` ); } // An explicit pin is trusted as-is — no asset check, network or otherwise. - const pinned = readDistTagPins()[distTag]; - if (pinned) return pinned; + const pins = readVersionPins(); + if (Object.hasOwn(pins, spec)) return pins[spec]; - const response = await fetch(NPM_DIST_TAGS_URL, { - signal: AbortSignal.timeout(15_000), - }); - if (!response.ok) { + const version = isRange + ? await resolveRange(spec) + : await resolveDistTag(spec); + + if (!(await debAssetExists(version))) { throw new Error( - `failed to resolve the "${distTag}" Supabase CLI dist-tag: ` + - `GET ${NPM_DIST_TAGS_URL} -> ${response.status} ${response.statusText}` + `"${spec}" resolves to supabase@${version}, but its release asset is missing ` + + `(checked amd64 and arm64 .deb assets at ${releaseTagUrl(version)})` ); } - const tags = await response.json(); + return version; +} + +async function resolveDistTag(distTag: string): Promise { + const tags = await fetchRegistryDoc(NPM_DIST_TAGS_URL); if (!isRecord(tags)) { throw new Error( `npm dist-tags for "supabase" were not a JSON object: ${JSON.stringify(tags)}` ); } - const version = tags[distTag]; - if (version === undefined) { + if (!Object.hasOwn(tags, distTag)) { throw new Error( `npm has no "${distTag}" dist-tag for "supabase"; available dist-tags: ` + Object.keys(tags).join(', ') ); } + const version = tags[distTag]; if (typeof version !== 'string' || !VERSION_RE.test(version)) { throw new Error( `npm dist-tags for "supabase" did not have a valid "${distTag}" version: ` + JSON.stringify(version) ); } - - // A transient failure here throws rather than silently walking back to an - // older beta (or never walking back, for other shapes). - if (await debAssetExists(version)) return version; - - // npm's dist-tag can point at a version whose GitHub release is still a - // draft with no downloadable .deb. Only -beta.N versions walk back to an - // older published version; other shapes are never guessed at. - if (!BETA_SUFFIX_RE.test(version)) { - throw new Error( - `npm's "${distTag}" dist-tag for "supabase" points at ${version}, but its ` + - `release asset is missing (checked amd64 and arm64 .deb assets at ` + - `${releaseTagUrl(version)})` - ); - } - - return resolveFallbackBetaVersion(version, distTag); + return version; } -/** - * Walks back through published `X.Y.Z-beta.N` versions strictly older than - * `unpublishedVersion`, newest first, for one with a downloadable `.deb`. - * Newer candidates are skipped too (likelier to be drafts). Only a 404 - * advances to the next candidate; any other probe error aborts the - * walk-back, since it means GitHub is unhealthy rather than that the - * candidate is absent. - */ -async function resolveFallbackBetaVersion( - unpublishedVersion: string, - distTag: string -): Promise { - const response = await fetch(NPM_PACKUMENT_URL, { - headers: { Accept: NPM_INSTALL_V1_ACCEPT }, - signal: AbortSignal.timeout(15_000), +/** Follows npm-pick-manifest: the `latest` dist-tag when it satisfies the range, otherwise the highest satisfying non-prerelease version. */ +async function resolveRange(range: string): Promise { + const packument = await fetchRegistryDoc(NPM_PACKUMENT_URL, { + Accept: NPM_INSTALL_V1_ACCEPT, }); - if (!response.ok) { - throw new Error( - `failed to look up published beta versions for the "${distTag}" dist-tag of "supabase": ` + - `GET ${NPM_PACKUMENT_URL} -> ${response.status} ${response.statusText}` - ); - } - const packument = await response.json(); const versions = isRecord(packument) ? packument.versions : undefined; if (!isRecord(versions)) { throw new Error( `npm packument for "supabase" did not include a "versions" object` ); } + const tags = isRecord(packument) ? packument['dist-tags'] : undefined; + const latest = + isRecord(tags) && Object.hasOwn(tags, 'latest') ? tags.latest : undefined; + if (typeof latest === 'string' && satisfies(latest, range)) return latest; - const candidates = Object.keys(versions) - .filter((candidate) => BETA_VERSION_RE.test(candidate)) - .filter( - (candidate) => compareBetaVersionsDesc(candidate, unpublishedVersion) > 0 - ) - .sort(compareBetaVersionsDesc) - .slice(0, MAX_BETA_FALLBACK_CANDIDATES); - - for (const candidate of candidates) { - if (await debAssetExists(candidate)) return candidate; - } - - throw new Error( - `no published beta Supabase CLI release has a downloadable .deb asset for the "${distTag}" dist-tag; ` + - `checked ${[unpublishedVersion, ...candidates].join(', ')} via ` + - `${NPM_PACKUMENT_URL}` - ); -} - -function parseBetaVersion( - version: string -): [major: number, minor: number, patch: number, beta: number] | undefined { - const match = BETA_VERSION_RE.exec(version); - if (!match) return undefined; - return [ - Number(match[1]), - Number(match[2]), - Number(match[3]), - Number(match[4]), - ]; -} - -/** - * Orders two `X.Y.Z-beta.N` versions newest-first, usable as an - * `Array#sort` comparator; compares components numerically so `beta.10` - * sorts ahead of `beta.9`. Throws if either string isn't parseable. - */ -export function compareBetaVersionsDesc(a: string, b: string): number { - const tupleA = parseBetaVersion(a); - const tupleB = parseBetaVersion(b); - if (!tupleA || !tupleB) { + const version = maxSatisfying(Object.keys(versions), range); + if (version === null) { throw new Error( - `compareBetaVersionsDesc expected "X.Y.Z-beta.N" versions, got ${JSON.stringify(a)} and ${JSON.stringify(b)}` + `no published "supabase" version on npm satisfies "${range}"` ); } - for (let index = 0; index < tupleA.length; index += 1) { - if (tupleA[index] !== tupleB[index]) return tupleB[index] - tupleA[index]; - } - return 0; + return version; } diff --git a/packages/sandbox/src/index.ts b/packages/sandbox/src/index.ts index d20c442b..e8ba86cd 100644 --- a/packages/sandbox/src/index.ts +++ b/packages/sandbox/src/index.ts @@ -1,4 +1,4 @@ -export { resolveCliDistTag } from './cli-channel.js'; +export { resolveCliVersionSpec } from './cli-channel.js'; export { DockerSandbox, dockerCli } from './docker-sandbox.js'; export type { DockerSandboxOptions, diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index e1f4efb7..3f22c29e 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -49,9 +49,10 @@ const STACK_CONFIG_RETRY_MS = 2_000; export interface LocalStackRuntimeOptions { /** * Supabase CLI version baked into the sandbox image: an exact version - * (e.g. `2.109.1`) or any npm dist-tag of `supabase` (e.g. `'latest'`, - * `'beta'`, `'next'`), resolved once per process. An eval's own - * `cliVersion:` frontmatter pin always wins over this option. + * (e.g. `2.109.1`), an npm dist-tag of `supabase` (e.g. `'latest'`, + * `'beta'`, `'next'`) or a semver range (e.g. `'^2.120.0'`), resolved once + * per process. An eval's own `cliVersion:` frontmatter pin always wins over + * this option. */ cliVersion?: string; /** @@ -124,12 +125,13 @@ const DEFAULT_MCP_FEATURES = ['docs']; export function localStackRuntime( options: LocalStackRuntimeOptions = {} ): LocalStackRuntime { + const spec = + options.cliVersion !== undefined && !isExactCliVersion(options.cliVersion) + ? options.cliVersion + : undefined; return { id: buildRuntimeId(options), - cliDistTag: - options.cliVersion !== undefined && !isExactCliVersion(options.cliVersion) - ? options.cliVersion - : undefined, + cliVersionSpec: spec, async startSession({ agent, cliVersion, @@ -144,13 +146,8 @@ export function localStackRuntime( // Stamped before setup so it's comparable with the scorer's PID-1 fallback. const sessionStartedMs = Date.now(); const docker = options.docker ?? 'available'; - // Only an unpinned eval inherits a dist-tag; an eval's own pin always wins. - const channel = - cliVersion === undefined && - options.cliVersion !== undefined && - !isExactCliVersion(options.cliVersion) - ? options.cliVersion - : undefined; + // Only an unpinned eval inherits the spec; an eval's own pin always wins. + const channel = cliVersion === undefined ? spec : undefined; const version = cliVersion ?? (await resolveCliVersionOption(options.cliVersion)) ?? diff --git a/packages/sandbox/test/cli-channel.test.ts b/packages/sandbox/test/cli-channel.test.ts index 2fc68231..f2e5331a 100644 --- a/packages/sandbox/test/cli-channel.test.ts +++ b/packages/sandbox/test/cli-channel.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { cliDebUrl } from '../src/cli-channel.js'; -const PINS_ENV = 'SUPABASE_CLI_DIST_TAG_PINS'; +const PINS_ENV = 'SUPABASE_CLI_VERSION_PINS'; const DIST_TAGS_URL = 'https://registry.npmjs.org/-/package/supabase/dist-tags'; const PACKUMENT_URL = 'https://registry.npmjs.org/supabase'; const INSTALL_V1_ACCEPT = 'application/vnd.npm.install-v1+json'; @@ -29,7 +29,7 @@ function headResponse(ok: boolean, status?: number, statusText?: string) { /** * Routes a single stubbed `fetch` by method + URL, mirroring the real * mixture of GET (dist-tags, packument) and HEAD (amd64 + arm64 asset check) - * calls resolveCliDistTag makes. + * calls resolveCliVersionSpec makes. */ function routedFetchMock(routes: { distTags?: unknown; @@ -58,6 +58,22 @@ function routedFetchMock(routes: { }); } +function packument(latest: string, versions: string[]) { + return { + 'dist-tags': { latest }, + versions: Object.fromEntries(versions.map((version) => [version, {}])), + }; +} + +function urlsOf( + fetchMock: ReturnType, + method: 'GET' | 'HEAD' +): string[] { + return fetchMock.mock.calls + .filter(([, init]) => (init?.method ?? 'GET') === method) + .map(([url]) => url); +} + beforeEach(() => { vi.resetModules(); delete process.env[PINS_ENV]; @@ -68,46 +84,56 @@ afterEach(() => { delete process.env[PINS_ENV]; }); -describe('resolveCliDistTag', () => { +describe('resolveCliVersionSpec', () => { it('resolves the "latest" dist-tag when its asset exists', async () => { const fetchMock = routedFetchMock({ distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, assetOk: () => true, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('latest')).resolves.toBe('1.2.3'); + await expect(resolveCliVersionSpec('latest')).resolves.toBe('1.2.3'); - const headCalls = fetchMock.mock.calls.filter( - ([, init]) => init?.method === 'HEAD' - ); - expect(headCalls.map(([url]) => url)).toEqual([ + expect(urlsOf(fetchMock, 'HEAD')).toEqual([ cliDebUrl('1.2.3', 'amd64'), cliDebUrl('1.2.3', 'arm64'), ]); }); - it('resolves the "beta" dist-tag when its asset exists', async () => { + it('resolves the "beta" and "next" dist-tags', async () => { const fetchMock = routedFetchMock({ - distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1', next: '3.0.0-next.2' }, assetOk: () => true, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('beta')).resolves.toBe('1.3.0-beta.1'); + await expect(resolveCliVersionSpec('beta')).resolves.toBe('1.3.0-beta.1'); + await expect(resolveCliVersionSpec('next')).resolves.toBe('3.0.0-next.2'); }); - it('resolves the "next" dist-tag when its asset exists', async () => { + it('resolves an arbitrary dist-tag name', async () => { const fetchMock = routedFetchMock({ - distTags: { latest: '1.2.3', beta: '1.3.0-beta.1', next: '3.0.0-next.2' }, + distTags: { latest: '1.2.3', canary: '1.4.0-canary.7' }, assetOk: () => true, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionSpec('canary')).resolves.toBe( + '1.4.0-canary.7' + ); + }); + + it('passes an exact version through without touching the network, stripping a leading v', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('next')).resolves.toBe('3.0.0-next.2'); + await expect(resolveCliVersionSpec('2.109.1')).resolves.toBe('2.109.1'); + await expect(resolveCliVersionSpec('v2.109.1')).resolves.toBe('2.109.1'); + expect(fetchMock).not.toHaveBeenCalled(); }); it('throws with the HTTP status when the registry request fails', async () => { @@ -120,9 +146,9 @@ describe('resolveCliDistTag', () => { }, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('latest')).rejects.toThrow( + await expect(resolveCliVersionSpec('latest')).rejects.toThrow( `${DIST_TAGS_URL} -> 500 Internal Server Error` ); }); @@ -130,21 +156,37 @@ describe('resolveCliDistTag', () => { it('throws listing the available dist-tags when the requested one is missing', async () => { const fetchMock = routedFetchMock({ distTags: { beta: '1.0.0-beta.1' } }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('latest')).rejects.toThrow( + await expect(resolveCliVersionSpec('latest')).rejects.toThrow( 'npm has no "latest" dist-tag for "supabase"; available dist-tags: beta' ); }); + it('does not resolve a spec named after a prototype member', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionSpec('constructor')).rejects.toThrow( + 'npm has no "constructor" dist-tag for "supabase"; available dist-tags: latest' + ); + await expect(resolveCliVersionSpec('toString')).rejects.toThrow( + 'npm has no "toString" dist-tag' + ); + }); + it('throws when the dist-tag value is not a valid version string', async () => { const fetchMock = routedFetchMock({ distTags: { latest: 'not-a-version' }, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('latest')).rejects.toThrow( + await expect(resolveCliVersionSpec('latest')).rejects.toThrow( /did not have a valid "latest" version/ ); }); @@ -152,26 +194,50 @@ describe('resolveCliDistTag', () => { it('treats an array dist-tags response as invalid rather than a record', async () => { const fetchMock = routedFetchMock({ distTags: ['not', 'a', 'record'] }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('latest')).rejects.toThrow( + await expect(resolveCliVersionSpec('latest')).rejects.toThrow( /were not a JSON object/ ); }); + it('throws without fetching when the spec is neither a version, range nor dist-tag name', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionSpec('1bad tag')).rejects.toThrow( + /not an exact Supabase CLI version, a semver range, or a valid npm dist-tag name/ + ); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('rejects an empty spec instead of reading it as the "*" range', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionSpec('')).rejects.toThrow( + /not an exact Supabase CLI version, a semver range, or a valid npm dist-tag name/ + ); + await expect(resolveCliVersionSpec(' ')).rejects.toThrow( + /not an exact Supabase CLI version/ + ); + expect(fetchMock).not.toHaveBeenCalled(); + }); + it('memoises a successful resolution so a second call does not refetch', async () => { const fetchMock = routedFetchMock({ distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, assetOk: () => true, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('latest')).resolves.toBe('1.2.3'); - await expect(resolveCliDistTag('latest')).resolves.toBe('1.2.3'); + await expect(resolveCliVersionSpec('latest')).resolves.toBe('1.2.3'); + await expect(resolveCliVersionSpec('latest')).resolves.toBe('1.2.3'); - // One GET (dist-tags) + two HEAD (amd64+arm64 asset check) — the second - // call hits the cache. + // One GET (dist-tags) + two HEAD (amd64+arm64 asset check). expect(fetchMock).toHaveBeenCalledTimes(3); }); @@ -193,386 +259,211 @@ describe('resolveCliDistTag', () => { throw new Error(`unexpected fetch: ${url}`); }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('latest')).rejects.toThrow('500'); - await expect(resolveCliDistTag('latest')).resolves.toBe('1.2.3'); - // A third call must still hit the cache populated by the successful - // retry — the rejected first promise's cleanup must not evict it. - await expect(resolveCliDistTag('latest')).resolves.toBe('1.2.3'); + await expect(resolveCliVersionSpec('latest')).rejects.toThrow('500'); + await expect(resolveCliVersionSpec('latest')).resolves.toBe('1.2.3'); + await expect(resolveCliVersionSpec('latest')).resolves.toBe('1.2.3'); expect(fetchMock).toHaveBeenCalledTimes(4); }); it('does not let a beta rejection clear the latest cache entry', async () => { - const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { - if (init?.method === 'HEAD') return headResponse(true); - if (url === DIST_TAGS_URL) { - return jsonResponse({ latest: '1.2.3', beta: 'not-a-version' }); - } - throw new Error(`unexpected fetch: ${url}`); + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: 'not-a-version' }, + assetOk: () => true, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('latest')).resolves.toBe('1.2.3'); - await expect(resolveCliDistTag('beta')).rejects.toThrow( + await expect(resolveCliVersionSpec('latest')).resolves.toBe('1.2.3'); + await expect(resolveCliVersionSpec('beta')).rejects.toThrow( /did not have a valid "beta" version/ ); - await expect(resolveCliDistTag('latest')).resolves.toBe('1.2.3'); - - // Two dist-tags GETs (latest, beta) + two HEAD (latest's amd64+arm64 - // asset check only — beta never gets that far). - expect(fetchMock).toHaveBeenCalledTimes(4); + await expect(resolveCliVersionSpec('latest')).resolves.toBe('1.2.3'); }); - it('falls back to the newest published beta.N-1 when the dist-tag asset is a 404', async () => { + it('fetches the dist-tags once across multiple specs', async () => { const fetchMock = routedFetchMock({ - distTags: { latest: '1.2.3', beta: '2.118.0-beta.52' }, - assetOk: (version) => version !== '2.118.0-beta.52', - packument: { - versions: { - '2.118.0-beta.52': {}, - '2.118.0-beta.51': {}, - '2.118.0-beta.50': {}, - '2.117.0': {}, - }, - }, + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1', next: '3.0.0-next.2' }, + assetOk: () => true, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); - - await expect(resolveCliDistTag('beta')).resolves.toBe('2.118.0-beta.51'); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - const packumentCall = fetchMock.mock.calls.find( - ([url]) => url === PACKUMENT_URL + await Promise.all( + ['latest', 'beta', 'next'].map((spec) => resolveCliVersionSpec(spec)) ); - expect(packumentCall?.[1]?.headers).toMatchObject({ - Accept: INSTALL_V1_ACCEPT, - }); - - const headUrls = fetchMock.mock.calls - .filter(([, init]) => init?.method === 'HEAD') - .map(([url]) => url); - expect(headUrls).toEqual([ - cliDebUrl('2.118.0-beta.52', 'amd64'), - cliDebUrl('2.118.0-beta.52', 'arm64'), - cliDebUrl('2.118.0-beta.51', 'amd64'), - cliDebUrl('2.118.0-beta.51', 'arm64'), - ]); - }); - - it("falls back across a minor version boundary to the previous minor's newest beta", async () => { - const fetchMock = routedFetchMock({ - distTags: { latest: '2.118.0', beta: '2.119.0-beta.1' }, - assetOk: (version) => version === '2.118.0-beta.60', - packument: { - versions: { - '2.119.0-beta.1': {}, - '2.118.0-beta.60': {}, - '2.118.0-beta.59': {}, - '2.117.0': {}, - }, - }, - }); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); - - await expect(resolveCliDistTag('beta')).resolves.toBe('2.118.0-beta.60'); - }); - - it('orders beta.10 ahead of beta.9 during the walk-back (not a string compare)', async () => { - const fetchMock = routedFetchMock({ - distTags: { latest: '1.2.3', beta: '2.118.0-beta.11' }, - // Only the unpublished dist-tag version (11) 404s; both walk-back - // candidates would succeed, so probe order is what decides the result. - assetOk: (version) => version !== '2.118.0-beta.11', - packument: { - versions: { - '2.118.0-beta.11': {}, - '2.118.0-beta.10': {}, - '2.118.0-beta.9': {}, - }, - }, - }); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); - - await expect(resolveCliDistTag('beta')).resolves.toBe('2.118.0-beta.10'); - - const headUrls = fetchMock.mock.calls - .filter(([, init]) => init?.method === 'HEAD') - .map(([url]) => url); - // A naive string compare would sort "beta.9" ahead of "beta.10" (since - // '9' > '1' character-wise), probing 9 before 10. - expect(headUrls).toEqual([ - cliDebUrl('2.118.0-beta.11', 'amd64'), - cliDebUrl('2.118.0-beta.11', 'arm64'), - cliDebUrl('2.118.0-beta.10', 'amd64'), - cliDebUrl('2.118.0-beta.10', 'arm64'), - ]); - }); - it('never selects a beta candidate newer than the unpublished dist-tag version', async () => { - const fetchMock = routedFetchMock({ - distTags: { latest: '1.2.3', beta: '2.118.0-beta.50' }, - // Every candidate downloads except the unpublished one — if the newer - // candidate (51) were ever probed, this would resolve to beta.51 - // instead of walking further back to beta.49. - assetOk: (version) => version !== '2.118.0-beta.50', - packument: { - versions: { - '2.118.0-beta.51': {}, - '2.118.0-beta.50': {}, - '2.118.0-beta.49': {}, - }, - }, - }); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); - - await expect(resolveCliDistTag('beta')).resolves.toBe('2.118.0-beta.49'); + expect(urlsOf(fetchMock, 'GET')).toEqual([DIST_TAGS_URL]); }); - it('aborts the walk-back and propagates when a candidate probe throws, rather than continuing to the next candidate', async () => { - const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { - if (init?.method === 'HEAD') { - if (url.includes('2.118.0-beta.52')) return headResponse(false); - if (url.includes('2.118.0-beta.51')) { - return headResponse(false, 500, 'Internal Server Error'); - } - if (url.includes('2.118.0-beta.50')) return headResponse(true); - return headResponse(false); - } - if (url === DIST_TAGS_URL) { - return jsonResponse({ latest: '1.2.3', beta: '2.118.0-beta.52' }); - } - if (url === PACKUMENT_URL) { - return jsonResponse({ - versions: { - '2.118.0-beta.52': {}, - '2.118.0-beta.51': {}, - '2.118.0-beta.50': {}, - }, + describe('missing release asset', () => { + it.each([ + ['latest', { latest: '1.2.3' }, '1.2.3'], + ['next', { next: '3.0.0-next.2' }, '3.0.0-next.2'], + ['beta', { beta: '2.118.0-beta.52' }, '2.118.0-beta.52'], + ])( + 'throws for the %s dist-tag without walking back to another version', + async (spec, distTags, version) => { + const fetchMock = routedFetchMock({ + distTags, + assetOk: () => false, + packument: packument(version, [version, '2.118.0-beta.51']), }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionSpec(spec)).rejects.toThrow( + `"${spec}" resolves to supabase@${version}, but its release asset is missing ` + + `(checked amd64 and arm64 .deb assets at https://github.com/supabase/cli/releases/tag/v${version})` + ); + + expect(urlsOf(fetchMock, 'GET')).toEqual([DIST_TAGS_URL]); + expect(urlsOf(fetchMock, 'HEAD')).toEqual([ + cliDebUrl(version, 'amd64'), + cliDebUrl(version, 'arm64'), + ]); } - throw new Error(`unexpected fetch: ${url}`); - }); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); - - await expect(resolveCliDistTag('beta')).rejects.toThrow( - /2\.118\.0-beta\.51.*-> 500 Internal Server Error/ ); - const headUrls = fetchMock.mock.calls - .filter(([, init]) => init?.method === 'HEAD') - .map(([url]) => url); - // beta.50 is never probed — the throw on beta.51 aborts the walk-back. - expect(headUrls).toEqual([ - cliDebUrl('2.118.0-beta.52', 'amd64'), - cliDebUrl('2.118.0-beta.52', 'arm64'), - cliDebUrl('2.118.0-beta.51', 'amd64'), - cliDebUrl('2.118.0-beta.51', 'arm64'), - ]); - }); + it('throws naming the URL and status when the asset HEAD is a GitHub error', async () => { + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (init?.method === 'HEAD') { + return url.includes('_amd64') + ? headResponse(false, 500, 'Internal Server Error') + : headResponse(true); + } + if (url === DIST_TAGS_URL) { + return jsonResponse({ beta: '2.118.0-beta.52' }); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - it('advances to the second walk-back candidate when the first is a 404', async () => { - const fetchMock = routedFetchMock({ - distTags: { latest: '1.2.3', beta: '2.118.0-beta.52' }, - assetOk: (version) => version === '2.118.0-beta.50', - packument: { - versions: { - '2.118.0-beta.52': {}, - '2.118.0-beta.51': {}, - '2.118.0-beta.50': {}, - }, - }, + await expect(resolveCliVersionSpec('beta')).rejects.toThrow( + `${cliDebUrl('2.118.0-beta.52', 'amd64')} -> 500 Internal Server Error` + ); }); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); - - await expect(resolveCliDistTag('beta')).resolves.toBe('2.118.0-beta.50'); - - const headUrls = fetchMock.mock.calls - .filter(([, init]) => init?.method === 'HEAD') - .map(([url]) => url); - expect(headUrls).toEqual([ - cliDebUrl('2.118.0-beta.52', 'amd64'), - cliDebUrl('2.118.0-beta.52', 'arm64'), - cliDebUrl('2.118.0-beta.51', 'amd64'), - cliDebUrl('2.118.0-beta.51', 'arm64'), - cliDebUrl('2.118.0-beta.50', 'amd64'), - cliDebUrl('2.118.0-beta.50', 'arm64'), - ]); - }); - it('caps the beta walk-back at MAX_BETA_FALLBACK_CANDIDATES, probing newest-first', async () => { - const versions: Record = {}; - for (let n = 50; n <= 59; n += 1) versions[`2.118.0-beta.${n}`] = {}; - const fetchMock = routedFetchMock({ - distTags: { latest: '1.2.3', beta: '2.118.0-beta.60' }, - assetOk: () => false, - packument: { versions }, - }); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); - - await expect(resolveCliDistTag('beta')).rejects.toThrow(); - - const probedVersions = [ - ...new Set( - fetchMock.mock.calls - .filter(([, init]) => init?.method === 'HEAD') - .map(([url]) => url.match(/supabase_(.+)_linux_/)?.[1]) - ), - ]; - // The unpublished dist-tag version itself is checked first (outside the - // walk-back budget), followed by exactly MAX_BETA_FALLBACK_CANDIDATES (5) - // older published betas, newest-first. - expect(probedVersions).toEqual([ - '2.118.0-beta.60', - '2.118.0-beta.59', - '2.118.0-beta.58', - '2.118.0-beta.57', - '2.118.0-beta.56', - '2.118.0-beta.55', - ]); - }); + it('throws for a range that resolves to a version without an asset', async () => { + const fetchMock = routedFetchMock({ + packument: packument('2.121.0', ['2.121.0']), + assetOk: () => false, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - it('throws naming the unpublished versions when no published beta has a downloadable asset', async () => { - const fetchMock = routedFetchMock({ - distTags: { latest: '1.2.3', beta: '2.118.0-beta.52' }, - assetOk: () => false, - packument: { - versions: { - '2.118.0-beta.52': {}, - '2.118.0-beta.51': {}, - }, - }, + await expect(resolveCliVersionSpec('^2.120.0')).rejects.toThrow( + '"^2.120.0" resolves to supabase@2.121.0, but its release asset is missing' + ); }); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); - - await expect(resolveCliDistTag('beta')).rejects.toThrow( - /2\.118\.0-beta\.52.*2\.118\.0-beta\.51/ - ); }); - it('throws instead of guessing when the latest dist-tag asset is a 404', async () => { - const fetchMock = routedFetchMock({ - distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, - assetOk: () => false, - }); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); + describe('semver ranges', () => { + it('picks the latest dist-tag when it satisfies the range', async () => { + const fetchMock = routedFetchMock({ + packument: packument('2.121.0', [ + '2.119.0', + '2.120.0', + '2.121.0', + '2.121.1', + ]), + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('latest')).rejects.toThrow( - 'checked amd64 and arm64 .deb assets at ' + - 'https://github.com/supabase/cli/releases/tag/v1.2.3' - ); - }); + await expect(resolveCliVersionSpec('^2.120.0')).resolves.toBe('2.121.0'); - it('throws instead of walking back when a non-beta-shaped dist-tag asset is a 404', async () => { - const fetchMock = routedFetchMock({ - distTags: { latest: '1.2.3', next: '3.0.0-next.2' }, - assetOk: () => false, - packument: { - versions: { '3.0.0-next.2': {}, '3.0.0-next.1': {} }, - }, + const packumentCall = fetchMock.mock.calls.find( + ([url]) => url === PACKUMENT_URL + ); + expect(packumentCall?.[1]?.headers).toMatchObject({ + Accept: INSTALL_V1_ACCEPT, + }); + expect(urlsOf(fetchMock, 'HEAD')).toEqual([ + cliDebUrl('2.121.0', 'amd64'), + cliDebUrl('2.121.0', 'arm64'), + ]); }); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('next')).rejects.toThrow( - `npm's "next" dist-tag for "supabase" points at 3.0.0-next.2, but its ` + - 'release asset is missing (checked amd64 and arm64 .deb assets at ' + - 'https://github.com/supabase/cli/releases/tag/v3.0.0-next.2)' - ); - - expect(fetchMock.mock.calls.some(([url]) => url === PACKUMENT_URL)).toBe( - false - ); - }); + it('falls back to the highest satisfying version when latest is outside the range', async () => { + const fetchMock = routedFetchMock({ + packument: packument('3.0.0', [ + '2.119.0', + '2.120.0', + '2.121.0', + '2.120.5', + '3.0.0', + ]), + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - it('throws naming the URL and status when the dist-tag asset HEAD is a GitHub error, without fetching the packument', async () => { - const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { - if (init?.method === 'HEAD') { - return url.includes('_amd64') - ? headResponse(false, 500, 'Internal Server Error') - : headResponse(true); - } - if (url === DIST_TAGS_URL) { - return jsonResponse({ latest: '1.2.3', beta: '2.118.0-beta.52' }); - } - if (url === PACKUMENT_URL) { - throw new Error('packument must not be fetched'); - } - throw new Error(`unexpected fetch: ${url}`); + await expect(resolveCliVersionSpec('^2.120.0')).resolves.toBe('2.121.0'); }); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('beta')).rejects.toThrow( - `${cliDebUrl('2.118.0-beta.52', 'amd64')} -> 500 Internal Server Error` - ); - - expect(fetchMock.mock.calls.some(([url]) => url === PACKUMENT_URL)).toBe( - false - ); - }); + it('excludes prereleases from a plain range, including a prerelease latest', async () => { + const fetchMock = routedFetchMock({ + packument: packument('2.122.0-rc.1', [ + '2.120.0', + '2.121.0', + '2.122.0-beta.1', + '2.122.0-rc.1', + ]), + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - it('resolves an arbitrary dist-tag name', async () => { - const fetchMock = routedFetchMock({ - distTags: { latest: '1.2.3', canary: '1.4.0-canary.7' }, - assetOk: () => true, + await expect(resolveCliVersionSpec('^2.120.0')).resolves.toBe('2.121.0'); }); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('canary')).resolves.toBe('1.4.0-canary.7'); - }); + it('throws naming the spec when nothing satisfies the range', async () => { + const fetchMock = routedFetchMock({ + packument: packument('2.121.0', ['2.120.0', '2.121.0']), + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - it('throws without fetching when the dist-tag name is invalid', async () => { - const fetchMock = vi.fn(); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); + await expect(resolveCliVersionSpec('^9.0.0')).rejects.toThrow( + 'no published "supabase" version on npm satisfies "^9.0.0"' + ); + expect(urlsOf(fetchMock, 'HEAD')).toEqual([]); + }); - await expect(resolveCliDistTag('1bad tag')).rejects.toThrow( - /neither an exact Supabase CLI version nor a valid npm dist-tag name/ - ); - expect(fetchMock).not.toHaveBeenCalled(); - }); + it('throws when the packument has no versions object', async () => { + const fetchMock = routedFetchMock({ packument: {} }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - it('walks back a beta-shaped version even when the dist-tag is not named beta', async () => { - const fetchMock = routedFetchMock({ - distTags: { canary: '2.118.0-beta.52' }, - assetOk: (version) => version === '2.118.0-beta.51', - packument: { - versions: { '2.118.0-beta.52': {}, '2.118.0-beta.51': {} }, - }, + await expect(resolveCliVersionSpec('^2.120.0')).rejects.toThrow( + /did not include a "versions" object/ + ); }); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('canary')).resolves.toBe('2.118.0-beta.51'); - }); + it('fetches the packument once across multiple range specs', async () => { + const fetchMock = routedFetchMock({ + packument: packument('2.121.0', ['1.5.0', '2.120.0', '2.121.0']), + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - it('throws instead of walking back a missing asset that is not beta-shaped, even on the beta dist-tag', async () => { - const fetchMock = routedFetchMock({ - distTags: { beta: '1.4.0-rc.2' }, - assetOk: () => false, - packument: { versions: { '1.4.0-rc.2': {} } }, - }); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); + await expect( + Promise.all([ + resolveCliVersionSpec('^2.120.0'), + resolveCliVersionSpec('^1.0.0'), + resolveCliVersionSpec('>=2'), + ]) + ).resolves.toEqual(['2.121.0', '1.5.0', '2.121.0']); - await expect(resolveCliDistTag('beta')).rejects.toThrow( - `npm's "beta" dist-tag for "supabase" points at 1.4.0-rc.2, but its release asset is missing` - ); - expect(fetchMock.mock.calls.some(([url]) => url === PACKUMENT_URL)).toBe( - false - ); + expect(urlsOf(fetchMock, 'GET')).toEqual([PACKUMENT_URL]); + }); }); describe(PINS_ENV, () => { @@ -580,23 +471,36 @@ describe('resolveCliDistTag', () => { process.env[PINS_ENV] = JSON.stringify({ latest: 'v9.9.9' }); const fetchMock = vi.fn(); vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('latest')).resolves.toBe('9.9.9'); + await expect(resolveCliVersionSpec('latest')).resolves.toBe('9.9.9'); expect(fetchMock).not.toHaveBeenCalled(); }); - it('only pins the tags it names and resolves the rest from npm', async () => { + it('pins a range spec', async () => { + process.env[PINS_ENV] = JSON.stringify({ '^2.120.0': '2.121.0' }); + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionSpec('^2.120.0')).resolves.toBe('2.121.0'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('only pins the specs it names and resolves the rest from npm', async () => { process.env[PINS_ENV] = JSON.stringify({ latest: '9.9.9' }); const fetchMock = routedFetchMock({ distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, assetOk: () => true, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('latest')).resolves.toBe('9.9.9'); - await expect(resolveCliDistTag('beta')).resolves.toBe('1.3.0-beta.1'); + await expect(resolveCliVersionSpec('latest')).resolves.toBe('9.9.9'); + await expect(resolveCliVersionSpec('beta')).resolves.toBe('1.3.0-beta.1'); + await expect(resolveCliVersionSpec('toString')).rejects.toThrow( + 'npm has no "toString" dist-tag' + ); }); it('treats a blank or whitespace-only value as unset', async () => { @@ -606,9 +510,9 @@ describe('resolveCliDistTag', () => { assetOk: () => true, }); vi.stubGlobal('fetch', fetchMock); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('latest')).resolves.toBe('1.2.3'); + await expect(resolveCliVersionSpec('latest')).resolves.toBe('1.2.3'); }); it.each([ @@ -622,23 +526,14 @@ describe('resolveCliDistTag', () => { ])('throws for %s', async (_name, value, message) => { process.env[PINS_ENV] = value; vi.stubGlobal('fetch', vi.fn()); - const { resolveCliDistTag } = await import('../src/cli-channel.js'); + const { resolveCliVersionSpec } = await import('../src/cli-channel.js'); - await expect(resolveCliDistTag('latest')).rejects.toThrow(message); + await expect(resolveCliVersionSpec('latest')).rejects.toThrow(message); }); }); }); describe('resolveCliVersionOption', () => { - it('passes an exact version through unchanged, without touching the network', async () => { - const fetchMock = vi.fn(); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersionOption } = await import('../src/cli-channel.js'); - - await expect(resolveCliVersionOption('2.109.1')).resolves.toBe('2.109.1'); - expect(fetchMock).not.toHaveBeenCalled(); - }); - it('passes undefined through unchanged, without touching the network', async () => { const fetchMock = vi.fn(); vi.stubGlobal('fetch', fetchMock); @@ -648,50 +543,26 @@ describe('resolveCliVersionOption', () => { expect(fetchMock).not.toHaveBeenCalled(); }); - it('strips a leading v from an exact version', async () => { + it('passes an exact version through unchanged, without touching the network', async () => { const fetchMock = vi.fn(); vi.stubGlobal('fetch', fetchMock); const { resolveCliVersionOption } = await import('../src/cli-channel.js'); - await expect(resolveCliVersionOption('v2.109.1')).resolves.toBe('2.109.1'); + await expect(resolveCliVersionOption('2.109.1')).resolves.toBe('2.109.1'); expect(fetchMock).not.toHaveBeenCalled(); }); - it('resolves a "latest" dist-tag against npm', async () => { + it('resolves a dist-tag and a range against npm', async () => { const fetchMock = routedFetchMock({ - distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + distTags: { latest: '1.2.3' }, + packument: packument('1.2.3', ['1.2.3']), assetOk: () => true, }); vi.stubGlobal('fetch', fetchMock); const { resolveCliVersionOption } = await import('../src/cli-channel.js'); await expect(resolveCliVersionOption('latest')).resolves.toBe('1.2.3'); - }); - - it('resolves an arbitrary dist-tag against npm', async () => { - const fetchMock = routedFetchMock({ - distTags: { latest: '1.2.3', canary: '1.4.0-canary.7' }, - assetOk: () => true, - }); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersionOption } = await import('../src/cli-channel.js'); - - await expect(resolveCliVersionOption('canary')).resolves.toBe( - '1.4.0-canary.7' - ); - }); - - it('throws listing the available dist-tags for an unknown tag', async () => { - const fetchMock = routedFetchMock({ - distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, - assetOk: () => true, - }); - vi.stubGlobal('fetch', fetchMock); - const { resolveCliVersionOption } = await import('../src/cli-channel.js'); - - await expect(resolveCliVersionOption('canary')).rejects.toThrow( - 'npm has no "canary" dist-tag for "supabase"; available dist-tags: latest, beta' - ); + await expect(resolveCliVersionOption('^1.0.0')).resolves.toBe('1.2.3'); }); }); @@ -704,7 +575,7 @@ describe('isExactCliVersion', () => { } ); - it.each(['latest', 'beta', 'next', 'canary', 'v1', '1.2'])( + it.each(['latest', 'beta', 'next', 'canary', 'v1', '1.2', '^2.120.0'])( 'rejects %s', async (value) => { const { isExactCliVersion } = await import('../src/cli-channel.js'); @@ -712,38 +583,3 @@ describe('isExactCliVersion', () => { } ); }); - -describe('compareBetaVersionsDesc', () => { - it('sorts newer beta numbers before older ones within the same minor', async () => { - const { compareBetaVersionsDesc } = await import('../src/cli-channel.js'); - - expect( - compareBetaVersionsDesc('2.118.0-beta.10', '2.118.0-beta.9') - ).toBeLessThan(0); - expect( - compareBetaVersionsDesc('2.118.0-beta.9', '2.118.0-beta.10') - ).toBeGreaterThan(0); - }); - - it('sorts a newer minor before an older minor regardless of beta number', async () => { - const { compareBetaVersionsDesc } = await import('../src/cli-channel.js'); - - expect( - compareBetaVersionsDesc('2.119.0-beta.1', '2.118.0-beta.60') - ).toBeLessThan(0); - }); - - it('treats equal versions as equal', async () => { - const { compareBetaVersionsDesc } = await import('../src/cli-channel.js'); - - expect(compareBetaVersionsDesc('2.118.0-beta.1', '2.118.0-beta.1')).toBe(0); - }); - - it('throws for a non "X.Y.Z-beta.N" version', async () => { - const { compareBetaVersionsDesc } = await import('../src/cli-channel.js'); - - expect(() => - compareBetaVersionsDesc('2.118.0-rc.1', '2.118.0-beta.1') - ).toThrow(); - }); -}); diff --git a/packages/sandbox/test/local-stack-docker.test.ts b/packages/sandbox/test/local-stack-docker.test.ts index 992002c3..47f756bc 100644 --- a/packages/sandbox/test/local-stack-docker.test.ts +++ b/packages/sandbox/test/local-stack-docker.test.ts @@ -43,6 +43,19 @@ describe('localStackRuntime id', () => { }); }); +describe('localStackRuntime cliVersionSpec', () => { + it.each(['latest', 'beta', '^2.120.0'])('exposes %s', (cliVersion) => { + expect(localStackRuntime({ cliVersion }).cliVersionSpec).toBe(cliVersion); + }); + + it.each([undefined, '2.109.1', 'v2.109.1'])( + 'is unset for %s', + (cliVersion) => { + expect(localStackRuntime({ cliVersion }).cliVersionSpec).toBeUndefined(); + } + ); +}); + describe('buildSupabaseShimScript', () => { it('emits DOCKER_HOST, the -x start branch for excluded services, and a passthrough exec', () => { const excluded: SupabaseService[] = ['gotrue', 'kong']; @@ -165,7 +178,7 @@ describe('buildLocalStackScoringContext environmentMarker', () => { }); } - it.each(['beta', 'canary'])( + it.each(['beta', 'canary', '^2.120.0'])( 'includes an optional channel %s when present', async (channel) => { const marker = { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4fac864d..81140c11 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -382,10 +382,16 @@ importers: ai: specifier: 'catalog:' version: 6.0.199(zod@4.4.3) + semver: + specifier: ^7.8.5 + version: 7.8.5 devDependencies: '@types/node': specifier: 'catalog:' version: 22.19.20 + '@types/semver': + specifier: ^7.8.0 + version: 7.8.0 typescript: specifier: 'catalog:' version: 5.9.3 @@ -2494,6 +2500,9 @@ packages: '@types/react@19.2.17': resolution: {integrity: sha512-MXfmqaVPEVgkBT/aY0aGCkRWWtByiYQXo3xdQ8r5RzuFrPiRn8Gar2tQdXSUQ2GKV3bkXckek89V8wQBY2Q/Aw==} + '@types/semver@7.8.0': + resolution: {integrity: sha512-1mAINjtQCXXeLkJ9ehXkwOcBpqtLxiVtKhpUf83DdRNdQKV0iXZpaHYqRr7nj+wvxuJzoAmAwXI+sCNMv1CzLQ==} + '@types/shell-quote@1.7.5': resolution: {integrity: sha512-+UE8GAGRPbJVQDdxi16dgadcBfQ+KG2vgZhV1+3A1XmHbmwcdwhCUwIdy+d3pAGrbvgRoVSjeI9vOWyq376Yzw==} @@ -4386,6 +4395,11 @@ packages: engines: {node: '>=10'} hasBin: true + semver@7.8.5: + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} + engines: {node: '>=10'} + hasBin: true + send@1.2.1: resolution: {integrity: sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==} engines: {node: '>= 18'} @@ -6812,6 +6826,8 @@ snapshots: dependencies: csstype: 3.2.3 + '@types/semver@7.8.0': {} + '@types/shell-quote@1.7.5': {} '@types/validate-npm-package-name@4.0.2': {} @@ -8651,6 +8667,8 @@ snapshots: semver@7.8.3: {} + semver@7.8.5: {} + send@1.2.1: dependencies: debug: 4.4.3(supports-color@10.2.2)