diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 7dc78e7a..29f5f12d 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -524,6 +524,17 @@ jobs: fi done + - name: Find the PRs for this branch + # Dispatched runs have a branch ref, not a PR ref, so look up the + # branch's open PRs for the job summary. + if: github.event_name == 'workflow_dispatch' + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: | + urls=$(gh pr list --head "$GITHUB_REF_NAME" --state open --json url,isCrossRepository --jq '.[] | select(.isCrossRepository == false) | .url' | paste -sd ' ' -) + echo "PR_URLS=$urls" >> "$GITHUB_ENV" + - name: Upload results to Braintrust # A Braintrust outage shouldn't block publishing results. continue-on-error: true diff --git a/apps/framework/scripts/upload-braintrust.test.ts b/apps/framework/scripts/upload-braintrust.test.ts index dcee13b3..ecff60b0 100644 --- a/apps/framework/scripts/upload-braintrust.test.ts +++ b/apps/framework/scripts/upload-braintrust.test.ts @@ -1,8 +1,10 @@ -import { describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { + ciRunUrl, experimentName, judgeMetrics, logTranscript, + relatedPrLines, runViewUrl, unwrapShell, type SpanSink, @@ -80,6 +82,39 @@ describe('utcStamp', () => { }); }); +describe('relatedPrLines', () => { + it('lists each PR, or says none', () => { + expect( + relatedPrLines([ + 'https://github.com/supabase/evals/pull/386', + 'https://github.com/supabase/evals/pull/384', + ]) + ).toEqual([ + '**Related PRs:**', + '', + '- [#386](https://github.com/supabase/evals/pull/386)', + '- [#384](https://github.com/supabase/evals/pull/384)', + '', + ]); + expect(relatedPrLines([])).toEqual(['**Related PRs:** (none)', '']); + }); +}); + +describe('ciRunUrl', () => { + afterEach(() => vi.unstubAllEnvs()); + + it('links the GitHub Actions run, and is empty outside CI', () => { + vi.stubEnv('GITHUB_SERVER_URL', 'https://github.com'); + vi.stubEnv('GITHUB_REPOSITORY', 'supabase/evals'); + vi.stubEnv('GITHUB_RUN_ID', '123'); + expect(ciRunUrl()).toEqual({ + ci_run_url: 'https://github.com/supabase/evals/actions/runs/123', + }); + vi.stubEnv('GITHUB_RUN_ID', ''); + expect(ciRunUrl()).toEqual({}); + }); +}); + describe('runViewUrl', () => { it('filters the experiments list by run id', () => { const url = runViewUrl( diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts index be1e2d17..4180d144 100644 --- a/apps/framework/scripts/upload-braintrust.ts +++ b/apps/framework/scripts/upload-braintrust.ts @@ -268,6 +268,26 @@ function prUrl(): Record { : {}; } +export function relatedPrLines(urls: string[]) { + return urls.length + ? [ + '**Related PRs:**', + '', + ...urls.map((url) => `- [#${url.split('/').pop()}](${url})`), + '', + ] + : ['**Related PRs:** (none)', '']; +} + +export function ciRunUrl() { + const { GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID } = process.env; + return GITHUB_SERVER_URL && GITHUB_REPOSITORY && GITHUB_RUN_ID + ? { + ci_run_url: `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`, + } + : {}; +} + async function collectRows( experimentMetadata: Map ): Promise> { @@ -731,6 +751,7 @@ async function main() { // Duplicated from repo_info because grouping only reads metadata. ...(info?.branch ? { branch: info.branch } : {}), ...prUrl(), + ...ciRunUrl(), }, }); @@ -782,10 +803,15 @@ async function main() { console.log(`🔗 All experiments in this run → ${runUrl}`); // https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-commands#adding-a-job-summary if (process.env.GITHUB_STEP_SUMMARY) { + // PR events know their PR from the ref. Dispatched runs get theirs from + // the "Find the PRs for this branch" step. + const { pr_url } = prUrl(); + const prUrls = pr_url + ? [pr_url] + : (process.env.PR_URLS?.split(' ').filter(Boolean) ?? []); const lines = [ - '## Braintrust', - '', - `[All experiments in this run](${runUrl})`, + ...relatedPrLines(prUrls), + `**Braintrust experiments:** [(view all)](${runUrl})`, '', ...uploaded.map(({ name, url }) => `- [${name}](${url})`), '',