diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 63e0d3162..22d4304e7 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -261,6 +261,7 @@ jobs: MULTI_AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-agentic-eval) SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix single) MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi) + EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix eval) MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix multi-eval) { echo "agentic-config=$AGENTIC" @@ -275,7 +276,7 @@ jobs: test-sweep-multi-node: needs: get-jobs - if: ${{ needs.get-jobs.outputs.multi-node-config != '[]' }} + if: ${{ needs.get-jobs.outputs.multi-node-config != '' && needs.get-jobs.outputs.multi-node-config != '[]' }} uses: ./.github/workflows/benchmark-multinode-tmpl.yml name: multi-node / strategy: @@ -330,7 +331,7 @@ jobs: test-sweep-multi-node-evals: needs: get-jobs - if: ${{ needs.get-jobs.outputs.multi-node-eval-config != '[]' }} + if: ${{ needs.get-jobs.outputs.multi-node-eval-config != '' && needs.get-jobs.outputs.multi-node-eval-config != '[]' }} uses: ./.github/workflows/benchmark-multinode-tmpl.yml name: multi-node eval / strategy: @@ -385,7 +386,7 @@ jobs: test-sweep-agentic: needs: get-jobs - if: ${{ needs.get-jobs.outputs.agentic-config != '[]' }} + if: ${{ needs.get-jobs.outputs.agentic-config != '' && needs.get-jobs.outputs.agentic-config != '[]' }} uses: ./.github/workflows/benchmark-tmpl.yml name: agentic / strategy: @@ -431,7 +432,7 @@ jobs: test-sweep-agentic-evals: needs: get-jobs - if: ${{ needs.get-jobs.outputs.agentic-eval-config != '[]' }} + if: ${{ needs.get-jobs.outputs.agentic-eval-config != '' && needs.get-jobs.outputs.agentic-eval-config != '[]' }} uses: ./.github/workflows/benchmark-tmpl.yml name: agentic eval / strategy: @@ -473,7 +474,7 @@ jobs: test-sweep-multi-node-agentic: needs: get-jobs - if: ${{ needs.get-jobs.outputs.multi-node-agentic-config != '[]' }} + if: ${{ needs.get-jobs.outputs.multi-node-agentic-config != '' && needs.get-jobs.outputs.multi-node-agentic-config != '[]' }} uses: ./.github/workflows/benchmark-multinode-tmpl.yml name: multi-node agentic / strategy: @@ -533,7 +534,7 @@ jobs: test-sweep-multi-node-agentic-evals: needs: get-jobs - if: ${{ needs.get-jobs.outputs.multi-node-agentic-eval-config != '[]' }} + if: ${{ needs.get-jobs.outputs.multi-node-agentic-eval-config != '' && needs.get-jobs.outputs.multi-node-agentic-eval-config != '[]' }} uses: ./.github/workflows/benchmark-multinode-tmpl.yml name: multi-node agentic eval / strategy: @@ -594,7 +595,7 @@ jobs: test-sweep-single-node: needs: get-jobs - if: ${{ needs.get-jobs.outputs.single-node-config != '[]' }} + if: ${{ needs.get-jobs.outputs.single-node-config != '' && needs.get-jobs.outputs.single-node-config != '[]' }} uses: ./.github/workflows/benchmark-tmpl.yml name: single-node / strategy: @@ -633,7 +634,7 @@ jobs: test-sweep-evals: needs: get-jobs - if: ${{ needs.get-jobs.outputs.eval-config != '[]' }} + if: ${{ needs.get-jobs.outputs.eval-config != '' && needs.get-jobs.outputs.eval-config != '[]' }} uses: ./.github/workflows/benchmark-tmpl.yml name: eval / strategy: diff --git a/utils/test_ci_priority.py b/utils/test_ci_priority.py index 1bef5fcda..9fa61a2a1 100644 --- a/utils/test_ci_priority.py +++ b/utils/test_ci_priority.py @@ -1,4 +1,5 @@ import json +import re import shlex from copy import deepcopy from decimal import Decimal @@ -260,3 +261,17 @@ def test_queue_tokens_change_between_run_attempts(): "123:2", ("0",), ) + + +def test_e2e_matrix_outputs_reference_initialized_shell_variables(): + workflow = ( + Path(__file__).parents[1] / ".github" / "workflows" / "e2e-tests.yml" + ).read_text() + assignments = set( + re.findall(r"^\s*([A-Z][A-Z0-9_]*)=\$\(", workflow, re.MULTILINE) + ) + output_variables = set( + re.findall(r'^\s*echo "[a-z-]+=\$([A-Z][A-Z0-9_]*)"', workflow, re.MULTILINE) + ) + + assert output_variables <= assignments