Implement test- naming convention for CI jobs

src/ci/citool/src/jobs.rs

@@ -108,7 +108,6 @@ pub fn load_job_db(db: &str) -> anyhow::Result<JobDatabase> {
     let mut db: JobDatabase = serde_yaml::from_value(db).context("failed to parse job database")?;
 
     register_pr_jobs_as_auto_jobs(&mut db)?;
-
     validate_job_database(&db)?;
 
     Ok(db)
@@ -127,7 +126,7 @@ pub fn load_job_db(db: &str) -> anyhow::Result<JobDatabase> {
 /// CI runs to be red until the cause is fixed.
 fn register_pr_jobs_as_auto_jobs(db: &mut JobDatabase) -> anyhow::Result<()> {
     for pr_job in &db.pr_jobs {
-        // It's acceptable to "override" a PR job in Auto job, for instance, `x86_64-gnu-tools` will
+        // It's acceptable to "override" a PR job in Auto job, for instance, `test-x86_64-gnu-tools` will
         // receive an additional `DEPLOY_TOOLSTATES_JSON: toolstates-linux.json` env when under Auto
         // environment versus PR environment.
         if db.find_auto_job_by_name(&pr_job.name).is_some() {
@@ -212,6 +211,18 @@ fn validate_job_database(db: &JobDatabase) -> anyhow::Result<()> {
         }
     }
 
+    // All jobs should follow a naming convention - either they are test or dist jobs.
+    for job in &db.auto_jobs {
+        let name = job.name.strip_prefix("optional-").unwrap_or(&job.name);
+        if name.starts_with("dist-") || name.starts_with("test-") {
+            continue;
+        }
+        return Err(anyhow!(
+            "Auto job `{job}` name must start with test- or dist-`.",
+            job = job.name
+        ));
+    }
+
     Ok(())
 }
 
@@ -381,10 +392,11 @@ pub fn calculate_job_matrix(
     })?;
     eprintln!("Run type: {run_type:?}");
 
-    let jobs = calculate_jobs(&run_type, &db, channel)?;
+    let mut jobs = calculate_jobs(&run_type, &db, channel)?;
     if jobs.is_empty() && !matches!(run_type, RunType::MainJob) {
         return Err(anyhow::anyhow!("Computed job list is empty"));
     }
+    jobs.sort_by_key(|j| j.name.clone());
 
     let run_type = match run_type {
         RunType::PullRequest => "pr",

src/ci/citool/src/jobs/tests.rs

@@ -159,7 +159,7 @@ envs:
   optional:
 
 pr:
-    - name: pr-ci-a
+    - name: test-pr-ci-a
       os: ubuntu
       env: {}
 try:
@@ -169,7 +169,10 @@ optional:
     )
     .unwrap();
 
-    assert_eq!(db.auto_jobs.iter().map(|j| j.name.as_str()).collect::<Vec<_>>(), vec!["pr-ci-a"])
+    assert_eq!(
+        db.auto_jobs.iter().map(|j| j.name.as_str()).collect::<Vec<_>>(),
+        vec!["test-pr-ci-a"]
+    )
 }
 
 #[test]
@@ -183,7 +186,7 @@ envs:
   optional:
 
 pr:
-    - name: tidy
+    - name: test-tidy
       env:
         DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
       continue_on_error: true
@@ -195,7 +198,7 @@ optional:
     )
     .unwrap();
 
-    assert_eq!(db.auto_jobs.iter().map(|j| j.name.as_str()).collect::<Vec<_>>(), vec!["tidy"]);
+    assert_eq!(db.auto_jobs.iter().map(|j| j.name.as_str()).collect::<Vec<_>>(), vec!["test-tidy"]);
     assert_eq!(db.auto_jobs[0].continue_on_error, Some(false));
     assert_eq!(
         db.auto_jobs[0].env,
@@ -218,10 +221,10 @@ envs:
 
 
 pr:
-    - name: pr-ci-a
+    - name: test-pr-ci-a
       os: ubuntu
       env: {}
-    - name: pr-ci-a
+    - name: test-pr-ci-a
       os: ubuntu
       env: {}
 try:
@@ -243,12 +246,12 @@ envs:
   optional:
 
 pr:
-    - name: tidy
+    - name: test-tidy
       os: ubuntu
       env: {}
 try:
 auto:
-    - name: tidy
+    - name: test-tidy
       env:
         DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
       continue_on_error: false
@@ -258,7 +261,7 @@ optional:
     )
     .unwrap();
 
-    assert_eq!(db.auto_jobs.iter().map(|j| j.name.as_str()).collect::<Vec<_>>(), vec!["tidy"]);
+    assert_eq!(db.auto_jobs.iter().map(|j| j.name.as_str()).collect::<Vec<_>>(), vec!["test-tidy"]);
     assert_eq!(db.auto_jobs[0].continue_on_error, Some(false));
     assert_eq!(
         db.auto_jobs[0].env,
@@ -280,14 +283,14 @@ envs:
   optional:
 
 pr:
-    - name: tidy
+    - name: test-tidy
       continue_on_error: true
       env:
         ENV_ALLOWED_TO_DIFFER: "hello world"
       os: ubuntu
 try:
 auto:
-    - name: tidy
+    - name: test-tidy
       continue_on_error: false
       env:
         ENV_ALLOWED_TO_DIFFER: "goodbye world"
@@ -300,7 +303,7 @@ optional:
     // `continue_on_error` and `env` are carve-outs *allowed* to diverge between PR and Auto job of
     // the same name. Should load successfully.
 
-    assert_eq!(db.auto_jobs.iter().map(|j| j.name.as_str()).collect::<Vec<_>>(), vec!["tidy"]);
+    assert_eq!(db.auto_jobs.iter().map(|j| j.name.as_str()).collect::<Vec<_>>(), vec!["test-tidy"]);
     assert_eq!(db.auto_jobs[0].continue_on_error, Some(false));
     assert_eq!(
         db.auto_jobs[0].env,
@@ -324,14 +327,14 @@ envs:
   optional:
 
 pr:
-    - name: tidy
+    - name: test-tidy
       continue_on_error: true
       env:
         ENV_ALLOWED_TO_DIFFER: "hello world"
       os: ubuntu
 try:
 auto:
-    - name: tidy
+    - name: test-tidy
       continue_on_error: false
       env:
         ENV_ALLOWED_TO_DIFFER: "goodbye world"
@@ -357,7 +360,7 @@ envs:
 pr:
 try:
 auto:
-    - name: tidy
+    - name: test-tidy
       continue_on_error: true
       os: windows
       env: {}
@@ -366,3 +369,26 @@ optional:
     )
     .unwrap();
 }
+
+#[test]
+#[should_panic = "must start with test-"]
+fn missing_test_job_prefix() {
+    let _ = load_job_db(
+        r#"
+envs:
+  pr:
+  try:
+  auto:
+  optional:
+
+pr:
+try:
+auto:
+    - name: tidy
+      os: windows
+      env: {}
+optional:
+"#,
+    )
+    .unwrap();
+}

src/ci/citool/tests/jobs.rs

@@ -6,7 +6,7 @@ const TEST_JOBS_YML_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/tes
 fn auto_jobs() {
     let stdout = get_matrix("push", "commit", "refs/heads/automation/bors/auto");
     insta::assert_snapshot!(stdout, @r#"
-    jobs=[{"name":"aarch64-gnu","full_name":"auto - aarch64-gnu","os":"ubuntu-22.04-arm","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"free_disk":true},{"name":"x86_64-gnu-llvm-18-1","full_name":"auto - x86_64-gnu-llvm-18-1","os":"ubuntu-24.04","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","DOCKER_SCRIPT":"stage_2_test_set1.sh","IMAGE":"x86_64-gnu-llvm-18","READ_ONLY_SRC":"0","RUST_BACKTRACE":1,"TOOLSTATE_PUBLISH":1},"free_disk":true},{"name":"aarch64-apple","full_name":"auto - aarch64-apple","os":"macos-15","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","DEVELOPER_DIR":"/Applications/Xcode_26.2.app/Contents/Developer","MACOSX_DEPLOYMENT_TARGET":11.0,"MACOSX_STD_DEPLOYMENT_TARGET":11.0,"NO_DEBUG_ASSERTIONS":1,"NO_LLVM_ASSERTIONS":1,"NO_OVERFLOW_CHECKS":1,"RUSTC_RETRY_LINKER_ON_SEGFAULT":1,"RUST_CONFIGURE_ARGS":"--enable-sanitizers --enable-profiler --set build.allocator=jemalloc","SCRIPT":"./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin","TOOLSTATE_PUBLISH":1}},{"name":"dist-i686-msvc","full_name":"auto - dist-i686-msvc","os":"windows-2022","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","CODEGEN_BACKENDS":"llvm,cranelift","DEPLOY_BUCKET":"rust-lang-ci2","DIST_REQUIRE_ALL_TOOLS":1,"RUST_CONFIGURE_ARGS":"--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler","SCRIPT":"python x.py dist bootstrap --include-default-paths","TOOLSTATE_PUBLISH":1}},{"name":"pr-check-1","full_name":"auto - pr-check-1","os":"ubuntu-24.04","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"continue_on_error":false,"free_disk":true},{"name":"pr-check-2","full_name":"auto - pr-check-2","os":"ubuntu-24.04","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"continue_on_error":false,"free_disk":true},{"name":"tidy","full_name":"auto - tidy","os":"ubuntu-24.04","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"continue_on_error":false,"free_disk":true,"doc_url":"https://foo.bar"}]
+    jobs=[{"name":"dist-i686-msvc","full_name":"auto - dist-i686-msvc","os":"windows-2022","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","CODEGEN_BACKENDS":"llvm,cranelift","DEPLOY_BUCKET":"rust-lang-ci2","DIST_REQUIRE_ALL_TOOLS":1,"RUST_CONFIGURE_ARGS":"--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler","SCRIPT":"python x.py dist bootstrap --include-default-paths","TOOLSTATE_PUBLISH":1}},{"name":"test-aarch64-apple","full_name":"auto - test-aarch64-apple","os":"macos-15","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","DEVELOPER_DIR":"/Applications/Xcode_26.2.app/Contents/Developer","MACOSX_DEPLOYMENT_TARGET":11.0,"MACOSX_STD_DEPLOYMENT_TARGET":11.0,"NO_DEBUG_ASSERTIONS":1,"NO_LLVM_ASSERTIONS":1,"NO_OVERFLOW_CHECKS":1,"RUSTC_RETRY_LINKER_ON_SEGFAULT":1,"RUST_CONFIGURE_ARGS":"--enable-sanitizers --enable-profiler --set build.allocator=jemalloc","SCRIPT":"./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin","TOOLSTATE_PUBLISH":1}},{"name":"test-aarch64-gnu","full_name":"auto - test-aarch64-gnu","os":"ubuntu-22.04-arm","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"free_disk":true},{"name":"test-pr-check-1","full_name":"auto - test-pr-check-1","os":"ubuntu-24.04","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"continue_on_error":false,"free_disk":true},{"name":"test-pr-check-2","full_name":"auto - test-pr-check-2","os":"ubuntu-24.04","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"continue_on_error":false,"free_disk":true},{"name":"test-tidy","full_name":"auto - test-tidy","os":"ubuntu-24.04","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"continue_on_error":false,"free_disk":true,"doc_url":"https://foo.bar"},{"name":"test-x86_64-gnu-llvm-18-1","full_name":"auto - test-x86_64-gnu-llvm-18-1","os":"ubuntu-24.04","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","DOCKER_SCRIPT":"stage_2_test_set1.sh","IMAGE":"x86_64-gnu-llvm-18","READ_ONLY_SRC":"0","RUST_BACKTRACE":1,"TOOLSTATE_PUBLISH":1},"free_disk":true}]
     run_type=auto
     "#);
 }
@@ -26,21 +26,21 @@ fn try_custom_jobs() {
         "push",
         r#"This is a test PR
 
-try-job: aarch64-gnu
+try-job: test-aarch64-gnu
 try-job: dist-i686-msvc"#,
         "refs/heads/automation/bors/try",
     );
-    insta::assert_snapshot!(stdout, @r###"
-    jobs=[{"name":"aarch64-gnu","full_name":"try - aarch64-gnu","os":"ubuntu-22.04-arm","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"free_disk":true},{"name":"dist-i686-msvc","full_name":"try - dist-i686-msvc","os":"windows-2022","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","CODEGEN_BACKENDS":"llvm,cranelift","DEPLOY_BUCKET":"rust-lang-ci2","DIST_REQUIRE_ALL_TOOLS":1,"RUST_CONFIGURE_ARGS":"--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler","SCRIPT":"python x.py dist bootstrap --include-default-paths","TOOLSTATE_PUBLISH":1}}]
+    insta::assert_snapshot!(stdout, @r#"
+    jobs=[{"name":"dist-i686-msvc","full_name":"try - dist-i686-msvc","os":"windows-2022","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","CODEGEN_BACKENDS":"llvm,cranelift","DEPLOY_BUCKET":"rust-lang-ci2","DIST_REQUIRE_ALL_TOOLS":1,"RUST_CONFIGURE_ARGS":"--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler","SCRIPT":"python x.py dist bootstrap --include-default-paths","TOOLSTATE_PUBLISH":1}},{"name":"test-aarch64-gnu","full_name":"try - test-aarch64-gnu","os":"ubuntu-22.04-arm","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"free_disk":true}]
     run_type=try
-    "###);
+    "#);
 }
 
 #[test]
 fn pr_jobs() {
     let stdout = get_matrix("pull_request", "commit", "refs/heads/pr/1234");
     insta::assert_snapshot!(stdout, @r#"
-    jobs=[{"name":"pr-check-1","full_name":"PR - pr-check-1","os":"ubuntu-24.04","env":{"PR_CI_JOB":1},"free_disk":true},{"name":"pr-check-2","full_name":"PR - pr-check-2","os":"ubuntu-24.04","env":{"PR_CI_JOB":1},"free_disk":true},{"name":"tidy","full_name":"PR - tidy","os":"ubuntu-24.04","env":{"PR_CI_JOB":1},"continue_on_error":true,"free_disk":true,"doc_url":"https://foo.bar"}]
+    jobs=[{"name":"test-pr-check-1","full_name":"PR - test-pr-check-1","os":"ubuntu-24.04","env":{"PR_CI_JOB":1},"free_disk":true},{"name":"test-pr-check-2","full_name":"PR - test-pr-check-2","os":"ubuntu-24.04","env":{"PR_CI_JOB":1},"free_disk":true},{"name":"test-tidy","full_name":"PR - test-tidy","os":"ubuntu-24.04","env":{"PR_CI_JOB":1},"continue_on_error":true,"free_disk":true,"doc_url":"https://foo.bar"}]
     run_type=pr
     "#);
 }

src/ci/citool/tests/test-jobs.yml

@@ -64,11 +64,11 @@ envs:
 # These jobs automatically inherit envs.pr, to avoid repeating
 # it in each job definition.
 pr:
-  - name: pr-check-1
+  - name: test-pr-check-1
     <<: *job-linux-4c
-  - name: pr-check-2
+  - name: test-pr-check-2
     <<: *job-linux-4c
-  - name: tidy
+  - name: test-tidy
     continue_on_error: true
     doc_url: https://foo.bar
     <<: *job-linux-4c
@@ -86,12 +86,12 @@ try:
 # These jobs automatically inherit envs.auto, to avoid repeating
 # it in each job definition.
 auto:
-  - name: aarch64-gnu
+  - name: test-aarch64-gnu
     <<: *job-aarch64-linux
 
   # The x86_64-gnu-llvm-18 job is split into multiple jobs to run tests in parallel.
   # x86_64-gnu-llvm-18-1 skips tests that run in x86_64-gnu-llvm-18-{2,3}.
-  - name: x86_64-gnu-llvm-18-1
+  - name: test-x86_64-gnu-llvm-18-1
     env:
       RUST_BACKTRACE: 1
       READ_ONLY_SRC: "0"
@@ -104,7 +104,7 @@ auto:
   #  macOS Builders  #
   ####################
 
-  - name: aarch64-apple
+  - name: test-aarch64-apple
     env:
       SCRIPT: ./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin
       RUST_CONFIGURE_ARGS: >-