CI: tolerate transient sccache backend failures (#3346)

closes #3273

Skip startup cache checks in Linux wheel builds so transient GHA cache
failures do not prevent compilation. Configure cache I/O backoff after
five errors within five seconds.

Apply the same backoff to Linux sdist builds using local-disk sccache.
Omit startup-check skipping there because the disk check performs no I/O.

Validation: actionlint, YAML configuration checks, and git diff --check.

Author: jpascucci-nv

.github/workflows/build-wheel.yml

@@ -238,6 +238,7 @@ jobs:
           # extras: uv puts uv on PATH (required on Windows).
           CIBW_BUILD_FRONTEND: "build[uv]"
           # CIBW mounts the host filesystem under /host
+          # Skip startup cache probes and back off on repeated cache errors (#3273).
           CIBW_ENVIRONMENT_LINUX: >
             CUDA_PATH=/host/${{ env.CUDA_PATH }}
             CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
@@ -247,6 +248,9 @@ jobs:
             CXX="/host/${{ env.SCCACHE_PATH }} c++"
             SCCACHE_GHA_ENABLED=true
             SCCACHE_PREPROCESSOR_CACHE_GHA_ENABLED=true
+            SCCACHE_SKIP_CACHE_CHECK=true
+            SCCACHE_CACHE_RATE_LIMIT_ON_ERROR_COUNT=5
+            SCCACHE_CACHE_RATE_LIMIT_ON_ERROR_WINDOW_SIZE_SECS=5
             ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
             ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
             ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}
@@ -335,6 +339,7 @@ jobs:
           # See the cuda.bindings step above for UV_* vs PIP_*.
           CIBW_BUILD_FRONTEND: "build[uv]"
           # CIBW mounts the host filesystem under /host
+          # Skip startup cache probes and back off on repeated cache errors (#3273).
           CIBW_ENVIRONMENT_LINUX: >
             CUDA_PATH=/host/${{ env.CUDA_PATH }}
             CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
@@ -346,6 +351,9 @@ jobs:
             CXX="/host/${{ env.SCCACHE_PATH }} c++"
             SCCACHE_GHA_ENABLED=true
             SCCACHE_PREPROCESSOR_CACHE_GHA_ENABLED=true
+            SCCACHE_SKIP_CACHE_CHECK=true
+            SCCACHE_CACHE_RATE_LIMIT_ON_ERROR_COUNT=5
+            SCCACHE_CACHE_RATE_LIMIT_ON_ERROR_WINDOW_SIZE_SECS=5
             ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
             ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
             ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}
@@ -614,6 +622,7 @@ jobs:
           # See the cuda.bindings step above for UV_* vs PIP_*.
           CIBW_BUILD_FRONTEND: "build[uv]"
           # CIBW mounts the host filesystem under /host
+          # Skip startup cache probes and back off on repeated cache errors (#3273).
           CIBW_ENVIRONMENT_LINUX: >
             CUDA_PATH=/host/${{ env.CUDA_PATH }}
             CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
@@ -625,6 +634,9 @@ jobs:
             CXX="/host/${{ env.SCCACHE_PATH }} c++"
             SCCACHE_GHA_ENABLED=true
             SCCACHE_PREPROCESSOR_CACHE_GHA_ENABLED=true
+            SCCACHE_SKIP_CACHE_CHECK=true
+            SCCACHE_CACHE_RATE_LIMIT_ON_ERROR_COUNT=5
+            SCCACHE_CACHE_RATE_LIMIT_ON_ERROR_WINDOW_SIZE_SECS=5
             ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
             ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
             ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}

.github/workflows/test-sdist-linux.yml

@@ -34,6 +34,9 @@ jobs:
       BUILD_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_build }}
       BUILD_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_build }}
       BUILD_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_build }}
+      # Back off on repeated local-disk cache I/O errors (#3273).
+      SCCACHE_CACHE_RATE_LIMIT_ON_ERROR_COUNT: "5"
+      SCCACHE_CACHE_RATE_LIMIT_ON_ERROR_WINDOW_SIZE_SECS: "5"
     timeout-minutes: 60
     runs-on: linux-amd64-cpu8
     steps: