CTK 13.4 prerelease integration

.github/actions/fetch_ctk/action.yml

@@ -11,6 +11,10 @@ inputs:
     required: true
   cuda-version:
     required: true
+  cuda-channel:
+    description: "CUDA package channel: stable redistributables or prerelease packages"
+    required: false
+    default: "stable"
   cuda-components:
     description: "A list of the CTK components to install as a comma-separated list. e.g. 'cuda_nvcc,cuda_nvrtc,cuda_cudart'"
     required: false
@@ -30,19 +34,52 @@ runs:
         # Use the runtime workspace mount so this also works inside container jobs.
         CTK_REDIST_TOOL="${GITHUB_WORKSPACE}/ci/tools/fetch_ctk_redistrib.py"
         CTK_CACHE_COMPONENTS=${{ inputs.cuda-components }}
-        CTK_JSON_URL="https://developer.download.nvidia.com/compute/cuda/redist/redistrib_${{ inputs.cuda-version }}.json"
-        CTK_CACHE_COMPONENTS="$(python "$CTK_REDIST_TOOL" filter-components \
-          --host-platform "${{ inputs.host-platform }}" \
-          --cuda-version "${{ inputs.cuda-version }}" \
-          --components "$CTK_CACHE_COMPONENTS" \
-          --metadata-url "$CTK_JSON_URL")"
+        CTK_PREVIEW_PACKAGES=
+        CTK_PREVIEW_INSTALLER_URL=
+        CTK_PREVIEW_INSTALLER_SHA256=
+        if [[ "${{ inputs.cuda-channel }}" == "stable" ]]; then
+          CTK_JSON_URL="https://developer.download.nvidia.com/compute/cuda/redist/redistrib_${{ inputs.cuda-version }}.json"
+          CTK_CACHE_COMPONENTS="$(python "$CTK_REDIST_TOOL" filter-components \
+            --host-platform "${{ inputs.host-platform }}" \
+            --cuda-version "${{ inputs.cuda-version }}" \
+            --components "$CTK_CACHE_COMPONENTS" \
+            --metadata-url "$CTK_JSON_URL")"
+        elif [[ "${{ inputs.cuda-channel }}" == "prerelease" ]]; then
+          if [[ "${{ inputs.host-platform }}" == linux* ]]; then
+            CTK_PREVIEW_PACKAGES="$(python "$CTK_REDIST_TOOL" preview-packages \
+              --host-platform "${{ inputs.host-platform }}" \
+              --cuda-version "${{ inputs.cuda-version }}" \
+              --components "$CTK_CACHE_COMPONENTS")"
+            CTK_CACHE_COMPONENTS="$CTK_PREVIEW_PACKAGES"
+          elif [[ "${{ inputs.host-platform }}" == win* ]]; then
+            IFS=$'\t' read -r CTK_PREVIEW_INSTALLER_URL CTK_PREVIEW_INSTALLER_SHA256 <<< \
+              "$(python "$CTK_REDIST_TOOL" preview-installer \
+                --host-platform "${{ inputs.host-platform }}" \
+                --cuda-version "${{ inputs.cuda-version }}")"
+            CTK_CACHE_COMPONENTS="${CTK_PREVIEW_INSTALLER_URL}:${CTK_PREVIEW_INSTALLER_SHA256}:${CTK_CACHE_COMPONENTS}"
+          else
+            echo "CUDA prerelease packages are not supported for host-platform ${{ inputs.host-platform }}" >&2
+            exit 1
+          fi
+        else
+          echo "Unsupported CUDA package channel: ${{ inputs.cuda-channel }}" >&2
+          exit 1
+        fi
 
         HASH=$(echo -n "${CTK_CACHE_COMPONENTS}" | sha256sum | awk '{print $1}')
-        echo "CTK_CACHE_KEY=mini-ctk-${{ inputs.cuda-version }}-${{ inputs.host-platform }}-$HASH" >> $GITHUB_ENV
+        CHANNEL_CACHE_SEGMENT=
+        if [[ "${{ inputs.cuda-channel }}" != "stable" ]]; then
+          CHANNEL_CACHE_SEGMENT="-${{ inputs.cuda-channel }}"
+        fi
+        echo "CTK_CACHE_KEY=mini-ctk${CHANNEL_CACHE_SEGMENT}-${{ inputs.cuda-version }}-${{ inputs.host-platform }}-$HASH" >> $GITHUB_ENV
         echo "CTK_CACHE_FILENAME=mini-ctk-${{ inputs.cuda-version }}-${{ inputs.host-platform }}-$HASH.tar.gz" >> $GITHUB_ENV
         echo "CTK_CACHE_COMPONENTS=${CTK_CACHE_COMPONENTS}" >> $GITHUB_ENV
+        echo "CTK_PREVIEW_PACKAGES=${CTK_PREVIEW_PACKAGES}" >> $GITHUB_ENV
+        echo "CTK_PREVIEW_INSTALLER_URL=${CTK_PREVIEW_INSTALLER_URL}" >> $GITHUB_ENV
+        echo "CTK_PREVIEW_INSTALLER_SHA256=${CTK_PREVIEW_INSTALLER_SHA256}" >> $GITHUB_ENV
 
     - name: Install dependencies
+      if: ${{ startsWith(inputs.host-platform, 'linux') }}
       uses: ./.github/actions/install_unix_deps
       continue-on-error: false
       with:
@@ -65,51 +102,135 @@ runs:
         # Everything under this folder is packed and stored in the GitHub Cache space,
         # and unpacked after retrieving from the cache.
         CACHE_TMP_DIR="./cache_tmp_dir"
-        rm -rf $CACHE_TMP_DIR
+        WORK_TMP_DIR="./cache_work_dir"
+        rm -rf $CACHE_TMP_DIR $WORK_TMP_DIR
         mkdir $CACHE_TMP_DIR
-
-        # The binary archives (redist) are guaranteed to be updated as part of the release posting.
-        # Use the runtime workspace mount so this also works inside container jobs.
         CTK_REDIST_TOOL="${GITHUB_WORKSPACE}/ci/tools/fetch_ctk_redistrib.py"
-        CTK_BASE_URL="https://developer.download.nvidia.com/compute/cuda/redist/"
-        CTK_JSON_URL="$CTK_BASE_URL/redistrib_${{ inputs.cuda-version }}.json"
-        CTK_JSON_FILE="$CACHE_TMP_DIR/redistrib.json"
-        curl -LSs "$CTK_JSON_URL" -o "$CTK_JSON_FILE"
-        if [[ "${{ inputs.host-platform }}" == linux* ]]; then
-          function extract() {
-            tar -xvf $1 -C $CACHE_TMP_DIR --strip-components=1
-          }
-        elif [[ "${{ inputs.host-platform }}" == "win-64" ]]; then
-          function extract() {
-            _TEMP_DIR_=$(mktemp -d)
-            unzip $1 -d $_TEMP_DIR_
-            cp -r $_TEMP_DIR_/*/* $CACHE_TMP_DIR
-            rm -rf $_TEMP_DIR_
-            # see commit NVIDIA/cuda-python@69410f1d9228e775845ef6c8b4a9c7f37ffc68a5
-            chmod 644 $CACHE_TMP_DIR/LICENSE
+
+        if [[ "${{ inputs.cuda-channel }}" == "prerelease" ]]; then
+          if [[ "${{ inputs.host-platform }}" == linux* ]]; then
+            source /etc/os-release
+            DISTRO_CODENAME="${VERSION_CODENAME:-}"
+            case "$DISTRO_CODENAME" in
+              bookworm|jammy|noble|resolute|trixie) ;;
+              *)
+                echo "Unsupported distribution for CUDA prerelease packages: ${DISTRO_CODENAME:-unknown}" >&2
+                exit 1
+                ;;
+            esac
+
+            KEYRING_DEB="$CACHE_TMP_DIR/nvidia-preview-keyring.deb"
+            curl -fLSs "https://packages.nvidia.com/${DISTRO_CODENAME}/nvidia-preview-keyring.deb" -o "$KEYRING_DEB"
+            sudo dpkg -i "$KEYRING_DEB"
+            sudo apt-get update
+
+            DEB_DIR="$CACHE_TMP_DIR/debs"
+            DEB_ROOT="$CACHE_TMP_DIR/deb-root"
+            mkdir -p "$DEB_DIR/partial" "$DEB_ROOT"
+            DEB_DIR="$(realpath "$DEB_DIR")"
+            IFS=, read -ra PREVIEW_PACKAGES <<< "$CTK_PREVIEW_PACKAGES"
+            sudo apt-get install --yes --download-only --no-install-recommends \
+              -o "Dir::Cache::archives=$DEB_DIR" \
+              "${PREVIEW_PACKAGES[@]}"
+            for package in "$DEB_DIR"/*.deb; do
+              dpkg-deb -x "$package" "$DEB_ROOT"
+            done
+
+            CUDA_SHORT_VERSION="${{ inputs.cuda-version }}"
+            CUDA_SHORT_VERSION="${CUDA_SHORT_VERSION%.*}"
+            CUDA_PACKAGE_ROOT="$DEB_ROOT/usr/local/cuda-${CUDA_SHORT_VERSION}"
+            if [[ ! -d "$CUDA_PACKAGE_ROOT/include" ]]; then
+              echo "CUDA prerelease packages did not provide $CUDA_PACKAGE_ROOT/include" >&2
+              exit 1
+            fi
+            cp -a "$CUDA_PACKAGE_ROOT/." "$CACHE_TMP_DIR/"
+            rm -rf "$DEB_DIR" "$DEB_ROOT" "$KEYRING_DEB"
+          elif [[ "${{ inputs.host-platform }}" == win* ]]; then
+            WORK_TMP_DIR="./cache_work_dir"
+            INSTALLER_PATH="$WORK_TMP_DIR/$(basename "$CTK_PREVIEW_INSTALLER_URL")"
+            EXTRACT_ROOT="$WORK_TMP_DIR/installer-root"
+            mkdir -p "$WORK_TMP_DIR"
+            curl -fLSs "$CTK_PREVIEW_INSTALLER_URL" -o "$INSTALLER_PATH"
+            echo "$CTK_PREVIEW_INSTALLER_SHA256  $INSTALLER_PATH" | sha256sum --check --strict -
+
+            SEVEN_ZIP="$(command -v 7z || true)"
+            if [[ -z "$SEVEN_ZIP" && -x "/c/Program Files/7-Zip/7z.exe" ]]; then
+              SEVEN_ZIP="/c/Program Files/7-Zip/7z.exe"
+            fi
+            if [[ -z "$SEVEN_ZIP" || ! -x "$SEVEN_ZIP" ]]; then
+              echo "7-Zip is required to extract the CUDA prerelease installer" >&2
+              exit 1
+            fi
+
+            IFS=, read -ra PREVIEW_WINDOWS_ARCHIVES <<< \
+              "$(python "$CTK_REDIST_TOOL" preview-windows-archives \
+                --host-platform "${{ inputs.host-platform }}" \
+                --cuda-version "${{ inputs.cuda-version }}" \
+                --components "${{ inputs.cuda-components }}")"
+
+            PREVIEW_WINDOWS_ARCHIVE_PATTERNS=()
+            for archive_dir in "${PREVIEW_WINDOWS_ARCHIVES[@]}"; do
+              PREVIEW_WINDOWS_ARCHIVE_PATTERNS+=("${archive_dir}/*")
+            done
+
+            mkdir -p "$EXTRACT_ROOT"
+            "$SEVEN_ZIP" x -y "$INSTALLER_PATH" "${PREVIEW_WINDOWS_ARCHIVE_PATTERNS[@]}" "-o$EXTRACT_ROOT"
+
+            python "$CTK_REDIST_TOOL" merge-windows-preview \
+              --host-platform "${{ inputs.host-platform }}" \
+              --cuda-version "${{ inputs.cuda-version }}" \
+              --components "${{ inputs.cuda-components }}" \
+              --extract-root "$EXTRACT_ROOT" \
+              --destination "$CACHE_TMP_DIR"
+
+            rm -rf "$WORK_TMP_DIR"
+          else
+            echo "CUDA prerelease extraction is not supported for host-platform ${{ inputs.host-platform }}" >&2
+            exit 1
+          fi
+        else
+          # The binary archives (redist) are guaranteed to be updated as part of the release posting.
+          # Use the runtime workspace mount so this also works inside container jobs.
+          CTK_BASE_URL="https://developer.download.nvidia.com/compute/cuda/redist/"
+          CTK_JSON_URL="$CTK_BASE_URL/redistrib_${{ inputs.cuda-version }}.json"
+          CTK_JSON_FILE="$CACHE_TMP_DIR/redistrib.json"
+          curl -fLSs "$CTK_JSON_URL" -o "$CTK_JSON_FILE"
+          if [[ "${{ inputs.host-platform }}" == linux* ]]; then
+            function extract() {
+              tar -xvf $1 -C $CACHE_TMP_DIR --strip-components=1
+            }
+          elif [[ "${{ inputs.host-platform }}" == win* ]]; then
+            function extract() {
+              _TEMP_DIR_=$(mktemp -d)
+              unzip $1 -d $_TEMP_DIR_
+              cp -r $_TEMP_DIR_/*/* $CACHE_TMP_DIR
+              rm -rf $_TEMP_DIR_
+              # see commit NVIDIA/cuda-python@69410f1d9228e775845ef6c8b4a9c7f37ffc68a5
+              chmod 644 $CACHE_TMP_DIR/LICENSE
+            }
+          fi
+          function populate_cuda_path() {
+            # take the component name as a argument
+            function download() {
+              curl -fLSs $1 -o $2
+            }
+            CTK_COMPONENT=$1
+            CTK_COMPONENT_REL_PATH="$(python "$CTK_REDIST_TOOL" component-relative-path \
+              --host-platform "${{ inputs.host-platform }}" \
+              --component "$CTK_COMPONENT" \
+              --metadata-path "$CTK_JSON_FILE")"
+            CTK_COMPONENT_URL="${CTK_BASE_URL}/${CTK_COMPONENT_REL_PATH}"
+            CTK_COMPONENT_COMPONENT_FILENAME="$(basename $CTK_COMPONENT_REL_PATH)"
+            download $CTK_COMPONENT_URL $CTK_COMPONENT_COMPONENT_FILENAME
+            extract $CTK_COMPONENT_COMPONENT_FILENAME
+            rm $CTK_COMPONENT_COMPONENT_FILENAME
           }
+
+          # Get headers and shared libraries in place
+          for item in $(echo $CTK_CACHE_COMPONENTS | tr ',' ' '); do
+              populate_cuda_path "$item"
+          done
         fi
-        function populate_cuda_path() {
-          # take the component name as a argument
-          function download() {
-            curl -LSs $1 -o $2
-          }
-          CTK_COMPONENT=$1
-          CTK_COMPONENT_REL_PATH="$(python "$CTK_REDIST_TOOL" component-relative-path \
-            --host-platform "${{ inputs.host-platform }}" \
-            --component "$CTK_COMPONENT" \
-            --metadata-path "$CTK_JSON_FILE")"
-          CTK_COMPONENT_URL="${CTK_BASE_URL}/${CTK_COMPONENT_REL_PATH}"
-          CTK_COMPONENT_COMPONENT_FILENAME="$(basename $CTK_COMPONENT_REL_PATH)"
-          download $CTK_COMPONENT_URL $CTK_COMPONENT_COMPONENT_FILENAME
-          extract $CTK_COMPONENT_COMPONENT_FILENAME
-          rm $CTK_COMPONENT_COMPONENT_FILENAME
-        }
-
-        # Get headers and shared libraries in place
-        for item in $(echo $CTK_CACHE_COMPONENTS | tr ',' ' '); do
-            populate_cuda_path "$item"
-        done
         # TODO: check Windows
         if [[ "${{ inputs.host-platform }}" == linux* && -d "${CACHE_TMP_DIR}/lib" ]]; then
           mv $CACHE_TMP_DIR/lib $CACHE_TMP_DIR/lib64
@@ -120,8 +241,12 @@ runs:
         # Note: try to escape | and > ...
         tar -czvf ${CTK_CACHE_FILENAME} ${CACHE_TMP_DIR}
 
-        # "Move" files from temp dir to CUDA_PATH
-        CUDA_PATH="./cuda_toolkit"
+        # Populate the final CUDA_PATH directly (never stage through ./cuda_toolkit
+        # unconditionally — a second call with a different cuda-path would leave
+        # symlinks from a prerelease CTK at ./cuda_toolkit/include etc., causing
+        # "cp: cannot overwrite non-directory" failures on the next restore).
+        CUDA_PATH="${{ inputs.cuda-path }}"
+        rm -rf $CUDA_PATH
         mkdir -p $CUDA_PATH
         # Unfortunately we cannot use "rsync -av $CACHE_TMP_DIR/ $CUDA_PATH" because
         # not all runners have rsync pre-installed (or even installable, such as
@@ -144,23 +269,23 @@ runs:
       run: |
         ls -l
         CACHE_TMP_DIR="./cache_tmp_dir"
-        CUDA_PATH="./cuda_toolkit"
+        CUDA_PATH="${{ inputs.cuda-path }}"
+        rm -rf $CUDA_PATH
         mkdir -p $CUDA_PATH
         tar -xzvf $CTK_CACHE_FILENAME
         # Can't use rsync here, see above
         cp -r $CACHE_TMP_DIR/* $CUDA_PATH
         rm -rf $CACHE_TMP_DIR $CTK_CACHE_FILENAME
         ls -l $CUDA_PATH
-        if [ ! -d "$CUDA_PATH/include" ]; then
+        # redistrib.json is present for stable-channel installs; include/ is
+        # present for prerelease installs (enforced at cache-creation time).
+        # Components that don't ship headers (e.g. cuda_sanitizer_api) have
+        # neither, so checking only for include/ incorrectly rejects them.
+        if [[ ! -e "$CUDA_PATH/redistrib.json" && ! -d "$CUDA_PATH/include" ]]; then
+          echo "CTK restore appears incomplete: neither redistrib.json nor include/ found in $CUDA_PATH" >&2
           exit 1
         fi
 
-    - name: Move CTK to the specified location
-      if: ${{ inputs.cuda-path != './cuda_toolkit' }}
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-        mv ./cuda_toolkit ${{ inputs.cuda-path }}
-
     - name: Set output environment variables
       shell: bash --noprofile --norc -xeuo pipefail {0}
       run: |

.github/workflows/build-docs.yml

@@ -64,8 +64,10 @@ jobs:
         run: |
           if [[ -f ci/versions.yml ]]; then
             BUILD_CTK_VER=$(yq '.cuda.build.version' ci/versions.yml)
+            BUILD_CTK_CHANNEL=$(yq '.cuda.build.channel // "stable"' ci/versions.yml)
           elif [[ -f ci/versions.json ]]; then
             BUILD_CTK_VER=$(jq -r '.cuda.build.version' ci/versions.json)
+            BUILD_CTK_CHANNEL=$(jq -r '.cuda.build.channel // "stable"' ci/versions.json)
           else
             echo "error: cannot find ci/versions.yml or ci/versions.json" >&2
             exit 1
@@ -75,6 +77,7 @@ jobs:
             exit 1
           fi
           echo "BUILD_CTK_VER=${BUILD_CTK_VER}" >> "$GITHUB_ENV"
+          echo "BUILD_CTK_CHANNEL=${BUILD_CTK_CHANNEL}" >> "$GITHUB_ENV"
 
       # TODO: This workflow runs on GH-hosted runner and cannot use the proxy cache
 
@@ -100,6 +103,7 @@ jobs:
         with:
           host-platform: linux-64
           cuda-version: ${{ env.BUILD_CTK_VER }}
+          cuda-channel: ${{ env.BUILD_CTK_CHANNEL }}
 
       - name: Set environment variables
         run: |