build: enable CUDA 12 development with utility fixes

.github/workflows/ci-nightly.yml

@@ -65,7 +65,7 @@ jobs:
 
       - name: Run CI tools unit tests
         run: |
-          python -m pip install pytest==9.1.0 pyyaml==6.0.3 "setuptools-scm[simple]>=8,!=10.1"
+          python -m pip install -e './ci[test]'
           python -m pytest -v ci/tools/tests
 
   find-wheels:

.github/workflows/ci-pixi-source-test.yml

@@ -12,8 +12,8 @@
 # only thing that runs the pixi source build end to end.
 #
 # Two tiers, to spend GPU minutes deliberately (GPUs are scarce):
-#   - build-smoke (PRs): CPU-only. Source-builds bindings + core, imports them,
-#     builds the cython test extensions and checks placement. Catches the
+#   - build-smoke (PRs): CPU-only. Source-builds both bindings roots and current
+#     Core, imports them, builds Cython extensions and checks placement. Catches the
 #     compile / ABI / .so-placement regressions WITHOUT a GPU.
 #   - build-identity-roundtrip (nightly + manual): CPU-only. cu13 -> cu12 ->
 #     cu13 in one checkout, so build artifacts cannot be reused across CUDA
@@ -42,19 +42,26 @@ on:
       # solve, so it can break the source build without a pixi.toml edit.
       - "**/pyproject.toml"
       - "cuda_bindings_13/build_hooks.py"
+      - "cuda_bindings_12/build_hooks.py"
       - "cuda_core/build_hooks.py"
       - "cuda_bindings_13/cuda/bindings/**" # generated bindings sources
+      - "cuda_bindings_12/cuda/bindings/**"
       - "cuda_bindings_13/tests/cython/**"
+      - "cuda_bindings_12/tests/**"
       - "cuda_core/tests/cython/**"
       - "ci/versions.yml"
+      - "ci/pyproject.toml"
+      - "ci/tools/bindings_config.py"
+      - "ci/tools/bindings_scm_version.py"
       - "ci/pixi-version.env"
       - ".github/actions/setup-pixi/action.yml"
       - ".github/workflows/ci-pixi-source-test.yml"
   workflow_dispatch:
     inputs:
       cuda-env:
         description: "pixi environment to test (cu13 or cu12)."
-        type: string
+        type: choice
+        options: ["cu13", "cu12"]
         default: "cu13"
 
 defaults:
@@ -73,12 +80,19 @@ env:
 jobs:
   # ── PR guard: CPU-only build + import + placement smoke ──
   build-smoke:
-    name: "build smoke (cu13, linux-64, CPU)"
+    name: "build smoke (${{ matrix.cuda-env }}, linux-64, CPU)"
     if: >-
       github.repository_owner == 'nvidia' &&
       (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch')
     runs-on: ubuntu-latest
     timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        cuda-env: ${{ fromJSON(github.event_name == 'workflow_dispatch' && format('["{0}"]', inputs.cuda-env) || '["cu12","cu13"]') }}
+    env:
+      CUDA_ENV: ${{ matrix.cuda-env }}
+      BINDINGS_ROOT: cuda_bindings_${{ matrix.cuda-env == 'cu12' && '12' || '13' }}
     steps:
       - name: Checkout ${{ github.event.repository.name }}
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -91,28 +105,46 @@ jobs:
           # third_party/cccl provides the CCCL headers cuda.core compiles against.
           submodules: true
 
+      - name: Set up Python for maintenance preview
+        if: ${{ matrix.cuda-env == 'cu12' }}
+        uses: ./.github/actions/setup-hardened-python
+        with:
+          python-version: "3.11"
+
+      - name: Bootstrap maintenance SCM in this disposable checkout
+        if: ${{ matrix.cuda-env == 'cu12' }}
+        env:
+          SOURCE_SHA: ${{ github.sha }}
+        run: |
+          python -m pip install -e ./ci
+          python -m ci.tools.bindings_scm_version --package-root cuda_bindings_12 --sha "$SOURCE_SHA"
+
       - name: Setup pixi
         uses: ./.github/actions/setup-pixi
 
       - name: Source-build + import + cython-placement smoke
-        env:
-          CUDA_ENV: ${{ inputs.cuda-env || 'cu13' }}
         run: |
           # pathfinder: pure-Python, no GPU.
           pixi run -e "${CUDA_ENV}" test-pathfinder
 
-          # bindings + core: force the source build (catches nvrtc/driver
+          # Force each bindings source build (catches nvrtc/driver
           # compile errors like #2182) and import them (catches ABI mismatches).
-          pixi run --manifest-path cuda_bindings_13 -e "${CUDA_ENV}" \
+          pixi run --manifest-path "${BINDINGS_ROOT}" -e "${CUDA_ENV}" \
             python -c "import cuda.bindings.driver, cuda.bindings.nvrtc, cuda.bindings.runtime; print('bindings import OK')"
-          pixi run --manifest-path cuda_core -e "${CUDA_ENV}" \
-            python -c "import cuda.core; print('core import OK')"
+          if [[ "${CUDA_ENV}" == cu13 ]]; then
+            pixi run --manifest-path cuda_core -e "${CUDA_ENV}" \
+              python -c "import cuda.core; print('core import OK')"
+          fi
 
           # cython test extensions: build them and confirm each .so landed next
           # to its .pyx in tests/cython (catches the placement regression #2180).
-          pixi run --manifest-path cuda_bindings_13 -e "${CUDA_ENV}" build-cython-tests
-          pixi run --manifest-path cuda_core -e "${CUDA_ENV}" build-cython-tests
-          for d in cuda_bindings_13/tests/cython cuda_core/tests/cython; do
+          pixi run --manifest-path "${BINDINGS_ROOT}" -e "${CUDA_ENV}" build-cython-tests
+          cython_dirs=("${BINDINGS_ROOT}/tests/cython")
+          if [[ "${CUDA_ENV}" == cu13 ]]; then
+            pixi run --manifest-path cuda_core -e "${CUDA_ENV}" build-cython-tests
+            cython_dirs+=(cuda_core/tests/cython)
+          fi
+          for d in "${cython_dirs[@]}"; do
             if ! compgen -G "${d}/*.cpython-*.so" > /dev/null; then
               echo "::error::no compiled cython test .so in ${d} (placement regression)"
               exit 1
@@ -131,8 +163,7 @@ jobs:
   # The round trip (not just cu13 -> cu12) is what catches the second half:
   # coming back to cu13 must not silently reuse the cu12 extension.
   #
-  # cuda_core only: cuda-bindings cannot be source-built in its cu12
-  # environment at all, for reasons unrelated to stale artifacts.
+  # Core only: bindings majors have separate source roots and environments.
   build-identity-roundtrip:
     name: "cu13 -> cu12 -> cu13 round trip (linux-64, CPU)"
     if: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && github.repository_owner == 'nvidia' }}
@@ -186,10 +217,14 @@ jobs:
 
   # ── Nightly: full `pixi run test` on a GPU runner ──
   full-test:
-    name: "pixi run test (${{ inputs.cuda-env || 'cu13' }}, linux-64, GPU)"
+    name: "pixi run test (${{ matrix.cuda-env }}, linux-64, GPU)"
     if: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && github.repository_owner == 'nvidia' }}
     runs-on: "linux-amd64-gpu-l4-latest-1" # same label scheme as test-wheel-linux.yml
     timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        cuda-env: ${{ fromJSON(github.event_name == 'workflow_dispatch' && format('["{0}"]', inputs.cuda-env) || '["cu12","cu13"]') }}
     container:
       options: -u root --security-opt seccomp=unconfined --shm-size 16g
       image: ubuntu:24.04
@@ -218,12 +253,32 @@ jobs:
           # third_party/cccl provides the CCCL headers cuda.core compiles against.
           submodules: true
 
+      - name: Trust the checkout for container source builds
+        # checkout's temporary Git configuration does not reach later steps.
+        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
       - name: Setup proxy cache
         uses: nv-gha-runners/setup-proxy-cache@main
         continue-on-error: true
 
+      - name: Set up Python for maintenance preview
+        if: ${{ matrix.cuda-env == 'cu12' }}
+        uses: ./.github/actions/setup-hardened-python
+        with:
+          python-version: "3.11"
+
+      - name: Bootstrap maintenance SCM in this disposable checkout
+        if: ${{ matrix.cuda-env == 'cu12' }}
+        env:
+          SOURCE_SHA: ${{ github.sha }}
+        run: |
+          python -m pip install -e ./ci
+          python -m ci.tools.bindings_scm_version --package-root cuda_bindings_12 --sha "$SOURCE_SHA"
+
       - name: Setup pixi
         uses: ./.github/actions/setup-pixi
 
       - name: pixi run test
-        run: pixi run -e "${{ inputs.cuda-env || 'cu13' }}" test
+        env:
+          CUDA_ENV: ${{ matrix.cuda-env }}
+        run: pixi run -e "${CUDA_ENV}" test

.github/workflows/ci.yml

@@ -265,7 +265,7 @@ jobs:
 
       - name: Test CI tools
         run: |
-          python -m pip install pytest==9.1.0 pyyaml==6.0.3 "setuptools-scm[simple]>=8,!=10.1"
+          python -m pip install -e './ci[test]'
           python -m pytest -v ci/tools/tests
 
       - name: Compute CI workplan

.pre-commit-config.yaml

@@ -83,7 +83,7 @@ repos:
         entry: python -m ci.tools.check_pixi_cuda_version
         language: python
         additional_dependencies: [pyyaml==6.0.3]
-        files: '^(ci/versions\.yml|ci/tools/(bindings_config|check_pixi_cuda_version)\.py|cuda_bindings_13/pixi\.toml|cuda_core/pixi\.toml)$'
+        files: '^(ci/versions\.yml|ci/tools/(bindings_config|check_pixi_cuda_version)\.py|cuda_bindings_(?:12|13)/pixi\.toml|cuda_core/pixi\.toml)$'
         pass_filenames: false
 
       - id: check-mempool-hygiene
@@ -162,7 +162,7 @@ repos:
     rev: "3e8a8703264a2f4a69428a0aa4dcb512790b2c8c"  # frozen: v6.0.0
     hooks:
     - id: check-added-large-files
-      exclude: '^(?:cuda_bindings_13/cuda/bindings/(?:nvml\.pyx|_v2/driver\.pyx)|cuda_bindings_12/cuda/bindings/(?:driver\.pyx|runtime\.pyx\.in|nvml\.pyx))$'
+      exclude: '^(?:cuda_bindings_13/cuda/bindings/(?:nvml\.pyx|_v2/driver\.pyx)|cuda_bindings_12/cuda/bindings/(?:driver\.pyx|runtime\.pyx\.in|nvml\.pyx)|cuda_bindings_(?:12|13)/pixi\.lock)$'
     - id: check-case-conflict
     - id: check-docstring-first
     - id: check-merge-conflict

AGENTS.md

@@ -8,10 +8,33 @@ guide for package-specific conventions and workflows.
 ## Package map
 
 - `cuda_pathfinder/`: Pure-Python library discovery and loading utilities.
-- `cuda_bindings_13/`: Low-level CUDA host API bindings (Cython-heavy).
+- `cuda_bindings_12/`: CUDA 12.9 maintenance bindings (Cython-heavy).
+- `cuda_bindings_13/`: Current CUDA 13 bindings (Cython-heavy).
 - `cuda_core/`: High-level Pythonic CUDA APIs built on top of bindings.
 - `cuda_python/`: Metapackage and docs aggregation.
 
+## Changes across bindings majors
+
+Both `cuda_bindings_12/` and `cuda_bindings_13/` are maintained on `main`.
+When implementing or reviewing a change in either root, inspect the other
+root's corresponding implementation, tests, and relevant build, packaging,
+or documentation files, including files absent from the diff. Check shared
+instructions and configuration affected by the change too. Apply this check
+in both directions, and when changing shared build helpers, generation
+inputs, CI selection, or release routing.
+
+- Make matching changes where the affected behavior applies to both roots.
+  Preserve intentional differences in supported APIs, toolkit inputs, source
+  layout, and compatibility behavior. Do not copy changes mechanically or add an unsupported API to make the roots match.
+- Follow each package's generation instructions: change the generation source
+  and regenerate each affected root against its own inputs. Do not hand-edit
+  generated output.
+- In the implementation or review summary, name the counterpart files inspected
+  and identify the matching changes or explain concretely why none are needed.
+  Reviewers must verify that assessment and flag omitted applicable changes;
+  absence from the diff is not evidence that the other root is unaffected.
+- Preserve byte-for-byte invariants for package-local copies of shared helpers.
+
 # Git-derived version lifecycle
 
 Package versions are derived from reachable Git tags by `setuptools-scm`.
@@ -36,6 +59,10 @@ For each package tag namespace and target release `X.Y.Z`:
   existing history.
 - Treat pushed tags as immutable. Correct an unsuitable tagged state with a
   new versioned milestone rather than moving or replacing the existing tag.
+- Before setting or changing `scm-bootstrap-release`, check canonical tags
+  and coordinate with release maintainers. Use an unused release number,
+  including releases tagged on other branches; keep the same target across
+  builds until release coordination requires a change.
 
 Use the package's established tag namespace, such as `vX.Y.Z...` for
 `cuda-bindings` and `cuda-python`, `cuda-core-vX.Y.Z...` for `cuda-core`, and

README.md

@@ -48,6 +48,14 @@ The list of available interfaces is:
 * cuFile
 * NVML
 
+## Bindings source roots
+
+Current CUDA 13 bindings live in [`cuda_bindings_13/`](cuda_bindings_13/), and
+CUDA 12 bindings sources have a separate
+[developer environment](cuda_bindings_12/README.md). Both roots build the same
+`cuda-bindings` distribution and `cuda.bindings` namespace; develop and test
+them in separate environments.
+
 ## License
 
 CUDA Python is licensed under the [Apache License 2.0](./LICENSE). Third-party
@@ -59,7 +67,8 @@ The root `LICENSE` governs the repository as a whole:
 
 | Subproject       | License    | License file                                             |
 | ---------------- | ---------- | -------------------------------------------------------- |
-| `cuda.bindings`  | Apache-2.0 | [`cuda_bindings_13/LICENSE`](./cuda_bindings_13/LICENSE)       |
+| `cuda.bindings` (CUDA 13) | Apache-2.0 | [`cuda_bindings_13/LICENSE`](./cuda_bindings_13/LICENSE) |
+| `cuda.bindings` (CUDA 12) | Apache-2.0 | [`cuda_bindings_12/LICENSE`](./cuda_bindings_12/LICENSE) |
 | `cuda.core`      | Apache-2.0 | [`cuda_core/LICENSE`](./cuda_core/LICENSE)               |
 | `cuda.pathfinder`| Apache-2.0 | [`cuda_pathfinder/LICENSE`](./cuda_pathfinder/LICENSE)   |
 | `cuda-python`    | Apache-2.0 | [`cuda_python/LICENSE`](./cuda_python/LICENSE)           |

ci/pyproject.toml

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+[build-system]
+requires = ["setuptools>=77.0.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "cuda-python-ci-tools"
+version = "0.0.0"
+description = "Private CI support for the CUDA Python repository"
+authors = [{ name = "NVIDIA Corporation" }]
+license = "Apache-2.0"
+requires-python = ">=3.11"
+dependencies = [
+  "packaging==26.0",
+  "PyYAML==6.0.3",
+  "setuptools-scm[simple]>=8,!=10.1",
+]
+
+[project.optional-dependencies]
+test = ["pytest==9.1.0", "wheel"]
+
+[tool.setuptools]
+packages = ["ci", "ci.tools"]
+package-dir = { ci = "." }

ci/tools/bindings_config.py

@@ -5,6 +5,9 @@
 
 from __future__ import annotations
 
+import argparse
+import json
+import sys
 from dataclasses import dataclass
 from pathlib import Path
 
@@ -35,10 +38,21 @@ def cuda_major(self) -> str:
     def cuda_variant(self) -> str:
         return f"cu{self.cuda_major}"
 
+    def to_dict(self) -> dict[str, str]:
+        return {
+            "package_root": self.package_root,
+            "toolkit_version": self.toolkit_version,
+            "release_status": self.release_status,
+            "ctk_target": self.ctk_target,
+            "cuda_major": self.cuda_major,
+            "cuda_variant": self.cuda_variant,
+        }
+
 
 @dataclass(frozen=True)
 class BindingsConfig:
     package_roots: tuple[BindingsPackage, ...]
+    python_supported: tuple[str, ...]
 
     def get_package(self, package_root: str) -> BindingsPackage:
         for package in self.package_roots:
@@ -57,4 +71,33 @@ def load_config(path: Path = DEFAULT_CONFIG) -> BindingsConfig:
     for field, release_status in (("build", "current"), ("prev_build", "maintenance")):
         major, minor, patch = map(int, raw["cuda"][field]["version"].split("."))
         packages.append(BindingsPackage(f"cuda_bindings_{major}", (major, minor, patch), release_status))
-    return BindingsConfig(tuple(packages))
+    return BindingsConfig(tuple(packages), tuple(raw["python"]["supported"]))
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG)
+    output = parser.add_mutually_exclusive_group()
+    output.add_argument("--package-roots", action="store_true")
+    output.add_argument("--release-status", choices=("current", "maintenance"))
+    args = parser.parse_args(argv)
+
+    config = load_config(args.config)
+    if args.release_status:
+        result = config.package_for_release_status(args.release_status).to_dict()
+    else:
+        packages = [package.to_dict() for package in config.package_roots]
+        result = (
+            packages
+            if args.package_roots
+            else {
+                "python": {"supported": list(config.python_supported)},
+                "package_roots": packages,
+            }
+        )
+    print(json.dumps(result, separators=(",", ":"), sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())