ci: build cuda.core against the cuda-bindings just built #270
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | |
| # | |
| # SPDX-License-Identifier: Apache-2.0 | |
| name: "CI: Coverage" | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' # This runs the workflow every day at 12:00 AM UTC | |
| workflow_dispatch: {} | |
| # TEMPORARY (do not merge): runs the Windows coverage job against a PR, via | |
| # the copy-pr-bot mirror branch. Same trigger as ci.yml. | |
| push: | |
| branches: | |
| - "pull-request/[0-9]+" | |
| # One coverage run per ref: the Windows leg holds an A100 runner for the whole | |
| # job, so superseded runs must not queue up behind each other. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} | |
| cancel-in-progress: true | |
| env: | |
| PY_VER: "3.14" | |
| LOCAL_CTK: "1" | |
| GPU: "a100" | |
| DRIVER: "latest" | |
| jobs: | |
| coverage-vars: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| CUDA_VER: ${{ steps.get-vars.outputs.cuda_ver }} | |
| steps: | |
| - name: Checkout ${{ github.event.repository.name }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Get CUDA version | |
| id: get-vars | |
| run: | | |
| cuda_ver=$(yq '.cuda.build.version' ci/versions.yml) | |
| echo "cuda_ver=$cuda_ver" >> $GITHUB_OUTPUT | |
| coverage-linux: | |
| name: Coverage (Linux) | |
| needs: [coverage-vars] | |
| # Only Windows crashes, so a PR run skips the Linux leg entirely. | |
| if: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }} | |
| runs-on: "linux-amd64-gpu-a100-latest-1" | |
| permissions: | |
| id-token: write | |
| contents: write | |
| defaults: | |
| run: | |
| shell: bash --noprofile --norc -xeuo pipefail {0} | |
| env: | |
| HOST_PLATFORM: "linux-64" | |
| ARCH: "x86_64" | |
| CUDA_VER: ${{ needs.coverage-vars.outputs.CUDA_VER }} | |
| # Our self-hosted runners require a container | |
| # TODO: use a different (nvidia?) container | |
| container: | |
| options: -u root --security-opt seccomp=unconfined --shm-size 16g | |
| image: ubuntu:24.04 | |
| env: | |
| NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} | |
| PIP_CACHE_DIR: "/tmp/pip-cache" | |
| steps: | |
| - name: Ensure GPU is working | |
| run: nvidia-smi | |
| # We have to install git before checking out the repo (so that we can | |
| # deploy the docs at the end). This means we can't use install_unix_deps | |
| # action so install the git package. | |
| - name: Install git | |
| run: | | |
| apt-get update | |
| apt-get install -y git | |
| - name: Checkout ${{ github.event.repository.name }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| # Treeless clone: setuptools-scm needs the commit graph (`git describe`) | |
| # but not historical blobs. | |
| fetch-depth: 0 | |
| filter: blob:none | |
| - name: Fix workspace ownership | |
| run: | | |
| chown -R $(id -u):$(id -g) "$GITHUB_WORKSPACE" | |
| - name: Install dependencies | |
| uses: ./.github/actions/install_unix_deps | |
| continue-on-error: false | |
| with: | |
| dependencies: "tree rsync libsqlite3-0 g++ jq wget libgl1 libegl1" | |
| dependent_exes: "tree rsync libsqlite3-0 g++ jq wget libgl1 libegl1" | |
| - name: Setup proxy cache | |
| uses: nv-gha-runners/setup-proxy-cache@main | |
| continue-on-error: true | |
| - name: Set environment variables | |
| env: | |
| BUILD_CUDA_VER: ${{ env.CUDA_VER }} | |
| CUDA_VER: ${{ env.CUDA_VER }} | |
| HOST_PLATFORM: ${{ env.HOST_PLATFORM }} | |
| LOCAL_CTK: ${{ env.LOCAL_CTK }} | |
| PY_VER: ${{ env.PY_VER }} | |
| SHA: ${{ github.sha }} | |
| run: | | |
| ./ci/tools/env-vars test | |
| echo "CUDA_PYTHON_COVERAGE=1" >> $GITHUB_ENV | |
| - name: Set up Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version: ${{ env.PY_VER }} | |
| env: | |
| # we use self-hosted runners on which setup-python behaves weirdly... | |
| AGENT_TOOLSDIRECTORY: "/opt/hostedtoolcache" | |
| - name: Set up mini CTK | |
| if: ${{ env.LOCAL_CTK == '1' }} | |
| uses: ./.github/actions/fetch_ctk | |
| continue-on-error: false | |
| with: | |
| host-platform: ${{ env.HOST_PLATFORM }} | |
| cuda-version: ${{ env.CUDA_VER }} | |
| - name: Create venv | |
| run: | | |
| python -m venv .venv | |
| - name: Build cuda-pathfinder | |
| run: | | |
| cd cuda_pathfinder | |
| ../.venv/bin/pip install -v . --group test | |
| - name: Build cuda-bindings | |
| run: | | |
| cd cuda_bindings | |
| ../.venv/bin/pip install -v . --group test | |
| - name: Build cuda-core | |
| run: | | |
| cd cuda_core | |
| ../.venv/bin/pip install -v . --group test | |
| - name: Install coverage tools | |
| run: | | |
| .venv/bin/pip install coverage pytest-cov Cython | |
| - name: Set cuda package install root | |
| run: | | |
| echo "INSTALL_ROOT=$(.venv/bin/python -c 'import cuda; import os; print(os.path.dirname(cuda.__path__[0]))')" >> $GITHUB_ENV | |
| echo "REPO_ROOT=$(pwd)" >> $GITHUB_ENV | |
| - name: Run cuda.pathfinder tests | |
| continue-on-error: true | |
| run: | | |
| cd $INSTALL_ROOT | |
| $REPO_ROOT/.venv/bin/pytest -v --cov=./cuda --cov-append --cov-context=test --cov-config=$REPO_ROOT/.coveragerc $REPO_ROOT/cuda_pathfinder/tests | |
| - name: Run cuda.bindings tests | |
| continue-on-error: true | |
| run: | | |
| cd $INSTALL_ROOT | |
| $REPO_ROOT/.venv/bin/pytest -v --cov=./cuda --cov-append --cov-context=test --cov-config=$REPO_ROOT/.coveragerc $REPO_ROOT/cuda_bindings/tests | |
| - name: Run cuda.core tests | |
| continue-on-error: true | |
| run: | | |
| cd $INSTALL_ROOT | |
| $REPO_ROOT/.venv/bin/pytest -v --cov=./cuda --cov-append --cov-context=test --cov-config=$REPO_ROOT/.coveragerc $REPO_ROOT/cuda_core/tests | |
| - name: Copy Linux coverage file to workspace | |
| run: | | |
| cp $INSTALL_ROOT/.coverage $REPO_ROOT/.coverage.linux | |
| echo "Copied .coverage.linux to $REPO_ROOT" | |
| ls -lh $REPO_ROOT/.coverage.linux | |
| - name: Upload Linux coverage data | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: coverage-data-linux | |
| path: .coverage.linux | |
| retention-days: 7 | |
| include-hidden-files: true | |
| if-no-files-found: error | |
| - name: Upload cuda source code for coverage mapping | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: cuda-source-linux | |
| path: ${{ env.INSTALL_ROOT }}/cuda/ | |
| retention-days: 7 | |
| if-no-files-found: error | |
| # Build Windows wheels on GitHub-hosted runner (has VS, no GPU) | |
| build-wheel-windows: | |
| name: Build Wheels (Windows) | |
| needs: [coverage-vars] | |
| runs-on: windows-2022 | |
| permissions: | |
| contents: read | |
| defaults: | |
| run: | |
| shell: bash --noprofile --norc -xeuo pipefail {0} | |
| env: | |
| HOST_PLATFORM: "win-64" | |
| CUDA_PYTHON_COVERAGE: "1" | |
| CUDA_VER: ${{ needs.coverage-vars.outputs.CUDA_VER }} | |
| steps: | |
| - name: Checkout ${{ github.event.repository.name }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| # Treeless clone: setuptools-scm needs the commit graph (`git describe`) | |
| # but not historical blobs. | |
| fetch-depth: 0 | |
| filter: blob:none | |
| - name: Set up Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version: ${{ env.PY_VER }} | |
| - name: Set up MSVC | |
| uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4 # v1.13.1 | |
| - name: Set up mini CTK | |
| uses: ./.github/actions/fetch_ctk | |
| continue-on-error: false | |
| with: | |
| host-platform: win-64 | |
| cuda-version: ${{ env.CUDA_VER }} | |
| - name: Create venv | |
| run: | | |
| python -m venv .venv | |
| - name: Build and install cuda.pathfinder | |
| run: | | |
| .venv/Scripts/pip install wheel setuptools Cython | |
| .venv/Scripts/pip wheel -v --no-deps ./cuda_pathfinder -w ./wheels/ | |
| - name: Build cuda.bindings wheel | |
| run: | | |
| cd cuda_bindings | |
| ../.venv/Scripts/pip wheel -v --no-deps . -w ../wheels/ | |
| # TEMPORARY (do not merge): build against the cuda-bindings wheel built a | |
| # step ago, not whatever PyPI offers. | |
| # | |
| # PIP_PRE is here so pip will consider the locally built wheel at all -- | |
| # it carries a .devN version, which is a pre-release. The side effect is | |
| # that PyPI's pre-releases become eligible too, and cuda-bindings | |
| # 13.4.0b1, published 2026-07-29, outranks the local build for | |
| # `cuda-bindings==13.*`. Its cydriver.pxd is generated from CTK 13.4 | |
| # headers, so CUmemLocation gains a `localized` field, while cuda.core | |
| # compiles against the 13.3.0 mini-CTK that has no such field. Cython | |
| # generates a struct converter over every field, and the build dies: | |
| # | |
| # _managed_memory_ops.cpp(18447): error C2039: 'localized' is not a | |
| # member of 'CUmemLocation_st' | |
| # | |
| # Windows coverage has produced nothing since; the job fails here and | |
| # Coverage (Windows) is skipped. cibuildwheel escapes it in | |
| # build-wheel.yml only because it sets PIP_FIND_LINKS without PIP_PRE. | |
| # | |
| # Pinning the exact local version keeps what PIP_PRE was for and drops | |
| # what it let in. The real fix belongs upstream; this only exists so the | |
| # crash diagnostics on this branch can run before that lands. | |
| - name: Build cuda.core wheel | |
| run: | | |
| export PIP_FIND_LINKS="$(pwd)/wheels" | |
| export PIP_PRE=1 | |
| bindings_whl="$(ls ./wheels/cuda_bindings-*.whl | head -1)" | |
| bindings_ver="$(basename "$bindings_whl" | cut -d- -f2)" | |
| # Drop the local segment: PEP 440 says a public `==` specifier ignores | |
| # a candidate's local label, so `==13.3.2.dev139` still selects | |
| # 13.3.2.dev139+g0123456 while avoiding local versions in a | |
| # constraints file, which pip has been unhappy with before. | |
| bindings_ver="${bindings_ver%%+*}" | |
| echo "cuda-bindings==${bindings_ver}" > "$GITHUB_WORKSPACE/constraints.txt" | |
| echo "=== pip constraints ===" | |
| cat "$GITHUB_WORKSPACE/constraints.txt" | |
| export PIP_CONSTRAINT="$GITHUB_WORKSPACE/constraints.txt" | |
| cd cuda_core | |
| ../.venv/Scripts/pip wheel -v --no-deps . -w ../wheels/ | |
| - name: List wheel artifacts | |
| run: | | |
| echo "=== Windows wheel artifacts ===" | |
| ls -lahR ./wheels/ | |
| - name: Upload Windows wheel artifacts | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: coverage-windows-wheels | |
| path: ./wheels/*.whl | |
| retention-days: 1 | |
| if-no-files-found: error | |
| # Run coverage tests on self-hosted GPU runner (no VS needed, installs pre-built wheels) | |
| coverage-windows: | |
| name: Coverage (Windows) | |
| needs: [coverage-vars, build-wheel-windows] | |
| runs-on: "windows-amd64-gpu-a100-latest-1" | |
| permissions: | |
| id-token: write | |
| contents: write | |
| env: | |
| HOST_PLATFORM: "win-64" | |
| ARCH: "amd64" | |
| CUDA_PYTHON_COVERAGE: "1" | |
| CUDA_VER: ${{ needs.coverage-vars.outputs.CUDA_VER }} | |
| # Diagnostics for the recurring exit-139 crash in the cuda.core step. | |
| # faulthandler names the Windows exception (access violation vs stack | |
| # overflow) and dumps every thread's Python stack; unlike pytest's | |
| # built-in faulthandler plugin, the env var is inherited by tests that | |
| # spawn their own `python -c` children, which crash too. Unbuffered | |
| # stdio keeps the last lines before a hard crash from dying in the | |
| # block buffer of a non-tty pipe. | |
| PYTHONFAULTHANDLER: "1" | |
| PYTHONUNBUFFERED: "1" | |
| defaults: | |
| run: | |
| shell: bash --noprofile --norc -xeuo pipefail {0} | |
| steps: | |
| - name: Checkout ${{ github.event.repository.name }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| fetch-depth: 1 | |
| - name: Setup proxy cache | |
| uses: nv-gha-runners/setup-proxy-cache@main | |
| continue-on-error: true | |
| # DRIVER above is 'latest' so install_gpu_driver.ps1 is intentionally | |
| # skipped (it errors on latest/earliest); configure_driver_mode.ps1 | |
| # still runs to put the pre-installed driver into TCC mode. | |
| - name: Configure driver mode | |
| shell: powershell | |
| env: | |
| DRIVER_MODE: "TCC" | |
| run: | | |
| ci/tools/configure_driver_mode.ps1 | |
| - name: Ensure GPU is working | |
| run: | | |
| nvidia-smi | |
| - name: Set up Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version: ${{ env.PY_VER }} | |
| - name: Set up mini CTK | |
| uses: ./.github/actions/fetch_ctk | |
| continue-on-error: false | |
| with: | |
| host-platform: win-64 | |
| cuda-version: ${{ env.CUDA_VER }} | |
| - name: Download Windows wheel artifacts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: coverage-windows-wheels | |
| path: ./wheels/ | |
| - name: List downloaded wheel artifacts | |
| run: | | |
| echo "=== Downloaded wheel artifacts ===" | |
| ls -lahR ./wheels/ | |
| - name: Create venv | |
| run: | | |
| python -m venv .venv | |
| - name: Install wheels from build job | |
| run: | | |
| .venv/Scripts/pip install ./wheels/cuda_pathfinder*.whl | |
| echo "Installed cuda.pathfinder" | |
| .venv/Scripts/pip install ./wheels/cuda_bindings*.whl | |
| echo "Installed cuda.bindings" | |
| .venv/Scripts/pip install ./wheels/cuda_core*.whl | |
| echo "Installed cuda.core" | |
| - name: Install test dependencies and coverage tools | |
| run: | | |
| .venv/Scripts/pip install -v ./cuda_python_test_helpers | |
| .venv/Scripts/pip install coverage pytest-cov Cython | |
| .venv/Scripts/pip install --group ./cuda_pathfinder/pyproject.toml:test | |
| .venv/Scripts/pip install --group ./cuda_bindings/pyproject.toml:test | |
| .venv/Scripts/pip install --group ./cuda_core/pyproject.toml:test | |
| - name: Get install root | |
| id: install-root | |
| run: | | |
| INSTALL_ROOT=$(.venv/Scripts/python -c 'import cuda; import os; print(os.path.dirname(cuda.__path__[0]))') | |
| echo "INSTALL_ROOT=$INSTALL_ROOT" >> $GITHUB_OUTPUT | |
| echo "Install root: $INSTALL_ROOT" | |
| - name: Run cuda.pathfinder tests | |
| continue-on-error: true | |
| run: | | |
| cd "${{ steps.install-root.outputs.INSTALL_ROOT }}" | |
| "$GITHUB_WORKSPACE/.venv/Scripts/pytest" -v --cov=./cuda --cov-append --cov-context=test --cov-config="$GITHUB_WORKSPACE/.coveragerc" "$GITHUB_WORKSPACE/cuda_pathfinder/tests" | |
| # Cython linetrace under coverage on Windows needs more stack than the | |
| # default 1 MB thread size. The helper runs pytest on an 8 MB thread. | |
| - name: Run cuda.bindings tests (with 8MB stack) | |
| continue-on-error: true | |
| run: | | |
| "$GITHUB_WORKSPACE/.venv/Scripts/python" "$GITHUB_WORKSPACE/ci/tools/run_pytest_with_stack.py" \ | |
| --cwd "${{ steps.install-root.outputs.INSTALL_ROOT }}" \ | |
| -v --cov=./cuda --cov-append --cov-context=test \ | |
| --cov-config="$GITHUB_WORKSPACE/.coveragerc" \ | |
| "$GITHUB_WORKSPACE/cuda_bindings/tests" | |
| # The whole crash reduces to one `import cuda.core` under coverage, with | |
| # no pytest, no conftest and no tests: the linetrace wheel and the | |
| # ctracer are the two conditions `test-wheel-windows.yml` lacks on this | |
| # same runner. Importing submodules one at a time was tried first and | |
| # measured nothing -- every one of them executes cuda/core/__init__.py | |
| # on the way in, so all six probes walked the same path and all six | |
| # failed identically. | |
| # | |
| # What is missing is the native half. The fault happens inside an | |
| # extension module's initialiser, and faulthandler's report ends at | |
| # `<cannot get C stack on this system>`. The usual ways to see past | |
| # that are all closed here: cdb is not installed (the SDK ships without | |
| # the Debugging Tools feature), procdump would need an outbound | |
| # download, a postmortem debugger under AeDebug never fires because this | |
| # image does not record user-process crashes with WER, and there are no | |
| # PDBs because build_hooks.py refuses debuggable builds on Windows. | |
| # | |
| # dbgcore.dll in System32 is always present though, so the process | |
| # reports on itself. ci/tools/native_crash_handler.py catches the fault | |
| # first-chance and writes the faulting address resolved to module + RVA, | |
| # the native return chain resolved the same way, the module map, and a | |
| # minidump, then lets the process die exactly as it would have. Which | |
| # binary faults is the open question, and module + RVA answers it | |
| # without symbols. | |
| # | |
| # Four runs, each answering something the others cannot: | |
| # | |
| # selftest faults on purpose, so a clean report proves the handler | |
| # works here and an empty one is not mistaken for "no | |
| # crash" | |
| # coverage the nightly's own conditions, on the 8 MB stack the | |
| # pytest wrapper uses | |
| # breadcrumb the same import with a bare trace function instead of | |
| # coverage. It arms Cython's linetrace the same way, so | |
| # a crash here means any tracer will do rather than | |
| # coverage specifically -- and the last line of its log | |
| # names the .pyx statement that was executing | |
| # fingerprint the environment around the import: driver, CPU, and | |
| # the loaded-module map, which is what a comparison | |
| # against a machine that does not crash needs | |
| # | |
| # COVERAGE_FILE keeps all of this out of the real .coverage, which | |
| # `coverage run` would otherwise truncate. | |
| - name: Record the native side of the fault | |
| if: always() | |
| continue-on-error: true | |
| env: | |
| COVERAGE_FILE: ${{ github.workspace }}/.coverage.probe | |
| run: | | |
| probe="$GITHUB_WORKSPACE/ci/tools/crash_probe.py" | |
| python="$GITHUB_WORKSPACE/.venv/Scripts/python" | |
| cd "${{ steps.install-root.outputs.INSTALL_ROOT }}" | |
| rc=0 | |
| "$python" "$probe" --selftest \ | |
| --native-log "$GITHUB_WORKSPACE/native-selftest.txt" \ | |
| --dump "$GITHUB_WORKSPACE/selftest.dmp" || rc=$? | |
| echo "SELFTEST rc=$rc" | |
| head -n 10 "$GITHUB_WORKSPACE/native-selftest.txt" || true | |
| rc=0 | |
| "$python" -m coverage run --rcfile="$GITHUB_WORKSPACE/.coveragerc" \ | |
| "$probe" --stack-mb 8 \ | |
| --native-log "$GITHUB_WORKSPACE/native-coverage.txt" \ | |
| --dump "$GITHUB_WORKSPACE/crash-coverage.dmp" || rc=$? | |
| echo "COVERAGE PROBE rc=$rc" | |
| cat "$GITHUB_WORKSPACE/native-coverage.txt" || true | |
| rc=0 | |
| "$python" "$probe" --stack-mb 8 \ | |
| --breadcrumb "$GITHUB_WORKSPACE/breadcrumb.txt" \ | |
| --native-log "$GITHUB_WORKSPACE/native-breadcrumb.txt" \ | |
| --dump "$GITHUB_WORKSPACE/crash-breadcrumb.dmp" || rc=$? | |
| echo "BREADCRUMB PROBE rc=$rc" | |
| echo "=== breadcrumb: $(wc -l < "$GITHUB_WORKSPACE/breadcrumb.txt") lines, last 25 ===" | |
| tail -n 25 "$GITHUB_WORKSPACE/breadcrumb.txt" || true | |
| rc=0 | |
| "$python" -m coverage run --rcfile="$GITHUB_WORKSPACE/.coveragerc" \ | |
| "$GITHUB_WORKSPACE/ci/tools/env_fingerprint.py" --stack-mb 8 \ | |
| "$GITHUB_WORKSPACE/fingerprint-windows.txt" || rc=$? | |
| echo "FINGERPRINT rc=$rc" | |
| # Only this step crashes. cuda.bindings runs the same wrapper, the same | |
| # 8 MB thread and the same coverage wheel, and finishes normally, so the | |
| # extra diagnostics are scoped to cuda.core alone. | |
| # | |
| # The crash log narrows the fault to "somewhere before pytest collects". | |
| # PYTHONVERBOSE narrows it to a module: the interpreter announces every | |
| # import as it begins one, so the last line written before the process | |
| # dies names the module it died in. That output is stderr, which is why | |
| # the session is redirected to a file rather than left in the step log -- | |
| # a crash takes the tail of a pipe with it, while each unbuffered write | |
| # to a file has already reached the OS. | |
| - name: Run cuda.core tests (with 8MB stack) | |
| continue-on-error: true | |
| env: | |
| PYTEST_CRASHLOG: ${{ github.workspace }}/crashlog-core.txt | |
| PYTEST_FAULTLOG: ${{ github.workspace }}/faulthandler-core.txt | |
| PYTHONVERBOSE: "1" | |
| run: | | |
| "$GITHUB_WORKSPACE/.venv/Scripts/python" "$GITHUB_WORKSPACE/ci/tools/run_pytest_with_stack.py" \ | |
| --isolate \ | |
| --cwd "${{ steps.install-root.outputs.INSTALL_ROOT }}" \ | |
| -v --cov=./cuda --cov-append --cov-context=test \ | |
| --cov-config="$GITHUB_WORKSPACE/.coveragerc" \ | |
| "$GITHUB_WORKSPACE/cuda_core/tests" \ | |
| > "$GITHUB_WORKSPACE/verbose-core.txt" 2>&1 | |
| # The step above has been exiting 139 on every run, and Git Bash reports | |
| # that as a bare "Segmentation fault". The crash log's final line says | |
| # how far the session actually got. | |
| - name: Report how far the cuda.core session got | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| log="$GITHUB_WORKSPACE/crashlog-core.txt" | |
| if [ -f "$log" ]; then | |
| echo "=== crashlog: $(wc -l < "$log") lines, last 20 ===" | |
| tail -n 20 "$log" | |
| else | |
| echo "No crash log: died before pytest started." | |
| fi | |
| echo "" | |
| fault="$GITHUB_WORKSPACE/faulthandler-core.txt" | |
| if [ -s "$fault" ]; then | |
| echo "=== faulthandler ===" | |
| cat "$fault" | |
| else | |
| # Empty is itself a result: the fault never reached Python's | |
| # handler, which points at native code running outside any Python | |
| # frame -- an extension module's initialisation, for instance. | |
| echo "=== faulthandler: no output ===" | |
| fi | |
| echo "" | |
| verbose="$GITHUB_WORKSPACE/verbose-core.txt" | |
| if [ -f "$verbose" ]; then | |
| echo "=== PYTHONVERBOSE: $(wc -l < "$verbose") lines, last 40 ===" | |
| tail -n 40 "$verbose" | |
| fi | |
| # Windows records an Application Error entry for every process it kills, | |
| # naming the faulting module and the offset within it, and it does so | |
| # whether or not crash dumps are configured. That is the one fact no | |
| # Python-level log can produce, and reading the log changes no machine | |
| # state, so nothing has to be restored afterwards. | |
| - name: Read the crash record from the Windows event log | |
| if: always() | |
| continue-on-error: true | |
| shell: powershell | |
| run: | | |
| $events = Get-WinEvent -FilterHashtable @{ | |
| LogName = 'Application' | |
| ProviderName = 'Application Error', 'Windows Error Reporting' | |
| StartTime = (Get-Date).AddHours(-2) | |
| } -ErrorAction SilentlyContinue | |
| if (-not $events) { | |
| Write-Output "No Application Error / WER records in the last 2 hours." | |
| Write-Output "(If this image keeps no such log, a minidump is the fallback.)" | |
| exit 0 | |
| } | |
| foreach ($e in $events | Select-Object -First 15) { | |
| Write-Output "===== $($e.TimeCreated.ToString('u')) $($e.ProviderName) Id=$($e.Id) =====" | |
| Write-Output $e.Message | |
| Write-Output "" | |
| } | |
| - name: Upload crash diagnostics | |
| if: always() | |
| continue-on-error: true | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: crash-diagnostics-windows | |
| path: | | |
| crashlog-core.txt | |
| faulthandler-core.txt | |
| verbose-core.txt | |
| native-selftest.txt | |
| native-coverage.txt | |
| native-breadcrumb.txt | |
| breadcrumb.txt | |
| fingerprint-windows.txt | |
| selftest.dmp | |
| crash-coverage.dmp | |
| crash-breadcrumb.dmp | |
| retention-days: 7 | |
| if-no-files-found: warn | |
| - name: Copy Windows coverage file to workspace | |
| run: | | |
| cp "${{ steps.install-root.outputs.INSTALL_ROOT }}/.coverage" "$GITHUB_WORKSPACE/.coverage.windows" | |
| echo "Copied .coverage.windows to $GITHUB_WORKSPACE" | |
| ls -lh "$GITHUB_WORKSPACE/.coverage.windows" | |
| - name: Upload Windows coverage data | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: coverage-data-windows | |
| path: .coverage.windows | |
| retention-days: 7 | |
| include-hidden-files: true | |
| if-no-files-found: error | |
| combine-and-deploy: | |
| name: Combine Coverage and Deploy | |
| needs: [coverage-linux, coverage-windows] | |
| runs-on: ubuntu-latest | |
| # always() alone would still run this after coverage-linux is skipped, and | |
| # it would both fail on the missing .coverage.linux and publish a half | |
| # report to gh-pages. | |
| if: ${{ always() && !startsWith(github.ref, 'refs/heads/pull-request/') }} | |
| permissions: | |
| id-token: write | |
| contents: write | |
| steps: | |
| - name: Checkout ${{ github.event.repository.name }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Set up Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version: ${{ env.PY_VER }} | |
| - name: Install coverage | |
| run: | | |
| # .coveragerc enables Cython.Coverage plugin; ensure it's available here too. | |
| pip install coverage[toml] Cython | |
| - name: Download Linux coverage data | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: coverage-data-linux | |
| path: ./ | |
| - name: Download cuda source code | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: cuda-source-linux | |
| path: ./cuda/ | |
| - name: Download Windows coverage data | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: coverage-data-windows | |
| path: ./ | |
| - name: Combine coverage data | |
| run: | | |
| echo "=== Working directory ===" | |
| pwd | |
| echo "" | |
| echo "=== All files in root ===" | |
| ls -la | |
| echo "" | |
| # Verify both Linux and Windows coverage files exist | |
| echo "=== Checking coverage files ===" | |
| if [ ! -f ".coverage.linux" ]; then | |
| echo "[ERROR] .coverage.linux not found!" | |
| exit 1 | |
| fi | |
| echo "[OK] Found .coverage.linux" | |
| if [ ! -f ".coverage.windows" ]; then | |
| echo "[ERROR] .coverage.windows not found!" | |
| exit 1 | |
| fi | |
| echo "[OK] Found .coverage.windows" | |
| echo "=== Available coverage files ===" | |
| ls -lh .coverage.* | |
| echo "=== Cuda source structure ===" | |
| find ./cuda -type d || { | |
| echo "[ERROR] No cuda source directory!" | |
| exit 1 | |
| } | |
| # Combine all .coverage.* files | |
| echo "" | |
| echo "=== Combining coverage data ===" | |
| coverage combine --rcfile=./.coveragerc --keep .coverage.* | |
| # Generate reports | |
| echo "" | |
| echo "=== Generating HTML, XML, and text reports ===" | |
| coverage html --rcfile=./.coveragerc | |
| coverage xml --rcfile=./.coveragerc -o coverage.xml | |
| coverage report --rcfile=./.coveragerc | |
| # Prepare for upload | |
| mkdir -p ./docs/coverage | |
| mv htmlcov/* ./docs/coverage/ | |
| mv coverage.xml ./docs/coverage/ | |
| echo "" | |
| echo "[SUCCESS] Coverage reports generated successfully" | |
| - name: Archive combined coverage results | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: coverage-combined | |
| path: docs/coverage/ | |
| retention-days: 7 | |
| include-hidden-files: true | |
| - name: Deploy to gh-pages | |
| uses: JamesIves/github-pages-deploy-action@d92aa235d04922e8f08b40ce78cc5442fcfbfa2f # v4.8.0 | |
| with: | |
| git-config-name: cuda-python-bot | |
| git-config-email: cuda-python-bot@users.noreply.github.com | |
| folder: docs/ | |
| target-folder: docs/ | |
| commit-message: "Deploy combined coverage (Linux + Windows): ${{ github.sha }}" | |
| clean: false |