release-readiness.yml

name: Release Readiness

# Heavy, optional checks that signal "are we in a state where a real release
# would be safe?". Decoupled from dev-publish so flakes here never block a
# publish, and from release-prod so heavy checks never run mid-release.
#
# A green run within the last 7 days is the manual go/no-go signal for a
# real release.
#
# What the install jobs below cover, and what they don't (#431):
# `install-tests` and `multi-python-install-verify` install the dev wheel
# most recently published to Artifact Registry by Dev Build & Publish, which
# only ever builds from `main` (see resolve-dev-version). Dispatching this
# workflow against a different ref does not change which wheel those two
# jobs install, so a green run from them is publish-path and cross-Python
# compatibility signal for main's latest wheel, never for the ref this
# workflow happens to be running against. For signal about a specific
# branch, read `wheel-install-verify` in on-push.yml instead — it builds the
# wheel from that ref's checkout. multi-python-install-verify's job name
# spells out "main dev wheel" so this scope is legible without opening this
# comment; install-tests' own header carries the same note.

on:
  workflow_dispatch:
    inputs:
      skip_integration:
        description: "Skip the integration-tests job (use while iterating on other jobs; tracked in CI-0019)"
        type: boolean
        default: false
  schedule:
    - cron: '0 6 * * 1'   # Mondays 06:00 UTC

concurrency:
  group: release-readiness
  cancel-in-progress: false

permissions:
  contents: read
  actions: read   # resolve-dev-version reads the latest Dev Build & Publish run + its artifact

env:
  PROTOC_VERSION: "28.3"
  # arduino/setup-protoc@v3.0.0 has no Node.js 24 release yet; remove once a node24 version ships
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true

jobs:
  # ---------------------------------------------------------------------------
  # Resolve the exact dev wheel version published by the most recent successful
  # Dev Build & Publish run. The install-tests and multi-python-install-verify
  # jobs pin to this version so they exercise the wheel built from current main,
  # not whatever the highest-versioned wheel happens to be on PyPI / AR.
  # (Without pinning, a stale prerelease on PyPI like 9.0.0rc1 would shadow
  # 8.1.2.dev* dev wheels per PEP 440 ordering.)
  # ---------------------------------------------------------------------------
  resolve-dev-version:
    name: Resolve latest dev wheel version
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      dev_version: ${{ steps.resolve.outputs.dev_version }}
      run_id: ${{ steps.find.outputs.run_id }}
    steps:
      - name: Find latest Dev Build & Publish run with a dev-version artifact
        id: find
        env:
          GH_TOKEN: ${{ github.token }}
          GH_REPO: ${{ github.repository }}
        run: |
          set -euo pipefail
          # Query the artifacts API directly instead of walking workflow runs.
          # Many dev-publish runs are no-ops (the check-changes gate skips
          # version/build/publish when there are no new commits) but still
          # report as success, so a fixed-size run window can miss the most
          # recent real publish. The artifacts API is sorted newest-first and
          # only returns runs that actually uploaded the artifact.
          #
          # The artifact's `workflow_run` object carries only id,
          # repository_id, head_repository_id, head_branch and head_sha — no
          # status or conclusion — so a candidate run can still be in
          # progress (its dev wheel not yet pushed to Artifact Registry) when
          # this is dispatched shortly after a merge to main. Walk the
          # newest-first candidates and check each run directly, taking the
          # first that has actually completed successfully.
          candidates=$(gh api \
              "repos/${GH_REPO}/actions/artifacts?name=dev-version&per_page=20" \
            --jq '[.artifacts[]
                    | select(.expired == false)
                    | select(.workflow_run.head_branch == "main")
                  ] | .[].workflow_run.id')

          run_id=""
          for candidate in $candidates; do
            state=$(gh api "repos/${GH_REPO}/actions/runs/${candidate}" \
              --jq '.status + " " + (.conclusion // "null")')
            if [ "$state" = "completed success" ]; then
              run_id="$candidate"
              break
            fi
          done

          if [ -z "$run_id" ]; then
            echo "::error::No non-expired 'dev-version' artifact on main from a completed, successful run"
            exit 1
          fi
          echo "run_id=${run_id}" >> "$GITHUB_OUTPUT"
          echo "Found dev-publish run: ${run_id}"

      - name: Download dev-version artifact
        env:
          GH_TOKEN: ${{ github.token }}
          GH_REPO: ${{ github.repository }}
        run: |
          set -euo pipefail
          gh run download "${{ steps.find.outputs.run_id }}" \
            --name dev-version \
            --dir ./dev-version

      - name: Resolve dev version
        id: resolve
        run: |
          set -euo pipefail
          version=$(tr -d '\n\r' < ./dev-version/dev_version.txt)
          if [ -z "$version" ]; then
            echo "::error::dev_version.txt is empty in run ${{ steps.find.outputs.run_id }}"
            exit 1
          fi
          echo "dev_version=${version}" >> "$GITHUB_OUTPUT"
          echo "Pinning installs to pinecone==${version}"

  # ---------------------------------------------------------------------------
  # Integration tests against a real Pinecone backend.
  #
  # This job is advisory by construction: Release Readiness has no
  # pull_request trigger, so nothing here gates a merge or a publish. It is
  # deliberately NOT a gate — per the release constraint that no CI gate may
  # depend on a live API key. Unit tests (on-push.yml) are the gate.
  #
  # `secrets.PINECONE_API_KEY` does not currently exist at repo or org level,
  # so it expands to the empty string and every keyed test skips itself: a
  # keyless run reports success with ~543/648 (~94%) of the suite inert. The
  # preflight step below turns that silent green into a loud red, because an
  # inert pass is worse than an honest failure when this workflow's green
  # result is the manual release go/no-go signal. Remediation is a one-time
  # human action: add a repo secret PINECONE_API_KEY scoped to a
  # test/CI Pinecone project, or dispatch with skip_integration=true.
  # Tracked in #295.
  # ---------------------------------------------------------------------------
  integration-tests:
    name: Integration tests (real backend)
    # Allow opting out via workflow_dispatch input while iterating on other
    # jobs. Schedule and default dispatch still run integration tests.
    if: ${{ github.event_name != 'workflow_dispatch' || !inputs.skip_integration }}
    runs-on: ubuntu-latest
    timeout-minutes: 90
    env:
      PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
    steps:
      - name: Preflight — integration credentials must be present
        run: |
          if [ -z "${PINECONE_API_KEY}" ]; then
            echo "::error::PINECONE_API_KEY is empty — ~94% of tests/integration would skip themselves and the job would report a meaningless success. Add a repo secret PINECONE_API_KEY, or dispatch with skip_integration=true to acknowledge the gap."
            {
              echo "### Integration tests: NOT RUN"
              echo
              echo "\`PINECONE_API_KEY\` is unset, so the suite would have been ~94% inert."
              echo "Failing loudly instead of reporting a green run with no coverage (#295)."
            } >> "$GITHUB_STEP_SUMMARY"
            exit 1
          fi
          echo "PINECONE_API_KEY present (${#PINECONE_API_KEY} chars)"

      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Install protoc
        uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b  # v3.0.0
        with:
          version: ${{ env.PROTOC_VERSION }}
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8  # stable
        with:
          toolchain: "1.98.0"

      - name: Cache cargo build artifacts
        uses: Swatinem/rust-cache@23869a5bd66c73db3c0ac40331f3206eb23791dc  # v2.9.1
        with:
          workspaces: rust
          shared-key: integration-tests
          save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Install uv
        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b  # v8.1.0
        with:
          enable-cache: true

      - name: Set up Python
        run: uv python install 3.12

      - name: Install deps + build Rust extension
        run: uv sync --group dev

      - name: pytest tests/integration
        run: uv run --no-sync pytest tests/integration -n 6 --dist=loadfile -v --junitxml=integration-results.xml

      - name: Upload integration results
        if: always()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f  # v7.0.0
        with:
          name: integration-results
          path: integration-results.xml

  # ---------------------------------------------------------------------------
  # Install the latest dev wheel from GCP Artifact Registry and smoke-test it.
  # Validates the publish/install path end-to-end (not just that the wheel was
  # built — the smoke job in dev-publish already covers that). Like
  # multi-python-install-verify below, the wheel installed here is always
  # main's, independent of the ref this workflow is dispatched against (#431).
  #
  # Same missing secret as integration-tests above: `secrets.PINECONE_API_KEY`
  # exists at neither repo nor org level, so it expands to the empty string and
  # all 15 smoke tests this job collects (pod collections are ignored below)
  # skip themselves — the smoke step exits 0 and
  # the job reports success having validated the install path but exercised
  # none of the wheel's behaviour. The preflight below turns that into an
  # honest red. It sits *after* the install/pip-check steps on purpose: those
  # are real signal that does not need a key, and killing the job before them
  # would trade one blind spot for another. Advisory by construction (this
  # workflow has no pull_request trigger), so failing on a missing key does not
  # make a gate depend on a live key. Remediation is a one-time human action:
  # add a repo secret PINECONE_API_KEY scoped to a test/CI Pinecone project.
  # Tracked in #315.
  # ---------------------------------------------------------------------------
  install-tests:
    name: Install from Artifact Registry + smoke
    needs: resolve-dev-version
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      contents: read
      id-token: write
    env:
      GCP_REGION: ${{ vars.GCP_REGION }}
      GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}
      GCP_REPO_NAME: ${{ vars.GCP_REPO_NAME }}
      PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
      DEV_VERSION: ${{ needs.resolve-dev-version.outputs.dev_version }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Authenticate to Google Cloud
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093  # v3.0.0
        with:
          workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
          service_account: ${{ vars.GCP_SERVICE_ACCOUNT }}

      - name: Set up Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: "3.12"

      - name: Install Artifact Registry keyring backend
        run: |
          python -m pip install --upgrade pip
          pip install keyring==25.6.0 keyrings.google-artifactregistry-auth==1.1.2

      - name: Install pinned dev wheel from Artifact Registry
        run: |
          set -euo pipefail
          INDEX_URL="https://${GCP_REGION}-python.pkg.dev/${GCP_PROJECT_ID}/${GCP_REPO_NAME}/simple/"
          echo "Installing pinecone==${DEV_VERSION} from ${INDEX_URL}"
          # Pin to the exact version produced by the resolved dev-publish run so
          # the smoke suite exercises the wheel built from current main, not a
          # higher-versioned PyPI prerelease that pip would otherwise prefer.
          # --pre still required because the version specifier itself contains .dev.
          #
          # pytest-timeout and anyio are not optional: they own the `timeout`
          # and `anyio_mode` keys in [tool.pytest.ini_options], and #306's
          # `--strict-config` turns an ini key whose owning plugin is absent
          # into a usage error. In pytest 9 it fires after collection, so the
          # step reported "FAILED, 7 skipped" — the suite's collect-time skips
          # followed by exit 4, with none of the 12 collected tests run (#399).
          # anyio also arrives transitively via httpx; naming it keeps that
          # from being load-bearing. Held to the ini block by
          # tests/unit/tooling/test_smoke_ci_plugin_deps.py.
          pip install --pre --extra-index-url "${INDEX_URL}" \
            "pinecone==${DEV_VERSION}" \
            pytest pytest-asyncio "pytest-timeout>=2.3" "anyio>=4.13.0" python-dotenv

      - name: Show installed version
        run: pip show pinecone | grep -E '^(Name|Version|Location):'

      - name: Verify dependency compatibility (pip check)
        run: pip check

      # The repo's `pinecone/` source dir would shadow the installed wheel
      # because Python prepends CWD to sys.path. The source tree has no
      # compiled `_grpc.abi3.so`, so any test that touches the gRPC path
      # fails with `ModuleNotFoundError: No module named 'pinecone._grpc'`.
      # Move the source aside so `import pinecone` resolves to site-packages.
      - name: Move source pinecone/ aside (avoid shadowing installed wheel)
        run: mv pinecone _pinecone_source

      - name: Preflight — smoke credentials must be present
        run: |
          if [ -z "${PINECONE_API_KEY}" ]; then
            echo "::error::PINECONE_API_KEY is empty — all 15 smoke tests this job collects would skip themselves and this job would report success having exercised none of the installed wheel. Add a repo secret PINECONE_API_KEY scoped to a test/CI Pinecone project."
            {
              echo "### Install + smoke: install path validated, smoke NOT RUN"
              echo
              echo "The wheel installed cleanly and \`pip check\` passed, so the publish/install"
              echo "path is verified. \`PINECONE_API_KEY\` is unset, so the smoke suite itself was"
              echo "skipped in full — failing loudly instead of reporting a green run with no"
              echo "behavioural coverage (#315)."
            } >> "$GITHUB_STEP_SUMMARY"
            exit 1
          fi
          echo "PINECONE_API_KEY present (${#PINECONE_API_KEY} chars)"

      - name: Run smoke suite
        id: smoke
        run: |
          pytest tests/smoke/ \
            -v -s -rs \
            --ignore=tests/smoke/test_pod_collections_sync.py \
            --ignore=tests/smoke/test_pod_collections_async.py \
            --junitxml=install-smoke-results.xml

      # `python -m`, not `python tests/smoke/scripts/cleanup_orphans.py`: the
      # plain-path form makes sys.path[0] the *script's* directory, so
      # `from tests.live_suite import load_env` cannot resolve and the step
      # died before deleting anything. It reported success for its entire
      # existence because `|| true` rewrote the exit code and continue-on-error
      # swallowed what was left (#412). A local `uv run` on that same path form
      # works only because the editable install drops a `pinecone.pth` naming
      # the project root — a property these wheel-install jobs do not have.
      #
      # `|| true` is gone deliberately. continue-on-error already keeps a
      # cleanup failure from failing the job; the `|| true` on top of it only
      # bought silence. The summary block is what makes a leak legible without
      # opening the log.
      - name: Orphan cleanup
        if: always()
        continue-on-error: true
        run: |
          set -o pipefail
          status=0
          python -m tests.smoke.scripts.cleanup_orphans 2>&1 | tee orphan-cleanup.log || status=$?
          {
            echo "### Orphan cleanup"
            echo
            echo '```'
            cat orphan-cleanup.log
            echo '```'
          } >> "$GITHUB_STEP_SUMMARY"
          if [ "${status}" -ne 0 ]; then
            echo "::warning::Orphan cleanup exited ${status}; leaked smoke resources may remain in the test project."
          fi
          exit "${status}"

      - name: Upload install-smoke results
        if: always()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f  # v7.0.0
        with:
          name: install-smoke-results
          path: install-smoke-results.xml

  # ---------------------------------------------------------------------------
  # Multi-Python install verify: exercises the abi3 promise on every supported
  # Python version (3.10–3.14) using the dev wheel resolved above, which is
  # always built from main regardless of what ref this workflow runs against
  # (#431) — read this job's result as "main's latest published wheel loads
  # its Rust extension and resolves its declared deps on py3.10-3.14", not as
  # signal about any other ref. The published wheel is cp310-abi3, so it must
  # install and load the Rust extension on each Python we claim to support.
  # ---------------------------------------------------------------------------
  multi-python-install-verify:
    name: Multi-Python install verify, main dev wheel (py${{ matrix.python-version }})
    needs: resolve-dev-version
    runs-on: ubuntu-latest
    timeout-minutes: 15
    permissions:
      contents: read
      id-token: write
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
    env:
      GCP_REGION: ${{ vars.GCP_REGION }}
      GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}
      GCP_REPO_NAME: ${{ vars.GCP_REPO_NAME }}
      DEV_VERSION: ${{ needs.resolve-dev-version.outputs.dev_version }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
      - name: Authenticate to Google Cloud
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093  # v3.0.0
        with:
          workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
          service_account: ${{ vars.GCP_SERVICE_ACCOUNT }}
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install AR keyring backend
        run: |
          python -m pip install --upgrade pip
          pip install keyring==25.6.0 keyrings.google-artifactregistry-auth==1.1.2
      - name: Install pinned dev wheel from Artifact Registry
        run: |
          set -euo pipefail
          INDEX_URL="https://${GCP_REGION}-python.pkg.dev/${GCP_PROJECT_ID}/${GCP_REPO_NAME}/simple/"
          # Pinned to the exact version from the resolved dev-publish run; see
          # comment on install-tests for rationale.
          pip install --pre --extra-index-url "${INDEX_URL}" "pinecone==${DEV_VERSION}"
      - name: Verify dependency compatibility (pip check)
        run: pip check
      - name: Show installed version + Python
        run: |
          python -V
          pip show pinecone | grep -E '^(Name|Version|Location):'
      # The repo's `pinecone/` source dir would shadow the installed wheel
      # because Python prepends CWD to sys.path. The source tree has no
      # compiled `_grpc.abi3.so`, so the import below fails with
      # `ModuleNotFoundError: No module named 'pinecone._grpc'`.
      # Move the source aside so `import pinecone` resolves to site-packages.
      - name: Move source pinecone/ aside (avoid shadowing installed wheel)
        run: mv pinecone _pinecone_source
      # `import pinecone` on its own proves almost nothing: __init__ is lazy, so
      # it reaches none of the modules that import third-party code. #411 was a
      # missing runtime dependency that made every lazy attribute unreachable on
      # 3.13/3.14 while this job stayed green on all five versions. Touching the
      # attributes is what exercises the imports. No API key needed.
      - name: Verify import + Rust extension on this Python version
        run: |
          python -c "
          import pinecone
          print('OK: import pinecone on Python', __import__('sys').version_info[:2])
          names = sorted(pinecone._LAZY_IMPORTS)
          assert len(names) >= 100, f'only {len(names)} lazy exports found'
          broken = []
          for name in names:
              try:
                  getattr(pinecone, name)
              except ModuleNotFoundError as exc:
                  broken.append(f'{name}: missing {exc.name}')
          assert not broken, 'undeclared runtime dependency: ' + '; '.join(broken[:8])
          print('OK:', len(names), 'lazy exports resolved')
          import pinecone._grpc
          print('OK: import pinecone._grpc')
          ch = pinecone._grpc.GrpcChannel(
              endpoint='https://example.invalid:443',
              api_key='test',
              api_version='2025-10',
              version='0.0.0',
          )
          print('OK: GrpcChannel constructed:', type(ch).__name__)
          "

  # ---------------------------------------------------------------------------
  # Cross-platform wheel build matrix without injecting dev versions or
  # publishing. Independent signal that release-builds work on every target.
  # ---------------------------------------------------------------------------
  cross-platform-build:
    name: Wheel build (${{ matrix.target }})
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu-latest
            target: x86_64-unknown-linux-gnu
            manylinux: auto
          - os: ubuntu-24.04-arm
            target: aarch64-unknown-linux-gnu
            manylinux: auto
          - os: ubuntu-latest
            target: x86_64-unknown-linux-musl
            manylinux: musllinux_1_2
          - os: ubuntu-24.04-arm
            target: aarch64-unknown-linux-musl
            manylinux: musllinux_1_2
          - os: macos-14
            target: x86_64-apple-darwin
          - os: macos-14
            target: aarch64-apple-darwin
          - os: windows-latest
            target: x86_64-pc-windows-msvc
    runs-on: ${{ matrix.os }}
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Install protoc
        if: runner.os != 'Linux'
        uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b  # v3.0.0
        with:
          version: ${{ env.PROTOC_VERSION }}
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Build wheels
        uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad  # v1.50.1
        with:
          target: ${{ matrix.target }}
          args: --release --out dist
          manylinux: ${{ matrix.manylinux || 'auto' }}
          before-script-linux: |
            TARGET="${{ matrix.target }}"
            PROTOC_VERSION="${{ env.PROTOC_VERSION }}"
            case "$TARGET" in
              x86_64-*)
                PROTOC_ARCH="linux-x86_64"
                PROTOC_SHA256="0ad949f04a6a174da83cdcbdb36dee0a4925272a5b6d83f79a6bf9852076d53f"
                ;;
              aarch64-*)
                PROTOC_ARCH="linux-aarch_64"
                PROTOC_SHA256="1de522032a8b194002fe35cab86d747848238b5e4de4f99648372079f5b46f9a"
                ;;
              *)
                echo "Unsupported target: $TARGET" >&2
                exit 1
                ;;
            esac
            PROTOC_ZIP="protoc-${PROTOC_VERSION}-${PROTOC_ARCH}.zip"
            curl -fsSLO "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/${PROTOC_ZIP}"
            echo "${PROTOC_SHA256}  ${PROTOC_ZIP}" | sha256sum -c -
            python3 -m zipfile -e "${PROTOC_ZIP}" /usr/local
            chmod +x /usr/local/bin/protoc
            rm "${PROTOC_ZIP}"

      - name: Validate wheel built
        shell: bash
        run: |
          count=$(ls dist/*.whl 2>/dev/null | wc -l)
          echo "Built ${count} wheel(s) for ${{ matrix.target }}"
          ls -la dist/
          if [ "$count" -lt 1 ]; then
            echo "::error::no wheels produced for ${{ matrix.target }}"
            exit 1
          fi

      # abi3audit parses the compiled extension inside the wheel and flags any CPython
      # symbols that aren't part of the stable ABI (PEP 384). A cp310-abi3-tagged wheel
      # that pulls in a non-stable-ABI symbol imports fine on the build Python but breaks
      # at runtime on other 3.x versions. This is the static complement to multi-python
      # smoke tests: the smoke tests exercise the wheel; abi3audit proves it is valid by
      # inspection. Runs on every OS and reads any wheel format (ELF/Mach-O/PE).
      #
      # Isolated in its own venv: pip-installing abi3audit into the runner's host
      # Python leaves abi3audit + packaging + rich in user site-packages, which then
      # makes the verify step's `pip check` fail because the verify Python sees those
      # leftover packages but pinecone's install does not pull in their transitive
      # deps (urllib3, platformdirs). The venv keeps abi3audit fully out of the
      # verify Python's view. See CI-0037 for the original failure (Release Readiness
      # run 25457165040).
      - name: abi3audit
        shell: bash
        run: |
          set -euo pipefail
          VENV_DIR="${RUNNER_TEMP}/abi3audit-venv"
          python3 -m venv "$VENV_DIR"
          if [ -d "$VENV_DIR/Scripts" ]; then
            VBIN="$VENV_DIR/Scripts"
          else
            VBIN="$VENV_DIR/bin"
          fi
          "$VBIN/pip" install --quiet abi3audit
          "$VBIN/python" -m abi3audit --strict --report dist/*.whl

      # auditwheel show inspects ELF binaries inside each manylinux wheel and reports
      # the actual minimum glibc version. A wheel tagged manylinux_2_17 but linking
      # newer symbols would silently break users on older systems; this catches that
      # before we ever publish.
      #
      # Restricted to glibc: auditwheel resolves library dependencies via the host's
      # ldd, so analyzing a musllinux wheel on a glibc runner blows up with
      # "ELFError: Magic number does not match" when it follows links into musl libc
      # paths that don't exist. Musl ABI is exercised by the in-alpine install step.
      #
      # Isolated in its own venv: pip-installing auditwheel into the runner's host
      # Python leaves auditwheel + packaging in user site-packages, which then makes
      # the verify step's `pip check` fail ("auditwheel requires packaging") because
      # PEP-668 fallbacks and pip self-upgrade can desync those installs. The venv
      # keeps auditwheel fully out of the verify Python's view.
      - name: auditwheel show (manylinux only)
        if: endsWith(matrix.target, '-unknown-linux-gnu')
        shell: bash
        run: |
          set -euo pipefail
          python3 -m venv /tmp/auditwheel-venv
          /tmp/auditwheel-venv/bin/pip install --quiet auditwheel
          for wheel in dist/*.whl; do
            echo "=== auditwheel show: $wheel ==="
            /tmp/auditwheel-venv/bin/python -m auditwheel show "$wheel"
          done

      # Install verification: prove the wheel installs cleanly on its target
      # platform and the Rust extension loads + is callable. The GrpcChannel
      # constructor is lazy (no network), so this is a fast purely-local check.
      #
      # Skipped for x86_64-apple-darwin: it's cross-compiled on an arm64 runner
      # and can't be exercised in-place. Tracked as a known gap.
      - name: Set up Python for install verification
        if: matrix.target != 'x86_64-apple-darwin' && !endsWith(matrix.target, '-musl')
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: "3.12"

      - name: Write install-verification script
        if: matrix.target != 'x86_64-apple-darwin'
        shell: bash
        run: |
          mkdir -p .ci
          cat > .ci/verify_install.py <<'PY'
          """Smoke-check that the just-installed pinecone wheel works.

          Exercises the platform-specific Rust extension by importing
          pinecone._grpc and constructing a GrpcChannel (lazy, no network).
          """
          import sys

          import pinecone  # noqa: F401  — top-level import must succeed
          print("OK: import pinecone")

          import pinecone._grpc  # native dylib must load on this platform
          print("OK: import pinecone._grpc")

          ch = pinecone._grpc.GrpcChannel(
              endpoint="https://example.invalid:443",
              api_key="test-key",
              api_version="2025-10",
              version="0.0.0-install-test",
          )
          print("OK: GrpcChannel constructed:", type(ch).__name__)
          sys.exit(0)
          PY

      - name: Install wheel + run verification (native host)
        if: matrix.target != 'x86_64-apple-darwin' && !endsWith(matrix.target, '-musl')
        shell: bash
        run: |
          set -euo pipefail
          wheel=$(ls dist/*.whl | head -n1)
          echo "Installing ${wheel} on $(uname -sm) with Python $(python -V)"
          python -m pip install --upgrade pip
          python -m pip install "${wheel}"
          python -m pip check
          python .ci/verify_install.py

      - name: Install wheel + run verification (musl, alpine container)
        if: endsWith(matrix.target, '-musl')
        shell: bash
        run: |
          set -euo pipefail
          if [[ "${{ matrix.target }}" == aarch64-* ]]; then
            IMAGE="arm64v8/python:3.12-alpine"
          else
            IMAGE="python:3.12-alpine"
          fi
          echo "Verifying musl wheel inside ${IMAGE}"
          docker run --rm \
            -v "$PWD/dist:/dist:ro" \
            -v "$PWD/.ci/verify_install.py:/verify_install.py:ro" \
            "$IMAGE" sh -c '
              set -e
              ls /dist/*.whl
              pip install --quiet /dist/*.whl
              pip check
              python /verify_install.py
            '

      - name: Note skipped install verification (x86_64-apple-darwin)
        if: matrix.target == 'x86_64-apple-darwin'
        run: |
          echo "::notice::install verification skipped for x86_64-apple-darwin: cross-compiled on arm64 runner, cannot run in place. Real users on Intel macs are the first to exercise this wheel."

  # ---------------------------------------------------------------------------
  # Sphinx docs build (warnings are errors).
  # ---------------------------------------------------------------------------
  doc-build:
    name: Sphinx docs build
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Install protoc
        uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b  # v3.0.0
        with:
          version: ${{ env.PROTOC_VERSION }}
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8  # stable
        with:
          toolchain: "1.98.0"

      - name: Cache cargo build artifacts
        uses: Swatinem/rust-cache@23869a5bd66c73db3c0ac40331f3206eb23791dc  # v2.9.1
        with:
          workspaces: rust
          shared-key: doc-build
          save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Install uv
        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b  # v8.1.0
        with:
          enable-cache: true

      - name: Set up Python
        run: uv python install 3.12

      - name: Install deps (incl. docs extra) + build Rust extension
        run: uv sync --group dev --extra docs

      - name: Build HTML docs
        run: uv run --no-sync make -C docs html SPHINXOPTS="-W"

      - name: Upload docs artifact
        if: always()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f  # v7.0.0
        with:
          name: docs-html
          path: docs/_build/html