diff --git a/.github/workflows/intake-archive.yml b/.github/workflows/intake-archive.yml new file mode 100644 index 0000000..841e826 --- /dev/null +++ b/.github/workflows/intake-archive.yml @@ -0,0 +1,333 @@ +name: intake-archive + +# Archive a non-binary package (module, script, or completion) from an immutable +# upstream commit, publish the archive as a GitHub release asset on THIS repo, and +# emit a numan-registry spec. Nothing is compiled here and nothing is signed here — +# numan-registry pins the emitted asset URL via add-package.py and signs the index +# with the official trust root. +# +# workflow_dispatch accepts at most 10 inputs, so `package` carries owner/name and +# `activation` carries kind[:import], a non-blank `deferral_reason` implies +# provisional intake, and the version is always derived from `ref` +# (scripts/intake_archive.py keeps --owner/--name/--version for local runs). + +on: + workflow_dispatch: + inputs: + git_url: + description: "Upstream clone URL or owner/name slug" + required: true + type: string + ref: + description: "Upstream tag, branch, or commit to archive" + required: true + type: string + entry: + description: "Entry file path inside the upstream repo, e.g. mod.nu" + required: true + type: string + package: + description: "Registry package as owner/name" + required: true + type: string + type: + description: "Package type" + required: true + type: choice + options: + - module + - script + - completion + description: + description: "Package description for the registry spec" + required: true + type: string + tags: + description: 'JSON array of tags, e.g. ["module"]' + required: true + type: string + nu_version: + description: "Nu compatibility range, e.g. >=0.114.0 <0.115.0" + required: true + type: string + activation: + description: "Activation as kind[:import], e.g. nu-module:all" + required: false + default: "" + type: string + deferral_reason: + description: "Why lifecycle-prove is deferred; non-blank means provisional intake" + required: false + default: "" + type: string + +permissions: + contents: read + +jobs: + archive: + runs-on: ubuntu-latest + outputs: + owner: ${{ steps.intake.outputs.owner }} + name: ${{ steps.intake.outputs.name }} + tag: ${{ steps.record.outputs.tag }} + version: ${{ steps.record.outputs.version }} + archive: ${{ steps.record.outputs.archive }} + sha256: ${{ steps.record.outputs.sha256 }} + resolved_sha: ${{ steps.record.outputs.resolved_sha }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Resolve, archive, and emit the registry spec + id: intake + shell: bash + env: + GIT_URL: ${{ inputs.git_url }} + UPSTREAM_REF: ${{ inputs.ref }} + ENTRY: ${{ inputs.entry }} + PACKAGE: ${{ inputs.package }} + PKG_TYPE: ${{ inputs.type }} + DESCRIPTION: ${{ inputs.description }} + TAGS: ${{ inputs.tags }} + NU_VERSION: ${{ inputs.nu_version }} + ACTIVATION: ${{ inputs.activation }} + DEFERRAL_REASON: ${{ inputs.deferral_reason }} + run: | + set -euo pipefail + if [[ "$PACKAGE" != */* || "$PACKAGE" == */*/* ]]; then + echo "FAIL: package must be owner/name: $PACKAGE" >&2 + exit 1 + fi + owner="${PACKAGE%%/*}" + name="${PACKAGE##*/}" + if [ -z "$owner" ] || [ -z "$name" ]; then + echo "FAIL: package needs both halves of owner/name: $PACKAGE" >&2 + exit 1 + fi + { + echo "owner=$owner" + echo "name=$name" + } >> "$GITHUB_OUTPUT" + + flags=() + if [ -n "$ACTIVATION" ]; then + activation_kind="${ACTIVATION%%:*}" + activation_import="${ACTIVATION##*:}" + if [ -z "$activation_kind" ]; then + echo "FAIL: activation must be kind[:import]: $ACTIVATION" >&2 + exit 1 + fi + flags+=(--activation-kind "$activation_kind") + # A trailing colon means "no import mode": an empty + # --activation-import would fail argparse's choices with a usage dump. + if [[ "$ACTIVATION" == *:* && -n "$activation_import" ]]; then + flags+=(--activation-import "$activation_import") + fi + fi + if [ -n "$DEFERRAL_REASON" ]; then + flags+=(--provisional --deferral-reason "$DEFERRAL_REASON") + fi + + python3 scripts/intake_archive.py \ + --git-url "$GIT_URL" \ + --ref "$UPSTREAM_REF" \ + --entry "$ENTRY" \ + --owner "$owner" \ + --name "$name" \ + --type "$PKG_TYPE" \ + --description "$DESCRIPTION" \ + --tags "$TAGS" \ + --nu-version "$NU_VERSION" \ + --release-root "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/download" \ + --archive-out dist \ + --out "spec-$owner-$name.json" \ + "${flags[@]}" | tee archive.tsv + + - name: Record the archive result for the publish job + id: record + shell: bash + run: | + set -euo pipefail + line="$(grep '^ARCHIVED' archive.tsv)" + { + echo "resolved_sha=$(printf '%s' "$line" | cut -f2)" + echo "version=$(printf '%s' "$line" | cut -f3)" + echo "tag=$(printf '%s' "$line" | cut -f4)" + echo "archive=$(printf '%s' "$line" | cut -f5)" + echo "sha256=$(printf '%s' "$line" | cut -f6)" + } >> "$GITHUB_OUTPUT" + + - name: Upload archive + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: archive-asset + path: dist/* + if-no-files-found: error + + - name: Upload spec for numan-registry intake + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: spec-${{ steps.intake.outputs.owner }}-${{ steps.intake.outputs.name }} + path: spec-${{ steps.intake.outputs.owner }}-${{ steps.intake.outputs.name }}.json + if-no-files-found: error + + - name: Upload archive record + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: archive-record + path: archive.tsv + if-no-files-found: error + + # This workflow deliberately pushes no commit, so the re-intake provenance + # intake_archive.py recorded leaves as an artifact for a maintainer to commit; + # the publish job's summary repeats that reminder. + - name: Upload recorded re-intake provenance + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: manifest-archives + path: manifest-archives.json + if-no-files-found: error + + publish: + needs: archive + runs-on: ubuntu-latest + permissions: + contents: write # Required only to publish the release tag, release, and assets. + concurrency: + group: publish-archive-${{ needs.archive.outputs.tag }} + cancel-in-progress: false + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Collect the archive asset + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: archive-asset + path: dist + + - name: Collect the emitted spec + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: spec-${{ needs.archive.outputs.owner }}-${{ needs.archive.outputs.name }} + path: spec + + # The artifact round-trip between the two jobs is the only gap between the + # bytes that were archived and the bytes this job publishes; build.yml + # closes the same gap with gen_spec.py's verify_packaged_assets. + - name: Verify the downloaded asset is the archived asset + shell: bash + env: + ARCHIVE: ${{ needs.archive.outputs.archive }} + ARCHIVE_SHA256: ${{ needs.archive.outputs.sha256 }} + run: | + set -euo pipefail + mapfile -t assets < <(find dist -type f -printf '%P\n' | sort) + if [ "${#assets[@]}" -ne 1 ] || [ "${assets[0]}" != "$ARCHIVE" ]; then + echo "FAIL: dist must hold exactly one asset named $ARCHIVE, found: ${assets[*]-}" >&2 + exit 1 + fi + printf '%s %s\n' "$ARCHIVE_SHA256" "dist/$ARCHIVE" | sha256sum --check --strict - + + - name: Refuse an existing release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.archive.outputs.tag }} + run: >- + python3 scripts/ensure_release_absent.py + --repo "$GITHUB_REPOSITORY" + --tag "$TAG" + + - name: Claim immutable tag and draft release + id: claim + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.archive.outputs.tag }} + RELEASE_NAME: ${{ inputs.package }} ${{ needs.archive.outputs.version }} + RELEASE_BODY: | + Non-binary archive intake for `${{ inputs.package }}`, + archived from `${{ inputs.git_url }}@${{ inputs.ref }}` + at commit `${{ needs.archive.outputs.resolved_sha }}`. + Deterministic `.tar.gz`; pinned + hash-verified + signed downstream in numan-registry. + run: >- + python3 scripts/release_transaction.py claim + --repo "$GITHUB_REPOSITORY" + --tag "$TAG" + --commit "$GITHUB_SHA" + --name "$RELEASE_NAME" + --body "$RELEASE_BODY" + + - name: Upload the asset to the owned draft + env: + GH_TOKEN: ${{ github.token }} + CLAIMED_RELEASE_ID: ${{ steps.claim.outputs.release_id }} + run: >- + python3 scripts/release_transaction.py upload + --repo "$GITHUB_REPOSITORY" + --release-id "$CLAIMED_RELEASE_ID" + --assets-dir dist + + - name: Verify and publish the complete draft + env: + GH_TOKEN: ${{ github.token }} + CLAIMED_RELEASE_ID: ${{ steps.claim.outputs.release_id }} + TAG: ${{ needs.archive.outputs.tag }} + run: >- + python3 scripts/release_transaction.py finalize + --repo "$GITHUB_REPOSITORY" + --release-id "$CLAIMED_RELEASE_ID" + --tag "$TAG" + --commit "$GITHUB_SHA" + --assets-dir dist + + - name: Summarize the registry handoff + shell: bash + env: + OWNER: ${{ needs.archive.outputs.owner }} + NAME: ${{ needs.archive.outputs.name }} + TAG: ${{ needs.archive.outputs.tag }} + VERSION: ${{ needs.archive.outputs.version }} + RESOLVED_SHA: ${{ needs.archive.outputs.resolved_sha }} + ARCHIVE_SHA256: ${{ needs.archive.outputs.sha256 }} + DEFERRAL_REASON: ${{ inputs.deferral_reason }} + run: | + set -euo pipefail + spec="spec-$OWNER-$NAME.json" + { + echo "### Non-binary archive intake: $OWNER/$NAME $VERSION" + echo "- release tag: \`$TAG\`" + echo "- upstream commit: \`$RESOLVED_SHA\`" + echo "- archive sha256: \`$ARCHIVE_SHA256\`" + echo "- Commit the \`manifest-archives\` artifact over \`manifest-archives.json\` to keep re-intake provenance; this run pushes no commit." + if [ -n "$DEFERRAL_REASON" ]; then + echo "- provisional intake, deferral reason: $DEFERRAL_REASON" + echo "- next, in numan-registry: \`python scripts/add-package.py --spec $spec --write --provisional --deferral-reason \"$DEFERRAL_REASON\"\`" + else + echo "- provisional intake: no" + echo "- next, in numan-registry: \`python scripts/add-package.py --spec $spec --write\`" + fi + echo "" + echo "
$spec" + echo "" + echo '```json' + cat "spec/$spec" + echo '```' + echo "" + echo "
" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Clean up this run's failed draft + if: (failure() || cancelled()) && steps.claim.outputs.release_id != '' + env: + GH_TOKEN: ${{ github.token }} + CLAIMED_RELEASE_ID: ${{ steps.claim.outputs.release_id }} + TAG: ${{ needs.archive.outputs.tag }} + run: >- + python3 scripts/release_transaction.py cleanup + --repo "$GITHUB_REPOSITORY" + --release-id "$CLAIMED_RELEASE_ID" + --tag "$TAG" + --commit "$GITHUB_SHA" diff --git a/docs/roadmap.md b/docs/roadmap.md index 2832da2..22795fd 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -21,7 +21,7 @@ Use this page for **operational** detail that belongs only to `numan-plugins`: - Workflow manifests under `.github/workflows/` (`build.yml`, - `repo-safety.yml`, `windows-recheck` / release paths). + `intake-archive.yml`, `repo-safety.yml`, `windows-recheck` / release paths). - Per-package target exclusions and reasons in `manifest.json` (`exclude_targets` / `exclude_reason`). - Backlog triage notes (`docs/backlog.json` schema + review log). @@ -126,7 +126,7 @@ are source-only plugins with tags and enough demand to justify CI-built assets: - [x] `drbrain/nu_plugin_prometheus` — promoted 2026-07-31 to `active[]` as `v0.12.0` (nu-plugin/nu-protocol 0.114.1; commit `3fed1d934ba201ce1d9b78ecb727695588de7ef9`). Windows locked green; `aarch64-unknown-linux-gnu` excluded (openssl-sys cross). CI-built release published; in official registry. `v0.11.0` was `PRE_0_112` (0.110.0). - [x] `galuszkak/nu_plugin_bigquery` — researched 2026-07-31: `v0.2.0` pins nu-plugin 0.112.2; eligible for intake via P6 Provisional Tier with deferral reason. -### Intake Reform Wave (P1 Commit-Snapshot, P4 Maintained Forks, P6 Provisional) +### Intake Reform Wave (P1 Commit-Snapshot, P2 Non-Binary Archive, P4 Maintained Forks, P6 Provisional) With the intake reform tooling merged into `numan-plugins` (commit-snapshot intake mode `#80`, fork identity `#82`): @@ -157,6 +157,47 @@ Proposed forks evaluate under ADR 0001 stewardship criteria (requiring `numan-ma - [ ] `galuszkak/nu_plugin_bigquery` — build and package normally in `numan-plugins`, then intake into `numan-registry` using `scripts/add-package.py --provisional --deferral-reason "..."` +`scripts/gen_spec.py --provisional --deferral-reason ""` emits the provisional +evidence tier (`evidence_tier` plus `deferral_reason`, and no `verified_with`), so +the generated spec is accepted by `add-package.py --provisional` unchanged. +`build.yml` has no dispatch input for the flags, because one build run is a matrix +over many plugins and the tier is per package: generate a provisional binary spec +by running `gen_spec.py` against a completed run's `packaged.tsv` and downloaded +assets. Either way the tier reaches the index from `add-package.py --provisional +--deferral-reason "..."`, which is what writes `evidence_tier` on the version entry. + +#### 4. Non-Binary Archive Intake (P2) + +Modules, scripts, and completions need no compilation, so they take the archive +lane instead of the cross-compile matrix: `scripts/intake_archive.py`, driven by +`.github/workflows/intake-archive.yml` (manual dispatch only). + +- Resolves the requested ref to a full 40-character upstream commit, + shallow-clones it, and verifies the declared entry file exists in the checkout. +- Builds a deterministic `.tar.gz` with `scripts/package_plugin.py`'s parameters + (sorted entries, fixed mtime, gzip mtime=0), so re-archiving the same commit + reproduces identical bytes. +- Publishes the archive as a release asset on this repository through the existing + release transaction (`ensure_release_absent.py`, then `release_transaction.py` + claim → upload → finalize, with cleanup on a failed run). The release tag is + `archive---` and the asset is + `--.tar.gz`. +- Emits an `artifact.kind: archive` spec carrying an inline `sha256`. That value is + not what the registry trusts — `add-package.py` downloads the asset and computes + the index hash itself — it is the digest of the archived bytes, which the publish + job re-hashes the collected artifact against before it claims a release, closing + the gap between what was archived and what gets published. +- Stages every activatable package provisionally: `add-package.py` requires + lifecycle evidence for any entry with an `activation`, and that evidence can only + come from proving the published asset, which does not exist until the release + completes. `numan-registry` replaces the provisional tier once prove succeeds. +- Records re-intake provenance (upstream URL, requested ref, resolved commit, + entry, owner, name, type) in `manifest-archives.json`. The workflow uploads the + updated file as an artifact instead of pushing a commit, so a maintainer commits + it alongside the registry handoff. +- Supports `--provisional` / `--deferral-reason` for a package whose + lifecycle-prove is deferred. + ### Deferred Until Upstream Changes - [ ] Pre-0.112 plugins stay deferred unless upstream bumps Nu minor or Numan elects a maintained fork under ADR 0001. @@ -168,6 +209,8 @@ Proposed forks evaluate under ADR 0001 stewardship criteria (requiring `numan-ma - [ ] Keep workflow permissions read-only except the release publication job. - [ ] Keep macOS runner labels current and covered by tests. - [ ] Keep deterministic archive tests for `.zip` and `.tar.gz`. +- [ ] Keep the non-binary archive lane's deterministic parameters, release tag + shape, and emitted spec shape covered by tests. - [ ] Keep release absence tests proving existing tags/assets fail before upload. - [ ] Keep manifest validation strict about duplicate package names, duplicate diff --git a/manifest-archives.json b/manifest-archives.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/manifest-archives.json @@ -0,0 +1 @@ +[] diff --git a/scripts/gen_spec.py b/scripts/gen_spec.py index 946e94f..50bc5f8 100644 --- a/scripts/gen_spec.py +++ b/scripts/gen_spec.py @@ -14,6 +14,11 @@ (from the manifest entry) so numan-registry `add-package.py` can pass it into the signed index. +`--provisional --deferral-reason ""` emits the provisional evidence tier +(`evidence_tier` plus `deferral_reason`) and drops `verified_with`, for a plugin +that builds but whose lifecycle-prove is deferred (e.g. it needs cloud +credentials). numan-registry's `add-package.py --provisional` accepts that spec. + Usage: python scripts/gen_spec.py \\ --name nu_plugin_regex \\ @@ -146,6 +151,8 @@ def build_spec( *, partial: bool = False, snapshot_date: str | None = None, + provisional: bool = False, + deferral_reason: str | None = None, ) -> dict: """ Build a registry specification from manifest metadata and packaged artifact records. @@ -160,14 +167,21 @@ def build_spec( snapshot_date (str | None): Override for the YYYYMMDD date used in a commit-snapshot version; defaults to today (UTC). Exists for deterministic testing. + provisional (bool): When True, emit `evidence_tier` and `deferral_reason` + in place of `verified_with` because lifecycle-prove is deferred. + deferral_reason (str | None): Why lifecycle-prove is deferred. Required + when `provisional` is True, rejected otherwise. Returns: dict: Registry specification containing plugin metadata and binary artifact targets. Raises: ValueError: If packaged targets include unexpected targets, if targets are - missing and `partial` is False, or if `partial` is True but no target - succeeded. + missing and `partial` is False, if `partial` is True but no target + succeeded, if `provisional` is True without a non-blank + `deferral_reason`, if a `deferral_reason` is given without + `provisional`, or if `provisional` is True while `entry` already + records `verified_with` evidence. """ actual = {row["target"] for row in packaged_rows} missing = sorted(set(expected) - actual) @@ -223,6 +237,34 @@ def build_spec( description += f" (numan-maintained fork; upstream: {upstream_repo})" source["upstream"] = f"https://github.com/{upstream_repo}" + if provisional: + if not (deferral_reason or "").strip(): + raise ValueError( + "provisional intake requires a non-blank deferral reason " + "(--deferral-reason)" + ) + if entry["verified_with"]: + raise ValueError( + "provisional intake rejected: verified_with is non-empty " + f"({', '.join(entry['verified_with'])}), so the plugin already has " + "lifecycle evidence" + ) + elif deferral_reason is not None: + raise ValueError( + "a deferral reason is only recorded for provisional intake; " + "pass --provisional or drop --deferral-reason" + ) + + # numan-registry's add-package.py aborts when --provisional is combined with a + # spec that merely CONTAINS verified_with, and its validate_spec only waives the + # lifecycle-evidence check when the key is absent, so a provisional spec has to + # omit verified_with rather than carry an empty list. + evidence = ( + {"evidence_tier": "provisional", "deferral_reason": deferral_reason.strip()} + if provisional + else {"verified_with": entry["verified_with"]} + ) + spec = { "owner": entry["owner"], "name": entry["name"], @@ -232,7 +274,7 @@ def build_spec( "tags": entry["tags"], "version": version, "nu_version": entry["nu_version"], - "verified_with": entry["verified_with"], + **evidence, "source": source, "artifact": { "kind": "binary", @@ -263,6 +305,18 @@ def main() -> int: "date the workflow used to derive the package/release version, or " "the generated spec's version won't match the published release tag", ) + ap.add_argument( + "--provisional", + action="store_true", + help="Emit evidence_tier + deferral_reason instead of verified_with, for a " + "plugin that builds but whose lifecycle-prove is deferred", + ) + ap.add_argument( + "--deferral-reason", + default=None, + help="Why lifecycle-prove is deferred; required with --provisional and " + "rejected without it", + ) args = ap.parse_args() manifest = json.loads((REPO_ROOT / "manifest.json").read_text(encoding="utf-8")) @@ -277,6 +331,8 @@ def main() -> int: expected_targets(manifest, entry), partial=args.partial, snapshot_date=args.snapshot_date, + provisional=args.provisional, + deferral_reason=args.deferral_reason, ) except ValueError as exc: print(f"FAIL: {exc}", file=sys.stderr) diff --git a/scripts/intake_archive.py b/scripts/intake_archive.py new file mode 100644 index 0000000..f0bfeec --- /dev/null +++ b/scripts/intake_archive.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python3 +"""Archive a non-binary package (module, script, completion) for registry intake. + +The plugin lane cross-compiles Rust; modules, scripts, and completions need no +compilation at all, so their intake is: + + 1. Resolve the requested ref (tag, branch, or commit) to its full 40-character + commit SHA with `git ls-remote`, the same immutable provenance anchor the + plugin lane records as `source_commit`. + 2. Shallow-clone the upstream repository and check out exactly that SHA. + 3. Verify the declared entry file exists inside the checkout. + 4. Archive the checkout (minus .git) as a deterministic `.tar.gz` -- sorted + entries, fixed mtime, gzip mtime=0 -- with package_plugin.py's parameters, + so re-archiving the same commit produces identical bytes. + 5. Emit a spec JSON in the shape numan-registry's scripts/add-package.py + expects for an `artifact.kind: archive` package, and record the intake in + manifest-archives.json for repeatable re-intake on a version bump. + +The archive spec carries an inline `artifact.sha256`. It is not what the registry +trusts -- add-package.py downloads the asset and computes the index hash itself, +ignoring this value -- it is the digest of the bytes archived here, carried +through so intake-archive.yml can re-hash the artifact it is about to publish +against it, and so a re-intake of the same commit can be compared. The spec also +omits a top-level `source` block: the registry index's source field is +Rust-shaped (it requires cargo_name) and non-binary entries leave it out, so +re-intake provenance lives in manifest-archives.json instead. + +An intake that declares an activation is always provisional. add-package.py +requires lifecycle evidence for every activatable entry, and that evidence can +only come from proving the published asset, which does not exist until this +lane's release completes; numan-registry replaces the provisional tier once prove +succeeds. numan-registry's own scripts/intake-archive.py works the same way. + +This script publishes nothing. .github/workflows/intake-archive.yml publishes the +archive through ensure_release_absent.py and release_transaction.py, so this +script stays hermetically testable and the audited release transaction is not +duplicated here. + +Usage: + python3 scripts/intake_archive.py \\ + --git-url https://github.com/owner/repo --ref v1.0.0 \\ + --entry mod.nu --owner owner --name cool-module --type module \\ + --description "..." --tags '["module"]' --nu-version ">=0.114.0" \\ + --release-root https://github.com/numan-cli/numan-plugins/releases/download \\ + --archive-out dist --out spec-owner-cool-module.json +""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +import re +import subprocess +import sys +import tarfile +import tempfile +from collections.abc import Callable +from pathlib import Path + +Runner = Callable[..., subprocess.CompletedProcess[str]] + +REPO_ROOT = Path(__file__).resolve().parent.parent +SHA_RE = re.compile(r"^[0-9a-f]{40}$") +COMMAND_TIMEOUT_SECONDS = 120 +FIXED_MTIME = 315532800 # 1980-01-01 UTC; matches package_plugin.py +VALID_TYPES = ("module", "script", "completion") +VALID_GIT_URL_RE = re.compile(r"^(https?://|git://|ssh://|git@[\w.-]+:)") +IDENT_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +VERSION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.+_-]*$") +MAX_ARCHIVE_FILES = 10_000 +MAX_ARCHIVE_BYTES = 100 * 1024 * 1024 + + +def normalize_git_url(value: str) -> str: + """Return a clone URL, expanding a bare `owner/name` slug to a GitHub URL.""" + if VALID_GIT_URL_RE.match(value): + return value + return f"https://github.com/{value}" + + +def validate_git_url(git_url: str) -> None: + """ + Validate that a clone URL uses a supported scheme and is not option-like. + + Raises: + ValueError: If the URL starts with '-' (which git would read as an + option) or does not use https://, http://, git://, ssh://, or git@. + """ + if git_url.startswith("-"): + raise ValueError(f"git URL may not start with '-': {git_url!r}") + if not VALID_GIT_URL_RE.match(git_url): + raise ValueError( + f"git URL must use https://, http://, git://, ssh://, or git@: {git_url!r}" + ) + + +def validate_identifier(field: str, value: str, pattern: re.Pattern[str] = IDENT_RE) -> None: + """ + Validate a value that is interpolated into the release tag and asset name. + + Raises: + ValueError: If the value is empty or contains anything outside + ``pattern``. An empty half of an `owner/name` pair would otherwise + produce a malformed tag (`archive-owner--1.0.0`) that the registry's + presence-only checks still accept, and a path-like value would write + the archive outside --archive-out. + """ + if not pattern.fullmatch(value): + raise ValueError(f"{field} must match {pattern.pattern}: {value!r}") + + +def resolve_ref(git_url: str, ref: str, runner: Runner = subprocess.run) -> str: + """ + Resolve a tag, branch, or commit ref to its full 40-character commit SHA. + + Annotated tags resolve to the commit they point at, then lightweight tags, + then branches, then any remaining ref shape. + + Returns: + str: The resolved commit SHA, or ``ref`` itself when it is already a + full SHA that the remote does not advertise. + + Raises: + ValueError: If the ref matches more than one remote ref, or cannot be + resolved and is not a full commit SHA. + """ + for candidate in ( + f"refs/tags/{ref}^{{}}", + f"refs/tags/{ref}", + f"refs/heads/{ref}", + ref, + ): + result = runner( + ["git", "ls-remote", git_url, candidate], + check=False, + capture_output=True, + text=True, + timeout=COMMAND_TIMEOUT_SECONDS, + ) + if result.returncode != 0 or not result.stdout.strip(): + continue + lines = result.stdout.strip().splitlines() + if len(lines) > 1: + raise ValueError(f"ref {ref!r} is ambiguous on {git_url}: {len(lines)} matches") + return lines[0].split()[0] + if SHA_RE.fullmatch(ref): + return ref + raise ValueError(f"could not resolve ref {ref!r} on {git_url}") + + +def run_git(args: list[str], runner: Runner, *, failure: str) -> None: + """Run a git command, raising ValueError with ``failure`` and git's stderr.""" + result = runner( + ["git", *args], + check=False, + capture_output=True, + text=True, + timeout=COMMAND_TIMEOUT_SECONDS, + ) + if result.returncode != 0: + raise ValueError(f"{failure}: {result.stderr.strip()}") + + +def shallow_clone_at( + git_url: str, + sha: str, + dest: Path, + runner: Runner = subprocess.run, +) -> None: + """Clone ``git_url`` into ``dest`` at depth 1 and check out exactly ``sha``.""" + run_git(["init", "--quiet", str(dest)], runner, failure=f"failed to init {dest}") + run_git( + ["-C", str(dest), "remote", "add", "origin", git_url], + runner, + failure=f"failed to add origin {git_url}", + ) + run_git( + ["-C", str(dest), "fetch", "--quiet", "--depth", "1", "origin", sha], + runner, + failure=f"failed to fetch {sha} from {git_url}", + ) + run_git( + ["-C", str(dest), "checkout", "--quiet", sha], + runner, + failure=f"failed to check out {sha}", + ) + + +def verify_entry(src_dir: Path, entry: str) -> Path: + """ + Resolve the declared entry file inside a checkout. + + Returns: + Path: The resolved entry file. + + Raises: + ValueError: If the entry path is absolute, resolves outside the + checkout, or is not a regular file. + """ + if Path(entry).is_absolute(): + raise ValueError(f"entry path must be relative to the checkout: {entry}") + resolved = (src_dir / entry).resolve() + try: + resolved.relative_to(src_dir.resolve()) + except ValueError as exc: + raise ValueError(f"entry path escapes checkout: {entry}") from exc + if not resolved.is_file(): + raise ValueError(f"entry file not found in checkout: {entry}") + return resolved + + +def sorted_files(root: Path) -> list[Path]: + """ + List the regular files under ``root`` (excluding .git) in archive order. + + Returns: + list[Path]: Paths relative to ``root``, sorted by POSIX path. + + Raises: + ValueError: If a symlink is present, or a path resolves outside ``root``. + """ + resolved_root = root.resolve() + files: list[Path] = [] + for path in root.rglob("*"): + rel = path.relative_to(root) + if path.is_symlink(): + raise ValueError(f"symlink not allowed in archive source: {rel.as_posix()}") + if not path.is_file(): + continue + try: + path.resolve().relative_to(resolved_root) + except ValueError as exc: + raise ValueError( + f"archive source path resolves outside checkout: {rel.as_posix()}" + ) from exc + if rel.parts[0] == ".git": + continue + files.append(rel) + return sorted(files, key=lambda rel: rel.as_posix()) + + +def build_archive(src_dir: Path, out: Path) -> None: + """ + Write a deterministic .tar.gz of ``src_dir``: sorted entries, fixed metadata. + + Raises: + ValueError: If the tree exceeds MAX_ARCHIVE_FILES or MAX_ARCHIVE_BYTES. + """ + rels = sorted_files(src_dir) + if len(rels) > MAX_ARCHIVE_FILES: + raise ValueError(f"{len(rels)} files exceeds the archive limit of {MAX_ARCHIVE_FILES}") + total = sum((src_dir / rel).stat().st_size for rel in rels) + if total > MAX_ARCHIVE_BYTES: + raise ValueError(f"{total} bytes exceeds the archive limit of {MAX_ARCHIVE_BYTES}") + + # Build beside the target and rename only once the archive is complete: a + # partial .tar.gz left behind by a failed write would trip main()'s + # refuse-to-overwrite guard and block the retry. + partial = out.with_name(out.name + ".partial") + try: + with partial.open("wb") as fh: + gz = gzip.GzipFile(filename="", mode="wb", fileobj=fh, mtime=0) + try: + with tarfile.open(fileobj=gz, mode="w", format=tarfile.PAX_FORMAT) as tar: + for rel in rels: + full = src_dir / rel + stat = full.stat() + info = tarfile.TarInfo(name=rel.as_posix()) + info.size = stat.st_size + info.mtime = FIXED_MTIME + info.mode = 0o755 if stat.st_mode & 0o111 else 0o644 + info.uid = info.gid = 0 + info.uname = info.gname = "" + with full.open("rb") as src: + tar.addfile(info, src) + finally: + gz.close() + partial.replace(out) + finally: + partial.unlink(missing_ok=True) + + +def derive_version(ref: str, resolved_sha: str) -> str: + """ + Derive the intake version when the caller supplies no explicit one. + + A semver-shaped ref (optionally 'v'-prefixed) becomes the version itself; + anything else falls back to the 0.1.0- convention the registry + already uses for branch-pinned script and completion entries. + """ + match = re.fullmatch(r"v?(\d+\.\d+\.\d+(?:[-+].+)?)", ref) + if match: + return match.group(1) + return f"0.1.0-{resolved_sha[:7]}" + + +def archive_filename(owner: str, name: str, version: str) -> str: + """Return the release asset filename for this intake.""" + return f"{owner}-{name}-{version}.tar.gz" + + +def release_tag(owner: str, name: str, version: str) -> str: + """Return the release tag for this intake.""" + return f"archive-{owner}-{name}-{version}" + + +def validate_activation( + *, + entry: str, + activation_kind: str | None, + activation_import: str | None, + provisional: bool, + deferral_reason: str | None, +) -> None: + """ + Check activation and provisional coherence before any work is done. + + Raises: + ValueError: If an activation is declared without provisional intake, if + provisional intake has no non-blank deferral reason, if a deferral + reason is given without provisional intake, or if a `mod.nu` entry + is activated with import mode 'module'. + """ + if activation_kind and not provisional: + raise ValueError( + "an activation requires provisional intake (no lifecycle evidence " + "was provided); pass --provisional with --deferral-reason" + ) + if provisional: + if not (deferral_reason or "").strip(): + raise ValueError( + "provisional intake requires a non-blank deferral reason " + "(--deferral-reason)" + ) + elif deferral_reason is not None: + raise ValueError( + "a deferral reason is only recorded for provisional intake; " + "pass --provisional or drop --deferral-reason" + ) + # Numan activates a module with `use ""`, the file form, so Nu's + # directory-name-becomes-module-name convention for mod.nu never applies: a + # mod.nu entry imported as 'module' would expose commands under a module + # literally named 'mod'. Import mode 'all' imports them unprefixed instead. + if ( + activation_kind == "nu-module" + and Path(entry).name == "mod.nu" + and (activation_import or "module") == "module" + ): + raise ValueError( + "a 'mod.nu' entry requires activation import 'all'; import 'module' " + "would activate a module named 'mod'" + ) + + +def parse_tags(raw: str) -> list[str]: + """ + Parse the --tags JSON array. + + Raises: + ValueError: If the value is not a JSON array of strings. + json.JSONDecodeError: If the value is not valid JSON. + """ + tags = json.loads(raw) + if not isinstance(tags, list) or not all(isinstance(tag, str) for tag in tags): + raise ValueError("--tags must be a JSON array of strings") + return tags + + +def build_spec( + *, + owner: str, + name: str, + description: str, + git_url: str, + pkg_type: str, + tags: list[str], + version: str, + nu_version: str, + entry: str, + url: str, + sha256: str, + activation_kind: str | None = None, + activation_import: str | None = None, + provisional: bool = False, + deferral_reason: str | None = None, +) -> dict: + """ + Build the numan-registry intake spec for an archive-kind package. + + `verified_with` is never emitted: add-package.py aborts when --provisional is + combined with a spec that merely contains the key, and a non-provisional + archive intake records its lifecycle evidence downstream after prove. + """ + evidence = ( + {"evidence_tier": "provisional", "deferral_reason": (deferral_reason or "").strip()} + if provisional + else {} + ) + spec: dict = { + "owner": owner, + "name": name, + "description": description, + "repo": git_url, + "type": pkg_type, + "tags": tags, + "version": version, + "nu_version": nu_version, + **evidence, + "artifact": { + "kind": "archive", + "url": url, + "entry": entry, + "sha256": sha256, + }, + } + if activation_kind: + activation = {"kind": activation_kind} + if activation_import: + activation["import"] = activation_import + spec["activation"] = activation + return spec + + +def record_archive_manifest( + path: Path, + *, + git_url: str, + ref: str, + resolved_sha: str, + entry: str, + name: str, + owner: str, + pkg_type: str, +) -> None: + """ + Upsert this intake's re-intake record into manifest-archives.json. + + Raises: + ValueError: If the existing file is not a JSON array of objects. + json.JSONDecodeError: If the existing file is not valid JSON. + """ + entries: list[dict] = [] + if path.exists(): + entries = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(entries, list) or not all( + isinstance(entry_record, dict) for entry_record in entries + ): + raise ValueError(f"{path} must contain a JSON array of objects") + record = { + "git": git_url, + "ref": ref, + "resolved_sha": resolved_sha, + "entry": entry, + "name": name, + "owner": owner, + "type": pkg_type, + } + entries = [ + existing + for existing in entries + if not (existing.get("owner") == owner and existing.get("name") == name) + ] + entries.append(record) + entries.sort(key=lambda existing: (existing.get("owner", ""), existing.get("name", ""))) + path.write_text(json.dumps(entries, indent=2) + "\n", encoding="utf-8") + + +def main(argv: list[str] | None = None) -> int: + """Archive an upstream ref, emit its registry spec, and record the intake.""" + ap = argparse.ArgumentParser() + ap.add_argument("--git-url", default=None, help="upstream clone URL") + ap.add_argument("--repo", default=None, help="upstream clone URL or owner/name slug") + ap.add_argument("--ref", required=True, help="upstream tag, branch, or commit to archive") + ap.add_argument("--entry", required=True, help="entry file path inside the repository") + ap.add_argument("--owner", required=True, help="registry package owner") + ap.add_argument("--name", required=True, help="registry package name") + ap.add_argument("--type", required=True, choices=VALID_TYPES, dest="pkg_type") + ap.add_argument("--description", required=True) + ap.add_argument("--tags", required=True, help="JSON array of tag strings") + ap.add_argument("--nu-version", required=True, help="Nu compatibility range") + ap.add_argument( + "--version", + default=None, + help="version for this intake; derived from --ref when omitted " + "(semver-shaped refs keep their version, others become 0.1.0-)", + ) + ap.add_argument("--activation-kind", default=None, help="activation kind, e.g. nu-module") + ap.add_argument("--activation-import", default=None, choices=("module", "all")) + ap.add_argument( + "--provisional", + action="store_true", + help="Emit evidence_tier + deferral_reason for an intake whose " + "lifecycle-prove is deferred", + ) + ap.add_argument( + "--deferral-reason", + default=None, + help="Why lifecycle-prove is deferred; required with --provisional and " + "rejected without it", + ) + ap.add_argument( + "--release-root", + required=True, + help="releases DOWNLOAD ROOT, e.g. " + "https://github.com/numan-cli/numan-plugins/releases/download; the " + "spec URL appends // once the version is derived", + ) + ap.add_argument("--archive-out", type=Path, default=Path("dist")) + ap.add_argument("--out", type=Path, required=True, help="spec JSON destination") + ap.add_argument( + "--manifest-archives", + type=Path, + default=REPO_ROOT / "manifest-archives.json", + ) + args = ap.parse_args(argv) + + try: + if not (args.git_url or args.repo): + raise ValueError("either --git-url or --repo is required") + git_url = normalize_git_url(args.git_url or args.repo) + validate_git_url(git_url) + validate_identifier("--owner", args.owner) + validate_identifier("--name", args.name) + validate_activation( + entry=args.entry, + activation_kind=args.activation_kind, + activation_import=args.activation_import, + provisional=args.provisional, + deferral_reason=args.deferral_reason, + ) + tags = parse_tags(args.tags) + + resolved_sha = resolve_ref(git_url, args.ref) + version = args.version or derive_version(args.ref, resolved_sha) + validate_identifier("--version", version, VERSION_RE) + tag = release_tag(args.owner, args.name, version) + archive_name = archive_filename(args.owner, args.name, version) + + args.archive_out.mkdir(parents=True, exist_ok=True) + archive_path = args.archive_out / archive_name + if archive_path.exists(): + raise ValueError(f"archive already exists, refusing to overwrite: {archive_path}") + + with tempfile.TemporaryDirectory() as tmp: + src_dir = Path(tmp) / "src" + shallow_clone_at(git_url, resolved_sha, src_dir) + verify_entry(src_dir, args.entry) + build_archive(src_dir, archive_path) + + digest = hashlib.sha256(archive_path.read_bytes()).hexdigest() + spec = build_spec( + owner=args.owner, + name=args.name, + description=args.description, + git_url=git_url, + pkg_type=args.pkg_type, + tags=tags, + version=version, + nu_version=args.nu_version, + entry=args.entry, + url=f"{args.release_root.rstrip('/')}/{tag}/{archive_name}", + sha256=digest, + activation_kind=args.activation_kind, + activation_import=args.activation_import, + provisional=args.provisional, + deferral_reason=args.deferral_reason, + ) + args.out.write_text(json.dumps(spec, indent=2) + "\n", encoding="utf-8") + record_archive_manifest( + args.manifest_archives, + git_url=git_url, + ref=args.ref, + resolved_sha=resolved_sha, + entry=args.entry, + name=args.name, + owner=args.owner, + pkg_type=args.pkg_type, + ) + + # Machine-readable line for the workflow to collect, like package_plugin.py's + # PACKAGED row: resolved_sha|version|tag|archive|sha256 + print(f"ARCHIVED\t{resolved_sha}\t{version}\t{tag}\t{archive_name}\t{digest}") + print( + f" archived {git_url}@{args.ref} -> {resolved_sha}\n" + f" wrote {archive_path} ({archive_path.stat().st_size} bytes) sha256={digest}\n" + f" wrote {args.out} and recorded re-intake in {args.manifest_archives}", + file=sys.stderr, + ) + except ( + OSError, + RuntimeError, + ValueError, + json.JSONDecodeError, + subprocess.TimeoutExpired, + ) as exc: + print(f"FAIL: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_docstring_coverage.py b/scripts/test_docstring_coverage.py index 1e5d377..06179c3 100644 --- a/scripts/test_docstring_coverage.py +++ b/scripts/test_docstring_coverage.py @@ -11,6 +11,7 @@ SAFETY_MODULES = ( "ensure_release_absent.py", "gen_spec.py", + "intake_archive.py", "release_transaction.py", "validate_manifest.py", ) diff --git a/scripts/test_gen_spec_source.py b/scripts/test_gen_spec_source.py index 4c178d9..d2da7b8 100644 --- a/scripts/test_gen_spec_source.py +++ b/scripts/test_gen_spec_source.py @@ -511,6 +511,122 @@ def test_build_spec_rejects_extra_targets(self): with self.assertRaisesRegex(ValueError, "unexpected targets"): self.gs.build_spec(entry, rows, "https://example.invalid", []) + def test_provisional_emits_evidence_tier_and_omits_verified_with(self): + entry = { + "owner": "galuszkak", "name": "nu_plugin_bigquery", "plugin_bin": "nu_plugin_bigquery", + "repo": "galuszkak/nu_plugin_bigquery", "tag": "v0.3.0", + "source_commit": "4" * 40, "version": "0.3.0", "nu_version": "*", + "verified_with": [], "description": "BigQuery access.", "tags": ["plugin"], + } + rows = [{"target": "linux", "filename": "p.tar.gz", "sha256": "a" * 64, "exe": "p"}] + out = self.gs.build_spec( + entry, + rows, + "https://example.invalid", + ["linux"], + provisional=True, + deferral_reason="needs GCP credentials", + ) + self.assertNotIn("verified_with", out) + self.assertEqual(out["evidence_tier"], "provisional") + self.assertEqual(out["deferral_reason"], "needs GCP credentials") + keys = list(out) + self.assertEqual( + keys[keys.index("nu_version") + 1 : keys.index("source")], + ["evidence_tier", "deferral_reason"], + ) + + def test_provisional_strips_deferral_reason_whitespace(self): + entry = { + "owner": "o", "name": "p", "plugin_bin": "p", "repo": "o/p", "tag": "v1", + "source_commit": "1" * 40, "version": "1.0.0", "nu_version": "*", + "verified_with": [], "description": "p", "tags": ["plugin"], + } + rows = [{"target": "linux", "filename": "p.tar.gz", "sha256": "a" * 64, "exe": "p"}] + out = self.gs.build_spec( + entry, + rows, + "https://example.invalid", + ["linux"], + provisional=True, + deferral_reason=" needs GCP credentials\n", + ) + self.assertEqual(out["deferral_reason"], "needs GCP credentials") + + def test_provisional_requires_deferral_reason(self): + entry = { + "owner": "o", "name": "p", "plugin_bin": "p", "repo": "o/p", "tag": "v1", + "source_commit": "1" * 40, "version": "1.0.0", "nu_version": "*", + "verified_with": [], "description": "p", "tags": ["plugin"], + } + rows = [{"target": "linux", "filename": "p.tar.gz", "sha256": "a" * 64, "exe": "p"}] + with self.assertRaisesRegex(ValueError, "deferral reason"): + self.gs.build_spec( + entry, rows, "https://example.invalid", ["linux"], provisional=True + ) + + def test_provisional_rejects_blank_deferral_reason(self): + entry = { + "owner": "o", "name": "p", "plugin_bin": "p", "repo": "o/p", "tag": "v1", + "source_commit": "1" * 40, "version": "1.0.0", "nu_version": "*", + "verified_with": [], "description": "p", "tags": ["plugin"], + } + rows = [{"target": "linux", "filename": "p.tar.gz", "sha256": "a" * 64, "exe": "p"}] + with self.assertRaisesRegex(ValueError, "deferral reason"): + self.gs.build_spec( + entry, + rows, + "https://example.invalid", + ["linux"], + provisional=True, + deferral_reason=" \t\n", + ) + + def test_provisional_rejects_entry_with_lifecycle_evidence(self): + entry = { + "owner": "o", "name": "p", "plugin_bin": "p", "repo": "o/p", "tag": "v1", + "source_commit": "1" * 40, "version": "1.0.0", "nu_version": "*", + "verified_with": ["0.114.1"], "description": "p", "tags": ["plugin"], + } + rows = [{"target": "linux", "filename": "p.tar.gz", "sha256": "a" * 64, "exe": "p"}] + with self.assertRaisesRegex(ValueError, "verified_with"): + self.gs.build_spec( + entry, + rows, + "https://example.invalid", + ["linux"], + provisional=True, + deferral_reason="needs GCP credentials", + ) + + def test_deferral_reason_without_provisional_is_rejected(self): + entry = { + "owner": "o", "name": "p", "plugin_bin": "p", "repo": "o/p", "tag": "v1", + "source_commit": "1" * 40, "version": "1.0.0", "nu_version": "*", + "verified_with": [], "description": "p", "tags": ["plugin"], + } + rows = [{"target": "linux", "filename": "p.tar.gz", "sha256": "a" * 64, "exe": "p"}] + with self.assertRaisesRegex(ValueError, "only recorded for provisional"): + self.gs.build_spec( + entry, + rows, + "https://example.invalid", + ["linux"], + deferral_reason="needs GCP credentials", + ) + + def test_non_provisional_spec_keeps_verified_with_only(self): + entry = { + "owner": "o", "name": "p", "plugin_bin": "p", "repo": "o/p", "tag": "v1", + "source_commit": "1" * 40, "version": "1.0.0", "nu_version": "*", + "verified_with": ["0.114.1"], "description": "p", "tags": ["plugin"], + } + rows = [{"target": "linux", "filename": "p.tar.gz", "sha256": "a" * 64, "exe": "p"}] + out = self.gs.build_spec(entry, rows, "https://example.invalid", ["linux"]) + self.assertEqual(out["verified_with"], ["0.114.1"]) + self.assertNotIn("evidence_tier", out) + self.assertNotIn("deferral_reason", out) + def _manifest_entry(self): return { "owner": "o", "name": "p", "plugin_bin": "p", "repo": "o/p", "tag": "v1", @@ -595,6 +711,90 @@ def test_main_failure_prints_fail_and_returns_1(self): self.assertEqual(rc, 1) self.assertFalse(out.is_file()) + def test_main_provisional_writes_evidence_tier_spec(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manifest_path = root / "manifest.json" + manifest_path.write_text( + json.dumps( + { + "default_targets": ["x86_64-unknown-linux-gnu"], + "active": [self._manifest_entry()], + } + ), + encoding="utf-8", + ) + assets_dir = root / "assets" + assets_dir.mkdir() + asset = assets_dir / "p-1.0.0-linux.tar.gz" + asset.write_bytes(b"data") + digest = hashlib.sha256(b"data").hexdigest() + packaged = root / "packaged.tsv" + packaged.write_text( + f"PACKAGED\tx86_64-unknown-linux-gnu\t{asset.name}\t{digest}\tp\n", + encoding="utf-8", + ) + out = root / "spec.json" + argv = [ + "gen_spec.py", + "--name", "p", + "--packaged", str(packaged), + "--assets-dir", str(assets_dir), + "--release-base", "https://example.invalid/release", + "--out", str(out), + "--provisional", + "--deferral-reason", "needs GCP credentials", + ] + with mock.patch.object(self.gs, "REPO_ROOT", root), mock.patch.object( + sys, "argv", argv + ): + rc = self.gs.main() + self.assertEqual(rc, 0) + spec = json.loads(out.read_text(encoding="utf-8")) + self.assertEqual(spec["evidence_tier"], "provisional") + self.assertEqual(spec["deferral_reason"], "needs GCP credentials") + self.assertNotIn("verified_with", spec) + + def test_main_provisional_without_reason_returns_1(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + manifest_path = root / "manifest.json" + manifest_path.write_text( + json.dumps( + { + "default_targets": ["x86_64-unknown-linux-gnu"], + "active": [self._manifest_entry()], + } + ), + encoding="utf-8", + ) + assets_dir = root / "assets" + assets_dir.mkdir() + asset = assets_dir / "p-1.0.0-linux.tar.gz" + asset.write_bytes(b"data") + digest = hashlib.sha256(b"data").hexdigest() + packaged = root / "packaged.tsv" + packaged.write_text( + f"PACKAGED\tx86_64-unknown-linux-gnu\t{asset.name}\t{digest}\tp\n", + encoding="utf-8", + ) + out = root / "spec.json" + argv = [ + "gen_spec.py", + "--name", "p", + "--packaged", str(packaged), + "--assets-dir", str(assets_dir), + "--release-base", "https://example.invalid/release", + "--out", str(out), + "--provisional", + ] + with mock.patch.object(self.gs, "REPO_ROOT", root), mock.patch.object( + sys, "argv", argv + ): + rc = self.gs.main() + self.assertEqual(rc, 1) + self.assertFalse(out.is_file()) + if __name__ == "__main__": suite = unittest.defaultTestLoader.loadTestsFromTestCase(BuildSpecSourceTests) diff --git a/scripts/test_intake_archive.py b/scripts/test_intake_archive.py new file mode 100644 index 0000000..abaedfe --- /dev/null +++ b/scripts/test_intake_archive.py @@ -0,0 +1,669 @@ +#!/usr/bin/env python3 +"""Unit checks for the non-binary archive intake lane.""" + +from __future__ import annotations + +import contextlib +import hashlib +import importlib.util +import io +import json +import os +import subprocess +import tarfile +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +SCRIPT = Path(__file__).resolve().parent / "intake_archive.py" +SHA = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0" + + +def load_intake_archive(): + spec = importlib.util.spec_from_file_location("intake_archive", SCRIPT) + mod = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(mod) + return mod + + +class RecordingRunner: + """Fake subprocess runner: records each argv and replays canned results.""" + + def __init__(self, results=()): + self.results = list(results) + self.calls: list[list[str]] = [] + + def __call__(self, args, **kwargs): + self.calls.append(list(args)) + returncode, stdout, stderr = self.results.pop(0) if self.results else (1, "", "boom") + return subprocess.CompletedProcess(list(args), returncode, stdout, stderr) + + @property + def refs(self) -> list[str]: + """Return the ref candidate of every recorded `git ls-remote` call.""" + return [call[3] for call in self.calls if call[1] == "ls-remote"] + + +class IntakeArchiveTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.ia = load_intake_archive() + + def _tree(self, root: Path) -> None: + (root / "nested").mkdir(parents=True) + (root / "mod.nu").write_text("export def hi [] { }\n", encoding="utf-8") + (root / "nested" / "extra.nu").write_text("# extra\n", encoding="utf-8") + (root / ".git").mkdir() + (root / ".git" / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8") + + def test_normalize_git_url_expands_owner_name_slug(self): + self.assertEqual( + self.ia.normalize_git_url("owner/repo"), "https://github.com/owner/repo" + ) + + def test_normalize_git_url_leaves_full_url_alone(self): + for url in ( + "https://github.com/owner/repo", + "git://example.invalid/repo.git", + "ssh://git@example.invalid/repo.git", + "git@github.com:owner/repo.git", + ): + self.assertEqual(self.ia.normalize_git_url(url), url) + + def test_validate_git_url_rejects_option_like_url(self): + with self.assertRaisesRegex(ValueError, "may not start with"): + self.ia.validate_git_url("--upload-pack=evil") + + def test_validate_git_url_rejects_unsupported_scheme(self): + with self.assertRaisesRegex(ValueError, "must use https"): + self.ia.validate_git_url("file:///tmp/repo") + + def test_validate_git_url_accepts_supported_schemes(self): + self.ia.validate_git_url("https://github.com/owner/repo") + self.ia.validate_git_url("git@github.com:owner/repo.git") + + def test_validate_identifier_rejects_empty_and_path_like_values(self): + for value in ("", " ", "..", "../escape", "owner/name", "-flag"): + with self.subTest(value=value): + with self.assertRaisesRegex(ValueError, "--name must match"): + self.ia.validate_identifier("--name", value) + + def test_validate_identifier_accepts_registry_and_version_shapes(self): + self.ia.validate_identifier("--owner", "nushell") + self.ia.validate_identifier("--name", "cool_module.nu-2") + self.ia.validate_identifier("--version", "0.1.0-abc1234", self.ia.VERSION_RE) + self.ia.validate_identifier("--version", "1.2.3+build.5", self.ia.VERSION_RE) + + def test_resolve_ref_returns_annotated_tag_sha(self): + runner = RecordingRunner([(0, f"{SHA}\trefs/tags/v1.0.0^{{}}\n", "")]) + resolved = self.ia.resolve_ref( + "https://github.com/owner/repo", "v1.0.0", runner=runner + ) + self.assertEqual(resolved, SHA) + self.assertEqual(runner.refs, ["refs/tags/v1.0.0^{}"]) + + def test_resolve_ref_falls_through_to_branch(self): + runner = RecordingRunner( + [(1, "", ""), (0, "", ""), (0, f"{SHA}\trefs/heads/main\n", "")] + ) + resolved = self.ia.resolve_ref("https://github.com/owner/repo", "main", runner=runner) + self.assertEqual(resolved, SHA) + self.assertEqual( + runner.refs, ["refs/tags/main^{}", "refs/tags/main", "refs/heads/main"] + ) + + def test_resolve_ref_rejects_ambiguous_match(self): + runner = RecordingRunner( + [(0, f"{SHA}\trefs/tags/x\n{'b' * 40}\trefs/tags/x-suffix\n", "")] + ) + with self.assertRaisesRegex(ValueError, "ambiguous"): + self.ia.resolve_ref("https://github.com/owner/repo", "x", runner=runner) + + def test_resolve_ref_accepts_full_sha_when_unadvertised(self): + runner = RecordingRunner() + self.assertEqual( + self.ia.resolve_ref("https://github.com/owner/repo", SHA, runner=runner), SHA + ) + + def test_resolve_ref_rejects_unresolvable_ref(self): + runner = RecordingRunner() + with self.assertRaisesRegex(ValueError, "could not resolve"): + self.ia.resolve_ref("https://github.com/owner/repo", "nope", runner=runner) + + def test_resolve_ref_tries_candidates_in_order(self): + runner = RecordingRunner() + with self.assertRaises(ValueError): + self.ia.resolve_ref("https://github.com/owner/repo", "topic", runner=runner) + self.assertEqual( + runner.refs, + ["refs/tags/topic^{}", "refs/tags/topic", "refs/heads/topic", "topic"], + ) + + def test_shallow_clone_at_runs_init_fetch_checkout(self): + runner = RecordingRunner([(0, "", "")] * 4) + self.ia.shallow_clone_at( + "https://github.com/owner/repo", SHA, Path("src"), runner=runner + ) + self.assertEqual([call[1] for call in runner.calls], ["init", "-C", "-C", "-C"]) + self.assertIn("fetch", runner.calls[2]) + self.assertIn("--depth", runner.calls[2]) + self.assertEqual(runner.calls[3][-1], SHA) + + def test_shallow_clone_at_raises_with_git_stderr(self): + runner = RecordingRunner([(0, "", ""), (0, "", ""), (128, "", "bad object")]) + with self.assertRaisesRegex(ValueError, "bad object"): + self.ia.shallow_clone_at( + "https://github.com/owner/repo", SHA, Path("src"), runner=runner + ) + + def test_verify_entry_accepts_nested_path(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._tree(root) + self.assertEqual( + self.ia.verify_entry(root, "nested/extra.nu"), + (root / "nested" / "extra.nu").resolve(), + ) + + def test_verify_entry_rejects_absolute_path(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._tree(root) + with self.assertRaisesRegex(ValueError, "must be relative"): + self.ia.verify_entry(root, str(root / "mod.nu")) + + def test_verify_entry_rejects_escaping_path(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "src" + root.mkdir() + (Path(tmp) / "outside.nu").write_text("# outside\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "escapes checkout"): + self.ia.verify_entry(root, "../outside.nu") + + def test_verify_entry_rejects_missing_file(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._tree(root) + with self.assertRaisesRegex(ValueError, "not found in checkout"): + self.ia.verify_entry(root, "missing.nu") + + def test_sorted_files_skips_git_and_sorts_deterministically(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._tree(root) + (root / "aaa.nu").write_text("# a\n", encoding="utf-8") + self.assertEqual( + [rel.as_posix() for rel in self.ia.sorted_files(root)], + ["aaa.nu", "mod.nu", "nested/extra.nu"], + ) + + def test_sorted_files_rejects_symlink(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._tree(root) + try: + (root / "link.nu").symlink_to(root / "mod.nu") + except OSError: + self.skipTest("symlink creation not permitted on this host") + with self.assertRaisesRegex(ValueError, "symlink not allowed"): + self.ia.sorted_files(root) + + def test_build_archive_is_byte_identical_across_runs(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + src = root / "src" + self._tree(src) + first = root / "first.tar.gz" + second = root / "second.tar.gz" + self.ia.build_archive(src, first) + self.ia.build_archive(src, second) + self.assertEqual(first.read_bytes(), second.read_bytes()) + + def test_build_archive_zeroes_the_gzip_mtime(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + src = root / "src" + self._tree(src) + out = root / "out.tar.gz" + self.ia.build_archive(src, out) + self.assertEqual(out.read_bytes()[4:8], b"\x00\x00\x00\x00") + + def test_build_archive_normalizes_member_metadata(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + src = root / "src" + self._tree(src) + out = root / "out.tar.gz" + self.ia.build_archive(src, out) + with tarfile.open(out, "r:gz") as tar: + members = tar.getmembers() + self.assertEqual( + [member.name for member in members], + ["mod.nu", "nested/extra.nu"], + ) + for member in members: + self.assertEqual(member.mtime, self.ia.FIXED_MTIME) + self.assertEqual((member.uid, member.gid), (0, 0)) + self.assertEqual((member.uname, member.gname), ("", "")) + + def test_build_archive_normalizes_member_modes(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + src = root / "src" + src.mkdir() + (src / "plain.nu").write_text("# plain\n", encoding="utf-8") + runnable = src / "run.nu" + runnable.write_text("# run\n", encoding="utf-8") + os.chmod(runnable, 0o755) + if not runnable.stat().st_mode & 0o111: + self.skipTest("host filesystem does not record the exec bit") + out = root / "out.tar.gz" + self.ia.build_archive(src, out) + with tarfile.open(out, "r:gz") as tar: + modes = {member.name: member.mode for member in tar.getmembers()} + self.assertEqual(modes, {"plain.nu": 0o644, "run.nu": 0o755}) + + def test_build_archive_leaves_no_partial_archive_on_failure(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + src = root / "src" + self._tree(src) + dist = root / "dist" + dist.mkdir() + with mock.patch.object( + self.ia.tarfile, "open", side_effect=OSError("no space left on device") + ): + with self.assertRaisesRegex(OSError, "no space left"): + self.ia.build_archive(src, dist / "out.tar.gz") + self.assertEqual(list(dist.iterdir()), []) + + def test_build_archive_rejects_too_many_files(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + src = root / "src" + self._tree(src) + with mock.patch.object(self.ia, "MAX_ARCHIVE_FILES", 1): + with self.assertRaisesRegex(ValueError, "exceeds the archive limit of 1"): + self.ia.build_archive(src, root / "out.tar.gz") + + def test_build_archive_rejects_oversized_tree(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + src = root / "src" + self._tree(src) + with mock.patch.object(self.ia, "MAX_ARCHIVE_BYTES", 4): + with self.assertRaisesRegex(ValueError, "bytes exceeds the archive limit of 4"): + self.ia.build_archive(src, root / "out.tar.gz") + + def test_derive_version_uses_semver_shaped_ref(self): + self.assertEqual(self.ia.derive_version("v1.2.3", SHA), "1.2.3") + self.assertEqual(self.ia.derive_version("1.2.3", SHA), "1.2.3") + self.assertEqual(self.ia.derive_version("v1.2.3-rc.1", SHA), "1.2.3-rc.1") + + def test_derive_version_falls_back_to_short_sha(self): + self.assertEqual(self.ia.derive_version("main", SHA), f"0.1.0-{SHA[:7]}") + + def test_release_tag_and_archive_filename_shapes(self): + self.assertEqual( + self.ia.release_tag("owner", "cool-module", "1.2.3"), + "archive-owner-cool-module-1.2.3", + ) + self.assertEqual( + self.ia.archive_filename("owner", "cool-module", "1.2.3"), + "owner-cool-module-1.2.3.tar.gz", + ) + + def _spec_kwargs(self, **overrides): + kwargs = { + "owner": "owner", + "name": "cool-module", + "description": "A cool module.", + "git_url": "https://github.com/owner/repo", + "pkg_type": "module", + "tags": ["module"], + "version": "1.2.3", + "nu_version": ">=0.114.0 <0.115.0", + "entry": "cool.nu", + "url": "https://example.invalid/download/archive-owner-cool-module-1.2.3/owner-cool-module-1.2.3.tar.gz", + "sha256": "c" * 64, + } + kwargs.update(overrides) + return kwargs + + def test_build_spec_emits_archive_artifact_with_inline_sha256(self): + spec = self.ia.build_spec(**self._spec_kwargs()) + self.assertEqual( + spec["artifact"], + { + "kind": "archive", + "url": self._spec_kwargs()["url"], + "entry": "cool.nu", + "sha256": "c" * 64, + }, + ) + self.assertNotIn("verified_with", spec) + self.assertNotIn("source", spec) + self.assertEqual(spec["repo"], "https://github.com/owner/repo") + + def test_build_spec_omits_activation_without_kind(self): + spec = self.ia.build_spec(**self._spec_kwargs(activation_import="all")) + self.assertNotIn("activation", spec) + + def test_build_spec_emits_activation_when_kind_given(self): + spec = self.ia.build_spec( + **self._spec_kwargs(activation_kind="nu-module", activation_import="all") + ) + self.assertEqual(spec["activation"], {"kind": "nu-module", "import": "all"}) + + def test_build_spec_provisional_emits_stripped_evidence_tier(self): + spec = self.ia.build_spec( + **self._spec_kwargs(provisional=True, deferral_reason=" needs a Nu 0.114 host\n") + ) + self.assertEqual(spec["evidence_tier"], "provisional") + self.assertEqual(spec["deferral_reason"], "needs a Nu 0.114 host") + self.assertNotIn("verified_with", spec) + keys = list(spec) + self.assertEqual( + keys[keys.index("nu_version") + 1 : keys.index("artifact")], + ["evidence_tier", "deferral_reason"], + ) + + def test_build_spec_non_provisional_omits_evidence_keys(self): + spec = self.ia.build_spec(**self._spec_kwargs()) + self.assertNotIn("evidence_tier", spec) + self.assertNotIn("deferral_reason", spec) + + def _activation_kwargs(self, **overrides): + kwargs = { + "entry": "cool.nu", + "activation_kind": None, + "activation_import": None, + "provisional": False, + "deferral_reason": None, + } + kwargs.update(overrides) + return kwargs + + def test_validate_activation_requires_provisional(self): + with self.assertRaisesRegex(ValueError, "requires provisional intake"): + self.ia.validate_activation( + **self._activation_kwargs(activation_kind="nu-module") + ) + + def test_validate_activation_requires_deferral_reason(self): + with self.assertRaisesRegex(ValueError, "deferral reason"): + self.ia.validate_activation(**self._activation_kwargs(provisional=True)) + + def test_validate_activation_rejects_blank_deferral_reason(self): + with self.assertRaisesRegex(ValueError, "deferral reason"): + self.ia.validate_activation( + **self._activation_kwargs(provisional=True, deferral_reason=" \t\n") + ) + + def test_validate_activation_rejects_reason_without_provisional(self): + with self.assertRaisesRegex(ValueError, "only recorded for provisional"): + self.ia.validate_activation( + **self._activation_kwargs(deferral_reason="needs a Nu host") + ) + + def test_validate_activation_rejects_mod_nu_with_module_import(self): + with self.assertRaisesRegex(ValueError, "requires activation import 'all'"): + self.ia.validate_activation( + **self._activation_kwargs( + entry="mod.nu", + activation_kind="nu-module", + activation_import="module", + provisional=True, + deferral_reason="needs a Nu host", + ) + ) + + def test_validate_activation_rejects_mod_nu_with_default_import(self): + with self.assertRaisesRegex(ValueError, "requires activation import 'all'"): + self.ia.validate_activation( + **self._activation_kwargs( + entry="pkg/mod.nu", + activation_kind="nu-module", + provisional=True, + deferral_reason="needs a Nu host", + ) + ) + + def test_validate_activation_allows_mod_nu_with_import_all(self): + self.ia.validate_activation( + **self._activation_kwargs( + entry="mod.nu", + activation_kind="nu-module", + activation_import="all", + provisional=True, + deferral_reason="needs a Nu host", + ) + ) + + def test_validate_activation_allows_named_entry_with_module_import(self): + self.ia.validate_activation( + **self._activation_kwargs( + entry="foo.nu", + activation_kind="nu-module", + activation_import="module", + provisional=True, + deferral_reason="needs a Nu host", + ) + ) + + def test_parse_tags_accepts_json_array_of_strings(self): + self.assertEqual(self.ia.parse_tags('["module", "nu"]'), ["module", "nu"]) + + def test_parse_tags_rejects_non_array(self): + with self.assertRaisesRegex(ValueError, "JSON array of strings"): + self.ia.parse_tags('{"tags": []}') + + def test_parse_tags_rejects_non_string_members(self): + with self.assertRaisesRegex(ValueError, "JSON array of strings"): + self.ia.parse_tags("[1, 2]") + + def _record(self, path: Path, **overrides): + kwargs = { + "git_url": "https://github.com/owner/repo", + "ref": "v1.2.3", + "resolved_sha": SHA, + "entry": "cool.nu", + "name": "cool-module", + "owner": "owner", + "pkg_type": "module", + } + kwargs.update(overrides) + self.ia.record_archive_manifest(path, **kwargs) + + def test_record_archive_manifest_creates_the_file(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "manifest-archives.json" + self._record(path) + text = path.read_text(encoding="utf-8") + self.assertTrue(text.endswith("\n")) + self.assertEqual( + json.loads(text), + [ + { + "git": "https://github.com/owner/repo", + "ref": "v1.2.3", + "resolved_sha": SHA, + "entry": "cool.nu", + "name": "cool-module", + "owner": "owner", + "type": "module", + } + ], + ) + + def test_record_archive_manifest_upserts_and_sorts(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "manifest-archives.json" + self._record(path, owner="zowner", name="zed") + self._record(path) + self._record(path, ref="v2.0.0", resolved_sha="b" * 40) + entries = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual( + [(entry["owner"], entry["name"]) for entry in entries], + [("owner", "cool-module"), ("zowner", "zed")], + ) + self.assertEqual(entries[0]["ref"], "v2.0.0") + self.assertEqual(entries[0]["resolved_sha"], "b" * 40) + + def test_record_archive_manifest_rejects_non_array_json(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "manifest-archives.json" + path.write_text('{"owner": "owner"}', encoding="utf-8") + with self.assertRaisesRegex(ValueError, "JSON array of objects"): + self._record(path) + + def test_record_archive_manifest_rejects_non_object_member(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "manifest-archives.json" + path.write_text('["owner/cool-module"]', encoding="utf-8") + with self.assertRaisesRegex(ValueError, "JSON array of objects"): + self._record(path) + + def _main_argv(self, root: Path, *extra: str) -> list[str]: + return [ + "--git-url", "https://github.com/owner/repo", + "--ref", "v1.2.3", + "--entry", "mod.nu", + "--owner", "owner", + "--name", "cool-module", + "--type", "module", + "--description", "A cool module.", + "--tags", '["module"]', + "--nu-version", ">=0.114.0 <0.115.0", + "--release-root", "https://example.invalid/releases/download/", + "--archive-out", str(root / "dist"), + "--out", str(root / "spec.json"), + "--manifest-archives", str(root / "manifest-archives.json"), + *extra, + ] + + def _clone_stub(self, tree=True): + def stub(git_url, sha, dest, runner=None): + dest.mkdir(parents=True, exist_ok=True) + if tree: + self._tree(dest) + return stub + + @contextlib.contextmanager + def _patched_network(self, clone_stub): + stdout = io.StringIO() + with mock.patch.object(self.ia, "resolve_ref", return_value=SHA), mock.patch.object( + self.ia, "shallow_clone_at", clone_stub + ), contextlib.redirect_stdout(stdout): + yield stdout + + def test_main_writes_archive_spec_and_manifest_record(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + with self._patched_network(self._clone_stub()) as stdout: + rc = self.ia.main(self._main_argv(root)) + self.assertEqual(rc, 0) + archive = root / "dist" / "owner-cool-module-1.2.3.tar.gz" + self.assertTrue(archive.is_file()) + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + spec = json.loads((root / "spec.json").read_text(encoding="utf-8")) + self.assertEqual( + spec["artifact"], + { + "kind": "archive", + "url": ( + "https://example.invalid/releases/download/" + "archive-owner-cool-module-1.2.3/owner-cool-module-1.2.3.tar.gz" + ), + "entry": "mod.nu", + "sha256": digest, + }, + ) + self.assertEqual(spec["version"], "1.2.3") + records = json.loads((root / "manifest-archives.json").read_text(encoding="utf-8")) + self.assertEqual(records[0]["resolved_sha"], SHA) + self.assertEqual(records[0]["ref"], "v1.2.3") + self.assertIn( + "ARCHIVED\t" + f"{SHA}\t1.2.3\tarchive-owner-cool-module-1.2.3\t" + f"owner-cool-module-1.2.3.tar.gz\t{digest}", + stdout.getvalue(), + ) + + def test_main_provisional_spec_carries_evidence_tier(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + argv = self._main_argv( + root, "--provisional", "--deferral-reason", "needs a Nu 0.114 host" + ) + with self._patched_network(self._clone_stub()): + rc = self.ia.main(argv) + self.assertEqual(rc, 0) + spec = json.loads((root / "spec.json").read_text(encoding="utf-8")) + self.assertEqual(spec["evidence_tier"], "provisional") + self.assertEqual(spec["deferral_reason"], "needs a Nu 0.114 host") + self.assertNotIn("verified_with", spec) + + def test_main_missing_entry_writes_neither_spec_nor_record(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + argv = self._main_argv(root) + argv[argv.index("--entry") + 1] = "missing.nu" + with self._patched_network(self._clone_stub()): + rc = self.ia.main(argv) + self.assertEqual(rc, 1) + self.assertFalse((root / "spec.json").exists()) + self.assertFalse((root / "manifest-archives.json").exists()) + self.assertEqual(list((root / "dist").iterdir()), []) + + def test_main_accepts_a_repo_slug(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + argv = self._main_argv(root) + argv[argv.index("--git-url")] = "--repo" + argv[argv.index("--repo") + 1] = "owner/repo" + with self._patched_network(self._clone_stub()): + rc = self.ia.main(argv) + self.assertEqual(rc, 0) + spec = json.loads((root / "spec.json").read_text(encoding="utf-8")) + self.assertEqual(spec["repo"], "https://github.com/owner/repo") + + def test_main_requires_a_repository(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + argv = self._main_argv(root) + index = argv.index("--git-url") + del argv[index : index + 2] + with self._patched_network(self._clone_stub()): + rc = self.ia.main(argv) + self.assertEqual(rc, 1) + self.assertFalse((root / "spec.json").exists()) + + def test_main_rejects_a_path_like_package_name(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + argv = self._main_argv(root) + argv[argv.index("--name") + 1] = "../escape" + with self._patched_network(self._clone_stub()): + rc = self.ia.main(argv) + self.assertEqual(rc, 1) + self.assertFalse((root / "spec.json").exists()) + self.assertFalse((root / "dist").exists()) + + def test_main_refuses_to_overwrite_an_existing_archive(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "dist").mkdir() + archive = root / "dist" / "owner-cool-module-1.2.3.tar.gz" + archive.write_bytes(b"published already") + with self._patched_network(self._clone_stub()): + rc = self.ia.main(self._main_argv(root)) + self.assertEqual(rc, 1) + self.assertEqual(archive.read_bytes(), b"published already") + self.assertFalse((root / "spec.json").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_workflow_safety.py b/scripts/test_workflow_safety.py index b617b8c..27b314e 100644 --- a/scripts/test_workflow_safety.py +++ b/scripts/test_workflow_safety.py @@ -11,6 +11,7 @@ ROOT = Path(__file__).resolve().parent.parent BUILD = (ROOT / ".github" / "workflows" / "build.yml").read_text(encoding="utf-8") SAFETY = (ROOT / ".github" / "workflows" / "repo-safety.yml").read_text(encoding="utf-8") +INTAKE = (ROOT / ".github" / "workflows" / "intake-archive.yml").read_text(encoding="utf-8") WORKFLOW_DIR = ROOT / ".github" / "workflows" WORKFLOW_PATHS = sorted({*WORKFLOW_DIR.glob("*.yml"), *WORKFLOW_DIR.glob("*.yaml")}) WORKFLOWS = [path.read_text(encoding="utf-8") for path in WORKFLOW_PATHS] @@ -20,6 +21,10 @@ r"^(\s*)TARGETS\s*=\s*(\[[\s\S]*?\n\1\])", re.MULTILINE, ) +UPLOAD_STEPS = ( + ("build.yml", BUILD, " - name: Upload assets to the owned draft\n"), + ("intake-archive.yml", INTAKE, " - name: Upload the asset to the owned draft\n"), +) def workflow_targets(workflow: str) -> list[dict[str, object]]: @@ -48,6 +53,33 @@ def workflow_targets(workflow: str) -> list[dict[str, object]]: return parsed +def shell_bodies(workflow: str) -> list[str]: + """ + Collect every ``run:`` body line in a workflow, inline and block form. + + Parameters: + workflow: Workflow text to scan. + + Returns: + Every line that the runner executes as shell text. + """ + lines = workflow.splitlines() + shell_lines: list[str] = [] + for index, line in enumerate(lines): + match = re.match(r"^(\s*)run:\s*(.*)$", line) + if match is None: + continue + indent = len(match.group(1)) + remainder = match.group(2) + if remainder not in ("|", ">-", ""): + shell_lines.append(remainder) + for body_line in lines[index + 1 :]: + if body_line.strip() and len(body_line) - len(body_line.lstrip()) <= indent: + break + shell_lines.append(body_line) + return shell_lines + + class WorkflowSafetyTests(unittest.TestCase): def test_publication_is_manual_dispatch_only(self): """ @@ -69,21 +101,7 @@ def test_every_action_is_pinned_to_a_commit(self): def test_publication_shell_never_interpolates_expressions(self): """Keep workflow expressions in env/with fields, never executable shell text.""" - lines = BUILD.splitlines() - shell_lines: list[str] = [] - for index, line in enumerate(lines): - match = re.match(r"^(\s*)run:\s*(.*)$", line) - if match is None: - continue - indent = len(match.group(1)) - remainder = match.group(2) - if remainder not in ("|", ">-", ""): - shell_lines.append(remainder) - for body_line in lines[index + 1 :]: - if body_line.strip() and len(body_line) - len(body_line.lstrip()) <= indent: - break - shell_lines.append(body_line) - self.assertNotIn("${{", "\n".join(shell_lines)) + self.assertNotIn("${{", "\n".join(shell_bodies(BUILD))) def test_only_release_job_can_write_contents(self): global_permissions, jobs = BUILD.split("jobs:", 1) @@ -96,12 +114,12 @@ def test_only_release_job_can_write_contents(self): def test_release_upload_uses_claimed_release_id(self): """Avoid softprops creating a second draft when the tag is briefly undiscoverable.""" - self.assertNotIn("softprops/action-gh-release", BUILD) - upload_step = BUILD.split( - " - name: Upload assets to the owned draft\n", 1 - )[1].split("\n - name:", 1)[0] - self.assertIn("release_transaction.py upload", upload_step) - self.assertIn("--release-id \"$CLAIMED_RELEASE_ID\"", upload_step) + for label, workflow, step_header in UPLOAD_STEPS: + with self.subTest(workflow=label): + self.assertNotIn("softprops/action-gh-release", workflow) + upload_step = workflow.split(step_header, 1)[1].split("\n - name:", 1)[0] + self.assertIn("release_transaction.py upload", upload_step) + self.assertIn('--release-id "$CLAIMED_RELEASE_ID"', upload_step) def test_matrix_env_shell_steps_force_bash(self): """Steps that expand $MATRIX_* must use bash so Windows pwsh does not empty them.""" @@ -124,6 +142,57 @@ def test_matrix_env_shell_steps_force_bash(self): f"step must force bash when expanding MATRIX env vars:\n{text}", ) + def test_archive_intake_is_manual_dispatch_only(self): + """The non-binary lane publishes a release, so it may never run on a push or PR.""" + trigger_block = INTAKE.split("permissions:", 1)[0] + self.assertIn(" workflow_dispatch:\n", trigger_block) + self.assertNotIn(" pull_request:\n", trigger_block) + self.assertNotIn(" push:\n", trigger_block) + + def test_archive_intake_shell_never_interpolates_expressions(self): + """Dispatch inputs reach the archive lane through env, never as shell text.""" + self.assertNotIn("${{", "\n".join(shell_bodies(INTAKE))) + + def test_only_archive_publish_job_can_write_contents(self): + global_permissions, jobs = INTAKE.split("jobs:", 1) + self.assertIn("permissions:\n contents: read", global_permissions) + self.assertEqual(INTAKE.count("contents: write"), 1) + publish_job = jobs.split(" publish:\n", 1)[1] + self.assertIn(" permissions:\n contents: write", publish_job) + + def test_archive_publish_rehashes_the_downloaded_asset(self): + """Hash-gate the artifact round-trip, as build.yml does before publishing.""" + publish_job = INTAKE.split("jobs:", 1)[1].split(" publish:\n", 1)[1] + gate = publish_job.split( + " - name: Verify the downloaded asset is the archived asset\n", 1 + )[1].split("\n - name:", 1)[0] + self.assertIn("ARCHIVE: ${{ needs.archive.outputs.archive }}", gate) + self.assertIn("ARCHIVE_SHA256: ${{ needs.archive.outputs.sha256 }}", gate) + self.assertIn("sha256sum --check --strict", gate) + # release_transaction.py upload publishes whatever dist holds, so the + # digest check alone would pass while an extra file rode along. + self.assertIn(r"mapfile -t assets < <(find dist -type f -printf '%P\n' | sort)", gate) + self.assertIn('[ "${#assets[@]}" -ne 1 ]', gate) + self.assertIn('[ "${assets[0]}" != "$ARCHIVE" ]', gate) + self.assertLess(gate.index("-ne 1"), gate.index("sha256sum")) + self.assertLess( + publish_job.index("sha256sum"), + publish_job.index("ensure_release_absent.py"), + ) + self.assertLess( + publish_job.index("sha256sum"), + publish_job.index("release_transaction.py upload"), + ) + + def test_archive_intake_publishes_through_the_release_transaction(self): + """Reuse the audited claim/upload/finalize flow instead of `gh release create`.""" + self.assertIn("ensure_release_absent.py", INTAKE) + self.assertIn("release_transaction.py claim", INTAKE) + self.assertIn("release_transaction.py upload", INTAKE) + self.assertIn("release_transaction.py finalize", INTAKE) + self.assertIn("release_transaction.py cleanup", INTAKE) + self.assertNotIn("gh release create", INTAKE) + def test_macos_uses_supported_runners(self): """Keep the executable matrix and manifest metadata on current macOS runners.""" targets = workflow_targets(BUILD)