ci: publish releases and pages, add weekly health check

.github/workflows/health.yml

@@ -0,0 +1,81 @@
+name: Health
+
+on:
+  schedule:
+    - cron: "23 6 * * 1"
+  workflow_dispatch:
+
+permissions: {}
+
+concurrency:
+  group: health
+  cancel-in-progress: false
+
+jobs:
+  check:
+    name: Check entries
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    permissions:
+      contents: read
+    outputs:
+      findings: ${{ steps.check.outputs.findings }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.14"
+      - run: python -m pip install --require-hashes -r requirements.txt
+      - name: Check every active entry
+        id: check
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          python scripts/health.py --scan-sources --out build/health.md
+          if [ -s build/health.md ]; then
+            echo "findings=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "findings=false" >> "$GITHUB_OUTPUT"
+          fi
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        if: steps.check.outputs.findings == 'true'
+        with:
+          name: health
+          path: build/health.md
+          if-no-files-found: error
+          retention-days: 7
+
+  issue:
+    name: Update the health issue
+    needs: check
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    permissions:
+      issues: write
+    steps:
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        if: needs.check.outputs.findings == 'true'
+        with:
+          name: health
+          path: ${{ runner.temp }}/health
+      - name: Create, update or close the issue
+        env:
+          GH_TOKEN: ${{ github.token }}
+          FINDINGS: ${{ needs.check.outputs.findings }}
+          REPORT: ${{ runner.temp }}/health/health.md
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          number=$(gh issue list --repo "$GITHUB_REPOSITORY" --label health --author "github-actions[bot]" --state open \
+            --json number --jq '.[0].number // empty')
+          if [ "$FINDINGS" = "true" ]; then
+            printf '\n[Workflow run](%s)\n' "$RUN_URL" >> "$REPORT"
+            if [ -n "$number" ]; then
+              gh issue edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$REPORT"
+            else
+              gh issue create --repo "$GITHUB_REPOSITORY" --title "Weekly health check" --label health --body-file "$REPORT"
+            fi
+          elif [ -n "$number" ]; then
+            gh issue close "$number" --repo "$GITHUB_REPOSITORY" --comment "Every active entry passed the check in $RUN_URL."
+          fi

.github/workflows/publish.yml

@@ -0,0 +1,105 @@
+name: Publish
+
+on:
+  workflow_dispatch:
+
+permissions: {}
+
+concurrency:
+  group: publish
+  cancel-in-progress: false
+
+jobs:
+  build:
+    name: Build
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    permissions:
+      contents: read
+    outputs:
+      changed: ${{ steps.build.outputs.changed }}
+      tag: ${{ steps.build.outputs.tag }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.14"
+      - run: python -m pip install --require-hashes -r requirements.txt
+      - name: Build and compare with the latest release
+        id: build
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          tag="v1.$(date -u +%Y%m%d).${GITHUB_RUN_NUMBER}"
+          python scripts/build.py --out build/dist --release "$tag"
+          latest=$(gh release list --repo "$GITHUB_REPOSITORY" --json tagName,isLatest --jq '.[] | select(.isLatest) | .tagName')
+          previous=""
+          if [ -n "$latest" ]; then
+            gh release download "$latest" --repo "$GITHUB_REPOSITORY" --pattern index.json --pattern SHA256SUMS --dir "$RUNNER_TEMP/latest"
+            previous=$(jq -r .contentSha256 "$RUNNER_TEMP/latest/index.json")
+          fi
+          if [ "$(jq -r .contentSha256 build/dist/index.json)" = "$previous" ]; then
+            # Same data, so Pages keeps serving the files of the latest release.
+            cp "$RUNNER_TEMP/latest/index.json" "$RUNNER_TEMP/latest/SHA256SUMS" build/dist/
+            (cd build/dist && sha256sum --check --strict --quiet SHA256SUMS)
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+            echo "No data change since $latest, so no release was made." >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            echo "tag=$tag" >> "$GITHUB_OUTPUT"
+          fi
+          mkdir -p build/site/v1 build/site/schema/v1
+          cp build/dist/* build/site/v1/
+          cp schema/*.json build/site/schema/v1/
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        if: steps.build.outputs.changed == 'true'
+        with:
+          name: release
+          path: build/dist/
+          if-no-files-found: error
+          retention-days: 7
+      - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
+        with:
+          path: build/site/
+
+  release:
+    name: Release
+    needs: build
+    if: needs.build.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: release
+          path: ${{ runner.temp }}/release
+      - name: Create the immutable release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ needs.build.outputs.tag }}
+          ASSETS: ${{ runner.temp }}/release
+        run: |
+          (cd "$ASSETS" && sha256sum --check --strict --quiet SHA256SUMS)
+          gh release create "$TAG" "$ASSETS"/* --repo "$GITHUB_REPOSITORY" --target "$GITHUB_SHA" \
+            --title "$TAG" --notes "Registry data built from ${GITHUB_SHA}." --latest
+
+  pages:
+    name: Pages
+    needs: [build, release]
+    if: ${{ !cancelled() && needs.build.result == 'success' && needs.release.result != 'failure' }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deploy.outputs.page_url }}
+    steps:
+      - id: deploy
+        uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1

.github/workflows/validate.yml

@@ -26,7 +26,7 @@ jobs:
         with:
           python-version: "3.14"
       - run: python -m pip install --require-hashes -r requirements.txt
-      - run: python -W error scripts/tests/test_validate.py
+      - run: python -W error -m unittest discover --start-directory scripts/tests
 
   entries:
     name: Entries

README.md

@@ -13,7 +13,7 @@ To list your integration, read [CONTRIBUTING.md](CONTRIBUTING.md).
 | `integrations/<slug>/` | `integration.json`, `icon.png`, optional `screenshot-1.png` to `screenshot-4.png` |
 | `schema/` | JSON Schema 2020-12 for manifests, curation files and the published index |
 | `curation/` | Maintainer files: official publishers, partners, reserved names, removed entries, protected icons, accepted scan hits |
-| `scripts/` | `validate.py`, the shared `registry.py`, and their tests |
+| `scripts/` | `validate.py`, `build.py` (the published data), `health.py` (the weekly check), the shared `registry.py`, and their tests |
 | `_owner/` | Repository settings and rulesets, applied once by an org owner |
 
 ## License

scripts/build.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import hashlib
+import io
+import json
+import shutil
+import sys
+from pathlib import Path
+from typing import Optional
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import registry as r  # noqa: E402
+import validate  # noqa: E402
+
+ICON_SIZES = (512, 128)
+SOURCE_ONLY = ("$schema", "schemaVersion")
+NOT_CONTENT = ("generatedAt", "commit", "release", "contentSha256")
+
+
+def encode_png(source: Path, size: tuple) -> bytes:
+    from PIL import Image, ImageCms
+
+    with Image.open(source) as opened:
+        opened.load()
+        profile = opened.info.get("icc_profile")
+        image = opened.convert("RGBA")
+    if profile:
+        icc = ImageCms.ImageCmsProfile(io.BytesIO(profile))
+        # The profile is not kept, and an untagged PNG is shown as sRGB.
+        if icc.profile.xcolor_space.strip() == "RGB":
+            alpha = image.getchannel("A")
+            image = ImageCms.profileToProfile(image.convert("RGB"), icc, ImageCms.createProfile("sRGB"), outputMode="RGB")
+            image.putalpha(alpha)
+    if image.size != size:
+        image = image.resize(size, Image.Resampling.LANCZOS)
+    if image.getchannel("A").getextrema() == (255, 255):
+        image = image.convert("RGB")
+    clean = Image.frombytes(image.mode, image.size, image.tobytes())
+    out = io.BytesIO()
+    clean.save(out, format="PNG", optimize=True)
+    return out.getvalue()
+
+
+def write_image(out: Path, name: str, data: bytes, size: tuple) -> dict:
+    digest = hashlib.sha256(data).hexdigest()
+    file = f"{name}-{digest[:12]}.png"
+    (out / file).write_bytes(data)
+    return {"file": file, "width": size[0], "height": size[1], "sha256": digest}
+
+
+def build_entry(entry: r.Entry, out: Path) -> dict:
+    manifest = entry.manifest
+    data = {k: v for k, v in manifest.items() if k not in SOURCE_ONLY}
+    data["summarySha256"] = hashlib.sha256(manifest["summary"].encode("utf-8")).hexdigest()
+    data["closedSource"] = manifest["source"]["type"] == "closed"
+    data["license"] = manifest.get("license")
+    url = r.source_url(manifest["source"])
+    if url:
+        data["source"] = dict(manifest["source"], url=url)
+    data["install"] = dict(manifest["install"], url=r.install_url(manifest["install"]))
+    icon = entry.directory / "icon.png"
+    data["icon"] = {
+        str(size): write_image(out, f"{entry.slug}-icon-{size}", encode_png(icon, (size, size)), (size, size))
+        for size in ICON_SIZES
+    }
+    data["screenshots"] = [
+        dict(write_image(out, f"{entry.slug}-{shot['file'][:-4]}", encode_png(entry.directory / shot["file"], r.SHOT_SIZE),
+                         r.SHOT_SIZE), alt=shot["alt"])
+        for shot in manifest.get("screenshots") or []
+    ]
+    return data
+
+
+def content_digest(index: dict) -> str:
+    content = {k: v for k, v in index.items() if k not in NOT_CONTENT}
+    text = json.dumps(content, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def fail(findings: list, reason: str) -> None:
+    for finding in findings:
+        print(f"error: {finding.file}: {finding.message}", file=sys.stderr)
+    raise SystemExit(f"{reason}; nothing was built")
+
+
+def build(root: Path, out: Path, commit: str, generated_at: str, release: Optional[str] = None) -> dict:
+    if out.exists() and any(out.iterdir()):
+        raise SystemExit(f"{out} is not empty")
+    ctx = validate.run(root, dt.date.fromisoformat(generated_at[:10]))
+    if ctx.report.errors:
+        fail(ctx.report.errors, "the registry does not validate")
+    out.mkdir(parents=True, exist_ok=True)
+    try:
+        return write(root, out, ctx, commit, generated_at, release)
+    except BaseException:
+        shutil.rmtree(out)
+        raise
+
+
+def write(root: Path, out: Path, ctx: validate.Context, commit: str, generated_at: str, release: Optional[str]) -> dict:
+    index = {"schemaVersion": 1, "generatedAt": generated_at, "commit": commit}
+    if release:
+        index["release"] = release
+    index["contentSha256"] = ""
+    index["integrations"] = [build_entry(entry, out) for _, entry in sorted(ctx.entries.items())]
+    index["removed"] = [{"slug": row["slug"], "removedAt": row["removedAt"], "reason": row["reason"]}
+                        for row in ctx.curated("removed.json", "removed")]
+    index["contentSha256"] = content_digest(index)
+    report = r.Report()
+    r.schema_errors(r.validator_for(root, r.INDEX_SCHEMA), index, "", "index.json", report)
+    if report.errors:
+        fail(report.errors, "index.json does not match schema/index.schema.json")
+    (out / "index.json").write_bytes((json.dumps(index, indent=2, ensure_ascii=False) + "\n").encode("utf-8"))
+    names = sorted(p.name for p in out.iterdir())
+    sums = "".join(f"{hashlib.sha256((out / name).read_bytes()).hexdigest()}  {name}\n" for name in names)
+    (out / "SHA256SUMS").write_bytes(sums.encode("ascii"))
+    return index
+
+
+def main(argv: Optional[list] = None) -> int:
+    parser = argparse.ArgumentParser(description="Build index.json, its images and SHA256SUMS for a release and Pages.")
+    parser.add_argument("--root", default=str(Path(__file__).resolve().parent.parent))
+    parser.add_argument("--out", default="build/dist", help="an empty or missing directory")
+    parser.add_argument("--release", help="release tag, v1.YYYYMMDD.N")
+    args = parser.parse_args(argv)
+    root = Path(args.root)
+    commit = validate.git(root, "rev-parse", "HEAD").strip()
+    committed = dt.datetime.fromisoformat(validate.git(root, "log", "-1", "--format=%cI").strip())
+    generated_at = committed.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    index = build(root, Path(args.out), commit, generated_at, args.release)
+    print(f"{len(index['integrations'])} entries, {len(index['removed'])} removed, contentSha256 {index['contentSha256']}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())