Skip to content

Index

Index #109

Workflow file for this run

name: Index
on:
schedule:
- cron: '0 3 * * *' # Daily at 03:00 UTC
workflow_dispatch:
inputs:
force:
description: 'Index even if PhysLib has not changed (use after widening MODULE_NAMES)'
type: boolean
default: false
# Never let two indexing runs touch the database or Heroku image at once.
concurrency:
group: index
cancel-in-progress: false
jobs:
index:
runs-on: ubuntu-latest
env:
PHYSLIB_REPO: https://github.com/leanprover-community/physlib
JIXIA_REPO: https://github.com/frenzymath/jixia
# Every Lean library PhysLib declares in its lakefile. PhyslibAlpha is not
# in the repo's defaultTargets, so it has to be built explicitly below --
# jixia reads .olean files and cannot analyse what lake never compiled.
MODULE_NAMES: Physlib,PhyslibAlpha,QuantumInfo
DRY_RUN: 'false'
# Each jixia worker loads ~2-3 GB of Mathlib; cap concurrency so the
# runner (16 GB) doesn't get OOM-killed during the load step.
JIXIA_MAX_WORKERS: '2'
CHROMA_PATH: chroma
# The web dyno loads chroma/ into RAM at boot alongside Node/Next.js, so
# this index competes with the app for the dyno's memory. Warn well before
# it gets close (Basic/Standard-1X = 512 MB, Standard-2X = 1024 MB).
CHROMA_MAX_MB: '250'
# A healthy image is ~3 GB. On 2026-08-16 a 21 GB image shipped (physlib/
# and jixia/ leaked into the build context) and the dyno could not boot.
IMAGE_MAX_MB: '6000'
HEALTHCHECK_URL: https://physlibsearch.net
CONNECTION_STRING: ${{ secrets.DATABASE_URL }}
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
GEMINI_MODEL: ${{ vars.GEMINI_MODEL || 'gemini-3-flash-preview' }}
GEMINI_FAST_MODEL: ${{ vars.GEMINI_FAST_MODEL || 'gemini-3-flash-preview' }}
GEMINI_EMBEDDING_MODEL: ${{ vars.GEMINI_EMBEDDING_MODEL || 'gemini-embedding-2-preview' }}
steps:
- name: Checkout main
uses: actions/checkout@v4
with:
ref: main
- name: Install Heroku CLI
run: curl https://cli-assets.heroku.com/install.sh | sh
# Pull chroma/ from the live Docker image so the pipeline runs incrementally
- name: Extract ChromaDB from current Heroku image
env:
HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
run: |
heroku container:login
docker pull registry.heroku.com/physlibsearch/web || echo "No existing image — starting fresh."
CID=$(docker create registry.heroku.com/physlibsearch/web 2>/dev/null) || true
if [ -n "$CID" ]; then
docker cp "$CID:/app/chroma" . 2>/dev/null || echo "No chroma/ in image — starting fresh."
docker rm "$CID"
fi
# Check if PhysLib has changed since the last successful run.
# The last SHA is stored as a Heroku config var to avoid git commits.
#
# The SHA only tracks PhysLib. It says nothing about whether OUR config
# changed, so widening MODULE_NAMES leaves work to do at an unchanged SHA --
# run with force=true after doing that, otherwise the run skips everything.
- name: Check PhysLib for new commits
id: check
env:
HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
run: |
CURRENT_SHA=$(git ls-remote "$PHYSLIB_REPO" HEAD | cut -f1)
LAST_SHA=$(heroku config:get LAST_PHYSLIB_SHA --app physlibsearch 2>/dev/null || echo "")
echo "current_sha=$CURRENT_SHA" >> "$GITHUB_OUTPUT"
if [ "${{ inputs.force }}" = "true" ]; then
echo "has_changes=true" >> "$GITHUB_OUTPUT"
echo "Forced: indexing $CURRENT_SHA regardless of the last indexed SHA."
elif [ "$CURRENT_SHA" = "$LAST_SHA" ]; then
echo "has_changes=false" >> "$GITHUB_OUTPUT"
echo "PhysLib unchanged at $CURRENT_SHA — nothing to do."
else
echo "has_changes=true" >> "$GITHUB_OUTPUT"
echo "PhysLib changed: $LAST_SHA -> $CURRENT_SHA"
fi
- name: Set up Python
if: steps.check.outputs.has_changes == 'true'
uses: actions/setup-python@v5
with:
python-version: '3.12'
cache: pip
- name: Install Python dependencies
if: steps.check.outputs.has_changes == 'true'
run: pip install -r requirements.txt
# PhysLib must be cloned before cache steps so hashFiles() can read lean-toolchain
- name: Clone PhysLib
if: steps.check.outputs.has_changes == 'true'
run: git clone --depth 1 "$PHYSLIB_REPO" physlib
- name: Cache elan toolchains
if: steps.check.outputs.has_changes == 'true'
uses: actions/cache@v4
with:
path: ~/.elan
key: elan-${{ hashFiles('physlib/lean-toolchain') }}
- name: Cache PhysLib lake build
if: steps.check.outputs.has_changes == 'true'
uses: actions/cache@v4
with:
path: physlib/.lake/build
# Namespaced by MODULE_NAMES: which libraries this holds is part of the
# cache's identity. The restore-key must stay inside that namespace --
# a broader fallback silently restores a build missing whole libraries,
# and lake then treats them as up to date, so their .olean files never
# appear and jixia fails per-module with "object file does not exist".
key: physlib-lake-${{ hashFiles('.github/workflows/weekly-index.yml') }}-${{ steps.check.outputs.current_sha }}
restore-keys: |
physlib-lake-${{ hashFiles('.github/workflows/weekly-index.yml') }}-
- name: Install elan
if: steps.check.outputs.has_changes == 'true'
run: |
if ! command -v elan &>/dev/null; then
curl -sSf https://raw.githubusercontent.com/leanprover/elan/master/elan-init.sh \
| sh -s -- -y --no-modify-path
fi
echo "$HOME/.elan/bin" >> "$GITHUB_PATH"
# Build every library named in MODULE_NAMES, not just lake's defaultTargets
# (which omit PhyslibAlpha). jixia analyses .olean files, so a library that
# is never compiled is silently absent from the index rather than an error.
- name: Build PhysLib
if: steps.check.outputs.has_changes == 'true'
run: |
cd physlib
lake exe cache get
lake build $(echo "$MODULE_NAMES" | tr ',' ' ')
timeout-minutes: 120
# jixia must be cloned before its cache step
- name: Clone jixia
if: steps.check.outputs.has_changes == 'true'
run: git clone --depth 1 "$JIXIA_REPO" jixia
- name: Cache jixia build
if: steps.check.outputs.has_changes == 'true'
uses: actions/cache@v4
with:
path: jixia/.lake/build
# Keyed on the compatibility patch too: editing the patch must not reuse
# a build made from the previous one. A stale cached .olean is exactly
# what hid the Analyzer/Process.lean failure during local testing.
key: jixia-v3-${{ hashFiles('physlib/lean-toolchain') }}-${{ hashFiles('jixia/lakefile.lean', 'jixia/lakefile.toml') }}-${{ hashFiles('patches/jixia-lean-4.32.patch') }}
restore-keys: jixia-v3-${{ hashFiles('physlib/lean-toolchain') }}-
- name: Build jixia
id: build_jixia
if: steps.check.outputs.has_changes == 'true'
# jixia reads PhysLib's compiled .olean files, which are version-locked
# to PhysLib's Lean toolchain. Build jixia with the same toolchain so the
# olean headers are compatible (otherwise: "incompatible header").
#
# PhysLib bumps Lean faster than jixia supports it (three bumps since July
# while jixia sat on v4.29.0), so the stock build usually fails. Fall back
# to the compatibility patches in patches/, trying each until one builds.
#
# Each patch targets a specific Lean release and they are NOT
# interchangeable -- e.g. `docString?` takes two fields in v4.32.0 and one
# in v4.33.0 -- so adding support for a new Lean release means dropping in
# another patch file here, with no change to this workflow.
#
# Stock is always tried first, so the day upstream catches up we go back to
# it automatically and the patches can be deleted. If nothing builds, the
# gate below fails the run rather than silently skipping.
continue-on-error: true
run: |
cp physlib/lean-toolchain jixia/lean-toolchain
TOOLCHAIN=$(cat physlib/lean-toolchain)
cd jixia
if lake build; then
echo "jixia built from upstream unpatched."
exit 0
fi
echo "::warning::Stock jixia build failed against $TOOLCHAIN; trying compatibility patches."
for PATCH in ../patches/jixia-lean-*.patch; do
[ -e "$PATCH" ] || continue
NAME=$(basename "$PATCH")
if ! git apply --check "$PATCH" 2>/dev/null; then
echo " $NAME: does not apply, skipping"
continue
fi
git apply "$PATCH"
if lake build; then
echo "jixia built with $NAME."
exit 0
fi
echo " $NAME: applied but did not build, reverting"
git apply -R "$PATCH"
lake clean || true
done
echo "::error title=No jixia build works::Neither upstream jixia nor any patch in patches/ builds against $TOOLCHAIN. A new patch is needed for this Lean release -- see patches/README.md."
exit 1
timeout-minutes: 30
# A jixia that compiles can still emit JSON this pipeline cannot parse: the
# v4.33.0 patch built fine, then failed two hours into the load because Lean
# changed docString from [text, bool] to a bare string. Run jixia over one
# small module and parse the result the same way the loader does, so that
# class of mismatch surfaces in seconds instead of mid-run.
- name: Smoke-test jixia output
id: smoke
if: steps.build_jixia.outcome == 'success'
continue-on-error: true
run: |
MODULE=$(find physlib/Physlib -name '*.lean' | head -1)
echo "Smoke-testing against $MODULE"
cd physlib
lake env ../jixia/.lake/build/bin/jixia -i \
-m /tmp/smoke.mod.json -d /tmp/smoke.decl.json -s /tmp/smoke.sym.json \
"../$MODULE" 2>&1 | tail -3
cd ..
python3 - <<'PY'
import sys
from jixia.structs import Declaration, Symbol
# Import for its compatibility shims before validating anything.
import database.jixia_db # noqa: F401
try:
decls = Declaration.from_json_file("/tmp/smoke.decl.json")
syms = Symbol.from_json_file("/tmp/smoke.sym.json")
except Exception as exc:
sys.exit(f"::error::jixia output does not match what the loader expects: {exc}")
print(f"parsed {len(decls)} declarations, {len(syms)} symbols")
PY
- name: Fail if jixia output is unparseable
if: steps.build_jixia.outcome == 'success' && steps.smoke.outcome != 'success'
run: |
echo "::error title=jixia output incompatible::jixia built, but its output could not be parsed. Lean likely changed a field's shape -- see the smoke-test step. Fix database/jixia_db.py rather than waiting for the multi-hour load to fail."
exit 1
# Decide whether indexing can proceed.
#
# "PhysLib unchanged" is a real no-op and passes quietly. But if PhysLib has
# new content we cannot index, the run FAILS: a daily green check that
# silently indexes nothing is how this went unnoticed for weeks. The site is
# unaffected either way -- it keeps serving the last good index.
- name: Gate on jixia compatibility
id: gate
if: always()
run: |
if [ "${{ steps.check.outputs.has_changes }}" != "true" ]; then
echo "proceed=false" >> "$GITHUB_OUTPUT"
echo "PhysLib unchanged — nothing to index."
elif [ "${{ steps.build_jixia.outcome }}" = "success" ]; then
echo "proceed=true" >> "$GITHUB_OUTPUT"
else
echo "proceed=false" >> "$GITHUB_OUTPUT"
TOOLCHAIN=$(cat physlib/lean-toolchain 2>/dev/null || echo unknown)
echo "::warning title=Indexing skipped::jixia failed to build against PhysLib's Lean toolchain ($TOOLCHAIN). The index was NOT updated; the site keeps serving the previous index. This clears once jixia supports this Lean release."
{
echo "## ⚠️ Indexing skipped — jixia/Lean incompatibility"
echo ""
echo "PhysLib is on \`$TOOLCHAIN\`, and jixia does not compile against it."
echo "jixia must be built with PhysLib's exact Lean version to read its \`.olean\` files."
echo ""
echo "**Impact:** the index was not updated. The site is unaffected and still serves the previous index."
echo ""
echo "**Resolution:** wait for jixia to support this Lean release, or pin PhysLib to a commit on a supported toolchain."
echo ""
echo "This run is FAILED on purpose: PhysLib has new content that is not being indexed."
} >> "$GITHUB_STEP_SUMMARY"
exit 1
fi
- name: Set JIXIA_PATH and LEAN_SYSROOT
if: steps.gate.outputs.proceed == 'true'
run: |
echo "JIXIA_PATH=$(pwd)/jixia/.lake/build/bin/jixia" >> "$GITHUB_ENV"
TOOLCHAIN=$(cat physlib/lean-toolchain)
echo "LEAN_SYSROOT=$HOME/.elan/toolchains/$TOOLCHAIN" >> "$GITHUB_ENV"
# Incremental pipeline — each step skips already-processed items.
#
# Every step below is idempotent and resumable, so transient failures
# (dropped Postgres connections over the multi-hour run, flaky Gemini
# calls) are retried rather than failing the whole run. A retry re-runs
# only the work that is still outstanding.
- name: Create/update schema
if: steps.gate.outputs.proceed == 'true'
run: python3 -m database schema
- name: Load jixia data into PostgreSQL
if: steps.gate.outputs.proceed == 'true'
run: |
for attempt in 1 2 3; do
if python3 -m database jixia ./physlib "$MODULE_NAMES"; then exit 0; fi
echo "::warning::jixia load attempt $attempt failed; retrying in 30s"
sleep 30
done
echo "::error::jixia load failed after 3 attempts"
exit 1
# A jixia build that compiles is not necessarily one that parses correctly,
# which matters most when the compatibility patch above is in use. The load
# is additive (ON CONFLICT DO NOTHING), so a broken analyzer shows up as a
# load that adds nothing rather than as an error. Catch that here.
- name: Sanity-check what the load produced
if: steps.gate.outputs.proceed == 'true'
run: |
python3 - <<'PY'
import os, sys, psycopg
from psycopg.types.json import Jsonb
expected = [n.strip() for n in os.environ["MODULE_NAMES"].split(",") if n.strip()]
with psycopg.connect(os.environ["CONNECTION_STRING"], autocommit=True) as conn, conn.cursor() as c:
c.execute("SELECT COUNT(*) FROM module")
modules = c.fetchone()[0]
c.execute("SELECT COUNT(*) FROM symbol")
symbols = c.fetchone()[0]
c.execute("SELECT COUNT(*) FROM declaration")
declarations = c.fetchone()[0]
# Every namespace in MODULE_NAMES must actually be present. A library
# that failed to build produces no modules and would otherwise vanish
# from the index silently -- which is how PhyslibAlpha went missing.
per_namespace = {}
for name in expected:
c.execute("SELECT COUNT(*) FROM module WHERE name->>0 = %s", (name,))
per_namespace[name] = c.fetchone()[0]
print(f"modules={modules} symbols={symbols} declarations={declarations}")
for name, count in per_namespace.items():
print(f" {name}: {count} modules")
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as fh:
fh.write(f"\nIndex after load: {modules} modules, {symbols} symbols, {declarations} declarations\n\n")
for name, count in per_namespace.items():
fh.write(f"- `{name}`: {count} modules\n")
missing = [n for n, count in per_namespace.items() if count == 0]
if missing:
sys.exit(f"::error::Indexed nothing for: {', '.join(missing)}. The library likely did not build -- check that it is a lean_lib in PhysLib's lakefile and that the build step compiled it.")
# These floors would only be crossed by a analyzer producing garbage;
# normal incremental runs land far above them.
if modules < 100 or symbols < 1000 or declarations < 1000:
sys.exit("::error::Implausibly small index after load — treating as a bad jixia build rather than publishing it.")
PY
- name: Informalize new declarations
if: steps.gate.outputs.proceed == 'true'
run: |
for attempt in 1 2 3; do
if python3 -m database informal --batch-size 50; then exit 0; fi
echo "::warning::informalize attempt $attempt failed; retrying in 30s"
sleep 30
done
echo "::error::informalize failed after 3 attempts"
exit 1
- name: Embed new declarations into ChromaDB
if: steps.gate.outputs.proceed == 'true'
run: |
for attempt in 1 2 3; do
if python3 -m database vector-db --batch-size 8; then exit 0; fi
echo "::warning::embedding attempt $attempt failed; retrying in 30s"
sleep 30
done
echo "::error::embedding failed after 3 attempts"
exit 1
# Guard against shipping an image whose ChromaDB would OOM the dyno.
# The web dyno loads this index into memory at boot alongside Node.
- name: Check ChromaDB size against dyno memory budget
if: steps.gate.outputs.proceed == 'true'
run: |
SIZE_MB=$(du -sm chroma | cut -f1)
echo "ChromaDB size: ${SIZE_MB} MB (warn threshold: ${CHROMA_MAX_MB} MB)"
echo "ChromaDB size: ${SIZE_MB} MB" >> "$GITHUB_STEP_SUMMARY"
if [ "$SIZE_MB" -gt "$CHROMA_MAX_MB" ]; then
echo "::warning title=ChromaDB approaching memory budget::chroma/ is ${SIZE_MB} MB (threshold ${CHROMA_MAX_MB} MB). The web dyno loads this into RAM at startup; upgrade the dyno or prune the index before it OOMs."
fi
# Record what is currently serving, so a bad deploy can be undone.
- name: Record current release for rollback
id: prev_release
if: steps.gate.outputs.proceed == 'true'
env:
HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
run: |
# Roll back onto the last release that actually shipped an image. Config-var
# releases don't carry one, so rolling back onto them would not change the
# running code. Container releases are identified by their description.
PREV=$(heroku releases --app physlibsearch --num 30 --json | python3 -c '
import json, sys
releases = json.load(sys.stdin)
for r in releases:
desc = r.get("description", "")
if desc.startswith("Deployed web") or desc.startswith("Rollback to"):
print(r["version"])
break
else:
sys.exit("no image-bearing release found in the last 30")
')
echo "prev=$PREV" >> "$GITHUB_OUTPUT"
echo "Will roll back to v$PREV if the new release is unhealthy."
# The build context contains the cloned+built physlib/ and jixia/ trees.
# If .dockerignore ever stops excluding them, the image balloons past what
# the platform can boot in its startup window and the site goes down. Check
# the built image before releasing it rather than after.
- name: Build image and check its size
if: steps.gate.outputs.proceed == 'true'
env:
HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
run: |
docker build -t physlibsearch-web .
SIZE_MB=$(docker image inspect physlibsearch-web --format '{{.Size}}' | awk '{printf "%d", $1/1048576}')
echo "Built image: ${SIZE_MB} MB (limit ${IMAGE_MAX_MB} MB)"
echo "Built image: ${SIZE_MB} MB" >> "$GITHUB_STEP_SUMMARY"
if [ "$SIZE_MB" -gt "$IMAGE_MAX_MB" ]; then
docker run --rm --entrypoint sh physlibsearch-web -c 'du -sh /app/* 2>/dev/null | sort -rh | head -10' || true
echo "::error title=Image too large to deploy::Built image is ${SIZE_MB} MB (limit ${IMAGE_MAX_MB} MB). The dyno cannot pull and boot this within the platform's startup window. Largest /app entries are listed above -- most likely .dockerignore stopped excluding physlib/ or jixia/."
exit 1
fi
- name: Release Docker image
if: steps.gate.outputs.proceed == 'true'
env:
HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
run: |
heroku container:push web --app physlibsearch
heroku container:release web --app physlibsearch
# Verify the new release actually serves traffic, and put the previous one
# back if it does not. A failed index is an inconvenience; a site that stays
# down until someone notices is not, and that is exactly what happened on
# 2026-08-16 (deployed 03:41, still down at 08:12).
- name: Verify deployment, roll back if unhealthy
id: verify
if: steps.gate.outputs.proceed == 'true'
env:
HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
run: |
echo "Waiting for the new release to serve traffic..."
for attempt in $(seq 1 20); do
CODE=$(curl -s -o /dev/null -w "%{http_code}" -m 20 "$HEALTHCHECK_URL" || echo 000)
if [ "$CODE" = "200" ]; then
echo "Site healthy (HTTP 200) after $attempt attempt(s)."
exit 0
fi
echo "attempt $attempt: HTTP $CODE — retrying in 15s"
sleep 15
done
echo "::error title=Deploy unhealthy — rolling back::New release never returned HTTP 200. Restoring v${{ steps.prev_release.outputs.prev }}."
heroku rollback "v${{ steps.prev_release.outputs.prev }}" --app physlibsearch
RESTORED=""
for attempt in $(seq 1 20); do
CODE=$(curl -s -o /dev/null -w "%{http_code}" -m 20 "$HEALTHCHECK_URL" || echo 000)
if [ "$CODE" = "200" ]; then
echo "Rollback healthy (HTTP 200) after $attempt attempt(s)."
RESTORED=1
break
fi
sleep 15
done
{
echo "## Deploy failed — rolled back"
echo ""
echo "The new image did not serve traffic, so v${{ steps.prev_release.outputs.prev }} was restored."
if [ -n "$RESTORED" ]; then
echo ""
echo "**The site is back up on the previous release.** The index work is safe in"
echo "Postgres; only the image that ships it failed. Re-running is safe."
else
echo ""
echo "**THE SITE IS STILL DOWN after rollback — this needs a human now.**"
echo "Check \`heroku logs --app physlibsearch\` and \`heroku releases\`."
fi
} >> "$GITHUB_STEP_SUMMARY"
exit 1
# Only record the indexed SHA once the deploy is verified. Otherwise a failed
# deploy would mark this PhysLib revision as done and the next run would skip
# it, silently stranding the work.
- name: Mark PhysLib revision as indexed
if: steps.gate.outputs.proceed == 'true' && steps.verify.outcome == 'success'
env:
HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
run: |
heroku config:set LAST_PHYSLIB_SHA=${{ steps.check.outputs.current_sha }} --app physlibsearch