docker-compose.dokploy.yml

# docker-compose.dokploy.yml
#
# Dokploy-managed variant of docker-compose.prod.yml. Differences:
#
#   * No own reverse proxy on 80/443 — Dokploy's Traefik does that.
#     Caddy stays, but only on the internal network (no published
#     ports) and serves as the path-based router + static file
#     server + cache-header layer. Traefik routes
#     `Host(knowledge-web.org)` → caddy:80.
#   * Joins the shared `dokploy-network` so Traefik can discover us.
#   * Postgres uses the EXISTING `knowledge_pgdata` external volume
#     so data survives the cutover. Same for the models volume.
#   * The on-disk ColBERT indexes still bind-mount from the host
#     `/root/knowledge/indexes` directory — that path stays the
#     source of truth across the migration.
#
# Env vars (set in Dokploy's project UI):
#   DOMAIN, POSTGRES_PASSWORD, GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET,
#   KNOWLEDGE_ADMIN_TOKEN, ADMIN_API_KEY, OPENAI_API_KEY
#
# ADMIN_API_KEY MUST be set to a long random string before deploy — the
# API fails closed when this env is missing and every admin endpoint
# (POST /api/pipeline, DELETE /indices/*, ingest, promote, …) will
# return 401. Generate with: `openssl rand -hex 32`.
#
# OPENAI_API_KEY is consumed by the knowledge-clean-daemon service
# (gpt-4.1-mini calls). The clean daemon refuses to start without it.

# Default log driver config — merged into every service via the
# `logging: *default-logging` line. Without this, docker's
# `json-file` driver has NO size cap and a single chatty container
# (the indexer logs ~10 lines/sec at peak) can fill the host disk
# inside weeks. 10 MB × 3 files = 30 MB hard cap per container,
# ~270 MB across the whole stack — plenty for postmortem, bounded
# enough that the host can never run out of room.
x-logging: &default-logging
  driver: json-file
  options:
    max-size: "10m"
    max-file: "3"

# YAML anchor shared by all four Python daemon services (formerly
# systemd units on the host). Same image, same network, same
# database — only the `command:` and the resource limits change
# per service.
x-daemon-base: &daemon-base
  logging: *default-logging
  build:
    context: .
    dockerfile: Dockerfile.daemons
  image: knowledge-daemons:latest
  restart: unless-stopped
  environment:
    - DATABASE_URL=postgresql://knowledge:${POSTGRES_PASSWORD:-knowledge}@postgres:5432/knowledge
    # Internal HTTP rather than https://knowledge-web.org — skips the
    # Traefik+Caddy+TLS round trip for daemon → API calls (only the
    # indexer hits the API regularly today, but the route is here for
    # any future daemon that needs it).
    - API_URL=http://knowledge-api:8080
    - KNOWLEDGE_ADMIN_TOKEN=${KNOWLEDGE_ADMIN_TOKEN}
    # ADMIN_API_KEY is required by the indexer daemon for every write
    # path on the API: DELETE /indices/__all__/documents (used by the
    # per-VIP incremental refresh), create_index, and the batched
    # update_with_encoding push. Without it, every daemon iteration
    # 401s, so the per-user index never rebuilds and __all__ stops
    # tracking new VIP docs (newly-bookmarked HF model cards never
    # surface on cross-library search). Shared via x-daemon-base so
    # any future write-path daemon inherits it.
    - ADMIN_API_KEY=${ADMIN_API_KEY}
    - OPENAI_API_KEY=${OPENAI_API_KEY:-}
    - PYTHONUNBUFFERED=1
  depends_on:
    postgres:
      condition: service_healthy
  networks:
    - default

services:
  postgres:
    image: postgres:16-alpine
    restart: unless-stopped
    logging: *default-logging
    # Memory tuning. PG's out-of-the-box defaults (work_mem=4MB,
    # shared_buffers=128MB) are sized for a Raspberry Pi; on this
    # 8 GB host the feed query alone wanted 27 MB of sort space
    # and was spilling to disk for ~6 seconds per cold call. The
    # values below match what `ALTER SYSTEM` was set to on prod
    # (see `postgresql.auto.conf` in the pgdata volume); we also
    # pass them on the command line so a fresh database gets the
    # same posture from boot 1.
    #
    # `shm_size: 1g` raises the container's /dev/shm from the 64 MB
    # Docker default. VACUUM ANALYZE and parallel seq scans fail
    # with "No space left on device" on the small default.
    shm_size: 1g
    command:
      - "postgres"
      - "-c"
      - "work_mem=32MB"
      - "-c"
      - "shared_buffers=1GB"
      - "-c"
      - "effective_cache_size=4GB"
      - "-c"
      - "maintenance_work_mem=256MB"
      - "-c"
      - "jit=off"
    environment:
      POSTGRES_USER: knowledge
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-knowledge}
      POSTGRES_DB: knowledge
    volumes:
      - knowledge_pgdata:/var/lib/postgresql/data
    networks:
      - default
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U knowledge"]
      interval: 5s
      timeout: 3s
      retries: 5

  knowledge-api:
    build:
      # Build context is the repo root so the API Dockerfile can COPY
      # both `api/` (Rust crate) and `sources/sql/` (schema files baked
      # into the binary via `include_str!`). The api/Dockerfile path is
      # passed explicitly because the context shifted up one level.
      context: .
      dockerfile: api/Dockerfile
    restart: unless-stopped
    logging: *default-logging
    # The container starts as root, the entrypoint chowns the bind-
    # mounted /data/indices + named volume /models + /app workdir,
    # then drops privileges to uid 1000 (the `knowledge` user from
    # the Dockerfile) via gosu before exec'ing the API binary. So
    # the running process is non-root — an RCE no longer pwns the
    # host directly.
    volumes:
      - /root/knowledge/indexes:/data/indices
      - knowledge_models:/models
      - ./run.py:/app/run.py:ro
      - ./sources:/app/sources:ro
      - ./pyproject.toml:/app/pyproject.toml:ro
      - ./uv.lock:/app/uv.lock:ro
      - ./sources.yml:/app/sources.yml:ro
      - ./README.md:/app/README.md:ro
    environment:
      - DATABASE_URL=postgresql://knowledge:${POSTGRES_PASSWORD:-knowledge}@postgres:5432/knowledge
      - API_URL=https://${DOMAIN:-knowledge-web.org}
      - RUST_LOG=info
      - OPENBLAS_NUM_THREADS=1
      - GITHUB_CLIENT_ID=${GITHUB_CLIENT_ID}
      - GITHUB_CLIENT_SECRET=${GITHUB_CLIENT_SECRET}
      - OAUTH_REDIRECT_URL=https://${DOMAIN:-knowledge-web.org}/auth/github/callback
      - OAUTH_POST_LOGIN_URL=/
      - KNOWLEDGE_ADMIN_TOKEN=${KNOWLEDGE_ADMIN_TOKEN}
      # Required: the admin API key consumed by RequireApiKey middleware.
      # Without it, admin endpoints (pipeline, index delete/promote, ingest)
      # return 401 — fail-closed.
      - ADMIN_API_KEY=${ADMIN_API_KEY}
      # Rate limiting. The search router gets the larger bucket; the
      # auth abuse-prone endpoints (signup/login/forgot/resend/reset)
      # are throttled separately at 1 req/s burst 10 per source IP
      # via SmartIpKeyExtractor (X-Forwarded-For aware).
      - RATE_LIMIT_ENABLED=${RATE_LIMIT_ENABLED:-true}
      - RATE_LIMIT_PER_SECOND=${RATE_LIMIT_PER_SECOND:-50}
      - RATE_LIMIT_BURST_SIZE=${RATE_LIMIT_BURST_SIZE:-100}
    command:
      - --host
      - "0.0.0.0"
      - --port
      - "8080"
      - --index-dir
      - /data/indices
      - --model
      - lightonai/answerai-colbert-small-v1-onnx
      - --int8
      - --parallel
      - "8"
      - --batch-size
      - "2"
      - --model-pool-size
      - "2"
    depends_on:
      postgres:
        condition: service_healthy
    networks:
      - default
    labels:
      # Opt in to the autoheal watchdog below: restart this container
      # if its healthcheck goes (and stays) unhealthy.
      - "autoheal=true"
    deploy:
      resources:
        limits:
          memory: 6G
        reservations:
          memory: 3G

  # Watchdog: Docker restart policies only cover CRASHED processes — a
  # container whose healthcheck flips to "unhealthy" is left running
  # forever. That's how the 2026-07-07 API deadlock stayed "Up
  # (unhealthy)" for two days while every /api request hung. This
  # sidecar restarts any container labeled autoheal=true whose health
  # is "unhealthy" on two consecutive passes (~2 min) — the double
  # check rides out one-off blips; real hangs never recover on their
  # own. Same docker-socket pattern as docker-cleanup below.
  autoheal:
    image: docker:25-cli
    restart: unless-stopped
    logging: *default-logging
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    entrypoint:
      - /bin/sh
      - -c
      - |
        set -eu
        echo "autoheal: watching containers labeled autoheal=true"
        PREV=""
        while true; do
          sleep 60
          UNHEALTHY=$$(docker ps --filter label=autoheal=true --filter health=unhealthy --format '{{.Names}}' || true)
          for NAME in $$UNHEALTHY; do
            case " $$PREV " in
              *" $$NAME "*)
                TS=$$(date -u +%Y-%m-%dT%H:%M:%SZ)
                echo "$$TS autoheal: restarting $$NAME (unhealthy on two consecutive checks)"
                docker restart "$$NAME" || true
                ;;
            esac
          done
          PREV="$$UNHEALTHY"
        done

  caddy:
    # Build a custom image that bakes the Caddyfile + the static
    # `web/` tree into the image instead of bind-mounting them at
    # runtime. This dodges Dokploy's deploy-time inode swap: every
    # redeploy clones the repo into a fresh directory, which leaves
    # the previously mounted host path as an orphan inode and makes
    # Caddy serve an empty /web until --force-recreate'd. Building
    # makes deploys ~10s slower but eliminates the race entirely.
    build:
      context: .
      dockerfile: Dockerfile.caddy
    restart: unless-stopped
    logging: *default-logging
    # No ports published — Traefik handles 80/443 externally and
    # forwards to this container on the dokploy-network.
    environment:
      DOMAIN: ${DOMAIN:-knowledge-web.org}
    depends_on:
      knowledge-api:
        condition: service_healthy
    networks:
      - default
      - dokploy-network
    labels:
      - "traefik.enable=true"
      # HTTPS — Traefik terminates TLS and forwards plain HTTP to caddy:80.
      - "traefik.http.routers.knowledge.rule=Host(`${DOMAIN:-knowledge-web.org}`)"
      - "traefik.http.routers.knowledge.entrypoints=websecure"
      - "traefik.http.routers.knowledge.tls.certresolver=letsencrypt"
      - "traefik.http.services.knowledge.loadbalancer.server.port=80"
      # Plain HTTP → permanent redirect to HTTPS.
      - "traefik.http.routers.knowledge-http.rule=Host(`${DOMAIN:-knowledge-web.org}`)"
      - "traefik.http.routers.knowledge-http.entrypoints=web"
      - "traefik.http.routers.knowledge-http.middlewares=knowledge-redirect"
      - "traefik.http.middlewares.knowledge-redirect.redirectscheme.scheme=https"
      - "traefik.http.middlewares.knowledge-redirect.redirectscheme.permanent=true"

  # Hourly logical Postgres backup with retention. Writes
  # /backups/knowledge-YYYYMMDD-HHMMSS.sql.gz inside the named volume
  # `knowledge_pgbackups` and prunes anything older than 7 days.
  #
  # No external dependencies — uses the same postgres:16-alpine image
  # so pg_dump matches the server version exactly.
  #
  # Recovery:
  #   docker compose exec -T postgres dropdb -U knowledge knowledge
  #   docker compose exec -T postgres createdb -U knowledge knowledge
  #   gunzip -c knowledge-<ts>.sql.gz | docker compose exec -T postgres \
  #       psql -U knowledge knowledge
  pg-backup:
    image: postgres:16-alpine
    restart: unless-stopped
    logging: *default-logging
    depends_on:
      postgres:
        condition: service_healthy
    environment:
      PGPASSWORD: ${POSTGRES_PASSWORD:-knowledge}
    volumes:
      - knowledge_pgbackups:/backups
    networks:
      - default
    entrypoint:
      - /bin/sh
      - -c
      - |
        set -eu
        echo "pg-backup: hourly job started"
        while true; do
          TS=$$(date -u +%Y%m%d-%H%M%S)
          OUT="/backups/knowledge-$$TS.sql.gz"
          if pg_dump -h postgres -U knowledge -d knowledge --no-owner --no-privileges \
             | gzip -9 > "$$OUT.partial"; then
            mv "$$OUT.partial" "$$OUT"
            echo "pg-backup: wrote $$OUT ($$(du -h "$$OUT" | cut -f1))"
            # Prune anything older than 7 days
            find /backups -name 'knowledge-*.sql.gz' -mtime +7 -delete
          else
            rm -f "$$OUT.partial"
            echo "pg-backup: FAILED — keeping previous backups" >&2
          fi
          sleep 3600
        done

  # ────────────────────────────────────────────────────────────────
  # Python daemons (formerly systemd units on the host). All four
  # share `x-daemon-base` above — only the `command:` and resource
  # limits change per service. Logs go to docker (visible via
  # `dokploy logs <service>` or `docker logs <container>`); the
  # continuous-pipeline state files (history, pid, stop flag) live
  # in the `knowledge_daemon_logs` volume so restarts don't lose
  # the rotation cursor.
  # ────────────────────────────────────────────────────────────────

  # VIP-first per-user pipeline runner. Loops over personalities,
  # invokes `run.py <slug> --source <non-twitter,...>`, sleeps,
  # repeats. Picks up every new source fetcher (including the
  # recently added `huggingface.Activity`) on container restart.
  knowledge-continuous:
    <<: *daemon-base
    command: ["bash", "sources/continuous_pipeline.sh"]
    volumes:
      - knowledge_daemon_logs:/app/logs
    deploy:
      resources:
        limits:
          # CPU cap stepped down again: 1.0 → 0.6 → 0.5 as part of
          # a global rebalance where every daemon's quota shrinks
          # so the four-daemon total lands at 1.0 vCPU exactly,
          # leaving the API a guaranteed ~1 vCPU on the 2-vCPU box
          # (up from the previous ~0.75). The continuous pipeline
          # is HTTP-bound on external fetchers (twitter, scholar,
          # huggingface) at steady state — typical use is ~2-5 %
          # of one core. The 0.5 cap still absorbs the periodic
          # tag/clean/index bursts without spilling into API time.
          memory: 2G
          cpus: "0.5"

  # ColBERT index repair + backfill. Hits the local knowledge-api on
  # the docker network rather than the public URL so the round trip
  # avoids Traefik+Caddy+TLS. CPU 0.5 mirrors the old CPUQuota=50%.
  knowledge-indexer:
    <<: *daemon-base
    command: ["python", "-m", "sources.indexer_daemon"]
    depends_on:
      postgres:
        condition: service_healthy
      knowledge-api:
        condition: service_started
    volumes:
      - knowledge_daemon_logs:/app/logs
    deploy:
      resources:
        limits:
          # CPU cap stepped down: 0.5 → 0.35 → 0.25 as part of the
          # daemons-total-1.0-vCPU rebalance. Indexer work is
          # latency-tolerant (backfill, not user-facing); the
          # steady-state actual usage hovers ~7 % of one core, so
          # 0.25 still leaves 3-4× burst headroom. Index rebuilds
          # that did peg the cap will just take a little longer —
          # users won't notice because the previous index keeps
          # serving until the rebuild finishes.
          memory: 2G
          cpus: "0.25"

  # Category-assignment daemon. CPU-bound on small Potion embeddings
  # (~10 % of one core on the old systemd unit).
  knowledge-categorize-daemon:
    <<: *daemon-base
    command: ["python", "-m", "sources.utils.categorize_daemon"]
    deploy:
      resources:
        limits:
          memory: 384M
          cpus: "0.10"

  # Daily Docker housekeeping. Mirrors the pg-backup loop pattern.
  # Reclaims stale images and old build cache; KEEPS the last 7
  # days of build cache so an incremental redeploy can reuse
  # cached Rust crate layers (without this filter we used to
  # rebuild the entire cargo dep tree from scratch — ~10 min per
  # deploy and a ~10 min API outage).
  #
  # Once every 24 h:
  #
  #   * `docker image prune -af`
  #     EVERY unused image is removed. By definition this leaves
  #     images currently used by a container alone.
  #   * `docker builder prune -af --filter until=168h`
  #     Build cache older than 7 days. The 168h filter is what
  #     lets the next deploy reuse most cargo / pip layers —
  #     incremental builds drop from ~10 min to ~30 s.
  #   * `docker network prune -f`
  #     orphaned networks from removed projects.
  #
  # NOTE: container prune is intentionally absent. During a
  # Dokploy redeploy the OLD container is briefly stopped before
  # the new one starts; a container prune at that instant nukes
  # it and tangles compose's recreate logic (lost a few minutes
  # of uptime to that race in May 2026).
  #
  # Volumes are NEVER pruned automatically — `knowledge_pgdata`,
  # `knowledge_models`, `knowledge_pgbackups`, `knowledge_daemon_logs`
  # all hold irreplaceable state. The CLI's prune commands above
  # don't touch volumes (volumes require an explicit
  # `docker volume prune`), so this is safe by construction.
  docker-cleanup:
    image: docker:25-cli
    restart: unless-stopped
    logging: *default-logging
    volumes:
      # The cleanup container drives the host's docker daemon via
      # the unix socket. No other privilege is needed — the socket
      # IS the privilege boundary.
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - default
    entrypoint:
      - /bin/sh
      - -c
      - |
        set -eu
        # Sleep before the FIRST pass: a Dokploy redeploy starts
        # every service at roughly the same time, including this
        # one. If the prune loop fires immediately we race against
        # docker recreating the other containers and end up nuking
        # half-started ones (the "No such container" / "name
        # already in use" failures we hit in May 2026 — every
        # redeploy left the site at HTTP 404 until the operator
        # intervened). 30 min cooldown gives the rest of the stack
        # plenty of time to settle.
        echo "docker-cleanup: daily job started; first pass in 30 min"
        sleep 1800
        while true; do
          TS=$$(date -u +%Y-%m-%dT%H:%M:%SZ)
          echo "$$TS docker-cleanup: pass starting"
          # NOTE: we deliberately do NOT run `docker container
          # prune -f`. During a Dokploy redeploy the OLD container
          # is briefly stopped before the new one starts; a
          # container prune at that instant removes it, which
          # tangles the compose recreate logic. Stopped one-off
          # containers are rare anyway, and image+builder prune
          # reclaims 99 % of the disk regardless.
          docker image prune -af                          2>&1 | tail -3 || true
          docker builder prune -af --filter "until=168h"  2>&1 | tail -3 || true
          docker network prune -f                         2>&1 | tail -3 || true
          echo "$$TS docker-cleanup: pass done"
          docker system df 2>&1 | tail -5 || true
          sleep 86400
        done

  # Hourly precomputed-feed snapshotter. Runs the scoring CTE over
  # a 180-day window once an hour and writes the result back to
  # `feed_snapshot`. The /api/timeline handler reads from this
  # table when it's fresh (<3 h), giving every logged-in feed
  # request a sub-100 ms indexed scan instead of a per-call rebuild.
  # Falls back to the live query when the snapshot is stale or
  # empty, so a daemon outage degrades to "previous behaviour"
  # rather than "broken feed".
  #
  # CPU need is tiny — the daemon spends ~5 s an hour running the
  # refresh CTE and sleeps the rest. 0.05 vCPU covers it (counted
  # against the daemons-total-1.0-vCPU budget elsewhere); memory
  # is just enough to hold psycopg connection state.
  knowledge-feed-snapshot:
    <<: *daemon-base
    command: ["python", "-m", "sources.utils.feed_snapshot_daemon"]
    depends_on:
      postgres:
        condition: service_healthy
    deploy:
      resources:
        limits:
          memory: 256M
          cpus: "0.05"

  # Daily HackerNews front page → per-user picks, injected into the
  # feed by the `hn_user_picks` join in handlers/follows.rs.
  #
  # This service is the whole reason HN cards appear at all: the job
  # used to live in `scripts/` behind `make hn-frontpage`, which
  # nothing scheduled and which `Dockerfile.daemons` doesn't even
  # copy into the image — so it ran exactly once, by hand, in May.
  #
  # Sleeps ~24 h out of every 24 h. The working minute is spent
  # waiting on ~50 HTTP round trips to knowledge-api (one batched
  # `__all__` search per library, scoped by owner), so the CPU ask is
  # nominal; memory holds a ~50 × 30 float matrix plus psycopg state.
  #
  # Tunables (override in the Dokploy UI):
  #   HN_REFRESH_INTERVAL_SECS  cadence, default 86400
  #   HN_TOP                    front-page stories fetched, default 30
  #   HN_TOP_PER_USER           max picks per user, default 10
  #   HN_THRESHOLD              z floor for a pick, default 0.5
  #                             (raise for fewer/more relevant picks)
  #   HN_REFERENCE_COHORT       extra libraries scored purely to give
  #                             the centering a baseline, default 48
  knowledge-hn-frontpage:
    <<: *daemon-base
    command: ["python", "-m", "sources.utils.hn_frontpage_daemon"]
    depends_on:
      postgres:
        condition: service_healthy
      # Every search goes through the API, and the daemon refreshes on
      # startup — without this it races the API's own restart on each
      # deploy. `service_started` isn't "listening", so the scorer
      # retries transient failures too.
      knowledge-api:
        condition: service_started
    deploy:
      resources:
        limits:
          memory: 256M
          cpus: "0.05"

  # Pedagogical title / summary rewriter via OpenAI (default model
  # gpt-4.1-mini; override `OPENAI_CLEAN_MODEL` in Dokploy env to swap
  # in a cheaper one, e.g. `gpt-4.1-nano`). I/O-bound on the OpenAI
  # API; the local CPU just streams the response back into PG, so
  # 0.15 covers it.
  knowledge-clean-daemon:
    <<: *daemon-base
    environment:
      - DATABASE_URL=postgresql://knowledge:${POSTGRES_PASSWORD:-knowledge}@postgres:5432/knowledge
      - API_URL=http://knowledge-api:8080
      - KNOWLEDGE_ADMIN_TOKEN=${KNOWLEDGE_ADMIN_TOKEN}
      - OPENAI_API_KEY=${OPENAI_API_KEY:-}
      # gpt-4.1-mini: the explainer prompt (Rust-book voice, grounded
      # background definitions) needs a model that actually follows a
      # writing style — in a side-by-side test gpt-4o-mini never adopted
      # the voice and guessed an acronym expansion. gpt-4.1-nano is the
      # cheaper fallback via the Dokploy UI.
      - OPENAI_CLEAN_MODEL=${OPENAI_CLEAN_MODEL:-gpt-4.1-mini}
      # Daemon only rewrites docs in the top-N of feed_snapshot.score.
      # Default 3000 covers a generous pagination tail beyond what
      # any user could realistically scroll to (cards page 1 = 50;
      # 60 pages ≫ what we see in analytics). Bump up if the
      # snapshot ranking changes and we want more headroom.
      - CLEAN_FEED_TOP_N=${CLEAN_FEED_TOP_N:-3000}
      - PYTHONUNBUFFERED=1
    command: ["python", "-m", "sources.utils.clean_daemon"]
    deploy:
      resources:
        limits:
          memory: 256M
          # Stepped 0.20 → 0.15 in the daemons-total-1.0-vCPU
          # rebalance; clean is OpenAI-I/O-bound so it spends most
          # of its time idle waiting for the API response anyway.
          cpus: "0.15"

volumes:
  knowledge_pgdata:
    external: true
  knowledge_models:
    external: true
  knowledge_pgbackups:
  # Logs + state files (continuous pipeline history / pid / rotation
  # cursor) for the daemons. Survives `docker compose down` so the
  # 12 h per-user cooldown isn't reset by a restart.
  knowledge_daemon_logs:

networks:
  default:
  dokploy-network:
    external: true