From 8fec38e4fc9a388a23815781d00e97416ca83b8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CSid?= Date: Mon, 27 Apr 2026 15:54:24 -0400 Subject: [PATCH 01/10] docs: add local web control plane setup --- .env.example | 53 ++++++++++++ docs/LOCAL_WEB_CONTROL_PLANE.md | 118 +++++++++++++++++++++++++++ docs/SETUP_GUIDE.md | 41 ++++++++-- package.json | 5 ++ packages/control-plane/package.json | 1 + scripts/d1-migrate-local.sh | 43 ++++++++++ scripts/local-env.mjs | 100 +++++++++++++++++++++++ scripts/modal-secrets.mjs | 122 ++++++++++++++++++++++++++++ 8 files changed, 477 insertions(+), 6 deletions(-) create mode 100644 .env.example create mode 100644 docs/LOCAL_WEB_CONTROL_PLANE.md create mode 100755 scripts/d1-migrate-local.sh create mode 100755 scripts/local-env.mjs create mode 100755 scripts/modal-secrets.mjs diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..89a3d537b --- /dev/null +++ b/.env.example @@ -0,0 +1,53 @@ +# Local full-stack development against real GitHub + Modal services. +# Copy to .env.local, fill values, then run `npm run dev:env`. + +# GitHub App OAuth +GITHUB_CLIENT_ID= +GITHUB_CLIENT_SECRET= + +# GitHub App installation access +GITHUB_APP_ID= +# Use escaped newlines for this single-line env file, for example: +# GITHUB_APP_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----" +GITHUB_APP_PRIVATE_KEY= +GITHUB_APP_INSTALLATION_ID= + +# Modal +MODAL_TOKEN_ID= +MODAL_TOKEN_SECRET= +MODAL_WORKSPACE= +MODAL_API_SECRET= + +# LLM keys injected by Modal +ANTHROPIC_API_KEY= + +# Shared app secrets +TOKEN_ENCRYPTION_KEY= +REPO_SECRETS_ENCRYPTION_KEY= +INTERNAL_CALLBACK_SECRET= +NEXTAUTH_SECRET= + +# Local service URLs +NEXTAUTH_URL=http://localhost:3000 +# Modal sandboxes run remotely and cannot call back to localhost on your machine. +# For Path C, expose the local control plane with a tunnel (for example `ngrok http 8787`) and set: +# CONTROL_PLANE_URL=https:// +# NEXT_PUBLIC_WS_URL=wss:// +# WORKER_URL=https:// +CONTROL_PLANE_URL=http://localhost:8787 +NEXT_PUBLIC_WS_URL=ws://localhost:8787 +WEB_APP_URL=http://localhost:3000 +WORKER_URL=http://localhost:8787 +DEPLOYMENT_NAME=local +SANDBOX_PROVIDER=modal + +# Access control. Prefer ALLOWED_USERS for local real-service runs. +ALLOWED_USERS= +ALLOWED_EMAIL_DOMAINS= +UNSAFE_ALLOW_ALL_USERS=false + +# Optional +SCM_PROVIDER=github +NEXT_PUBLIC_SCM_PROVIDER=github +NEXT_PUBLIC_SANDBOX_PROVIDER=modal +LOG_LEVEL=debug diff --git a/docs/LOCAL_WEB_CONTROL_PLANE.md b/docs/LOCAL_WEB_CONTROL_PLANE.md new file mode 100644 index 000000000..11682c041 --- /dev/null +++ b/docs/LOCAL_WEB_CONTROL_PLANE.md @@ -0,0 +1,118 @@ +# Local Web And Control Plane + +Run the web app and control plane on your machine while using real GitHub App credentials and real +Modal sandboxes. + +## Services + +- Web: `http://localhost:3000` +- Control plane: `http://localhost:8787` +- Modal: deployed `open-inspect` app in your Modal workspace +- GitHub: real GitHub App OAuth and installation APIs + +## Setup + +Complete [Step 0 in SETUP_GUIDE.md](./SETUP_GUIDE.md#step-0-bootstrap-the-repo) first. + +1. Copy the root env template: + +```bash +cp .env.example .env.local +``` + +2. Fill in `.env.local` with the real GitHub App, Modal, and app secrets. + +Use escaped newlines for `GITHUB_APP_PRIVATE_KEY` because `.env.local` is a single-line env file: + +```bash +GITHUB_APP_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----" +``` + +3. Expose the local control plane to Modal. + +Modal sandboxes run remotely, so they cannot connect back to `localhost:8787` on your machine. Start +a tunnel to the local control plane port: + +```bash +ngrok http 8787 +``` + +Update `.env.local` with the tunnel URL: + +```bash +CONTROL_PLANE_URL=https:// +NEXT_PUBLIC_WS_URL=wss:// +WORKER_URL=https:// +``` + +Keep `NEXTAUTH_URL=http://localhost:3000`; the web app still runs locally. + +4. Generate package env files: + +```bash +npm run dev:env +``` + +This writes: + +- `packages/web/.env.local` +- `packages/control-plane/.dev.vars` + +5. Configure Modal secrets from `.env.local`: + +```bash +npm run dev:modal-secrets +``` + +At minimum Modal needs: + +- `llm-api-keys`: `ANTHROPIC_API_KEY` +- `github-app`: `GITHUB_APP_ID`, `GITHUB_APP_PRIVATE_KEY`, `GITHUB_APP_INSTALLATION_ID` +- `internal-api`: `MODAL_API_SECRET`, `INTERNAL_CALLBACK_SECRET`, `ALLOWED_CONTROL_PLANE_HOSTS`, + `CONTROL_PLANE_URL` + +For remote Modal callbacks, `ALLOWED_CONTROL_PLANE_HOSTS` must include the tunnel host. The +`dev:modal-secrets` script derives this from `CONTROL_PLANE_URL` unless you set +`ALLOWED_CONTROL_PLANE_HOSTS` explicitly. + +6. Deploy Modal: + +```bash +cd packages/modal-infra +uv sync --frozen --extra dev +uv run modal deploy deploy.py +``` + +7. Apply local D1 migrations: + +```bash +npm run dev:db:local +``` + +8. Run the control plane: + +```bash +npm run dev:control-plane +``` + +9. In another terminal, run the web app: + +```bash +npm run dev:web +``` + +10. Open `http://localhost:3000`. + +## GitHub App Callback + +Your real GitHub App must include this callback URL for local web auth: + +```text +http://localhost:3000/api/auth/callback/github +``` + +## Notes + +- The control plane uses local Wrangler storage for D1, KV, R2, and Durable Objects. +- Modal remains remote, so sandboxes must be able to call back through the tunnel. If the ngrok URL + changes, update `.env.local`, then rerun `npm run dev:env` and `npm run dev:modal-secrets`. diff --git a/docs/SETUP_GUIDE.md b/docs/SETUP_GUIDE.md index 6d6188ae6..6014b7e64 100644 --- a/docs/SETUP_GUIDE.md +++ b/docs/SETUP_GUIDE.md @@ -4,11 +4,12 @@ This is the primary setup guide for users and contributors. It is organized by goal so you can pick the fastest path: -| Path | Best For | Time | -| ------ | -------------------------------------------------------- | ---------- | -| Path A | Run the web app locally against an existing backend | ~10-20 min | -| Path B | Contribute code locally (lint/typecheck/tests) | ~15-30 min | -| Path C | Deploy your own full stack (Cloudflare + Modal + Vercel) | ~1-3 hours | +| Path | Best For | Time | +| ------ | --------------------------------------------------------- | ---------- | +| Path A | Run the web app locally against an existing backend | ~10-20 min | +| Path B | Contribute code locally (lint/typecheck/tests) | ~15-30 min | +| Path C | Run local web + control plane against real GitHub + Modal | ~30-60 min | +| Path D | Deploy your own full stack (Cloudflare + Modal + Vercel) | ~1-3 hours | ## Important Context @@ -30,6 +31,12 @@ Optional (needed for `modal-infra` development): - `uv` (recommended) or `pip` - Modal CLI (`modal`) +Optional (needed for Path C local web + control plane): + +- Wrangler CLI (`npx wrangler` from repo dependencies is fine) +- `jq` for local D1 migration scripts +- A tunnel tool such as `ngrok` so remote Modal sandboxes can call back to the local control plane + Optional (needed for full deployment): - Terraform `1.6+` @@ -41,6 +48,12 @@ Quick check: node -v npm -v git --version +python --version +uv --version +modal --version +npx wrangler --version +jq --version +ngrok --version ``` ## Step 0: Bootstrap the Repo @@ -183,7 +196,22 @@ pip install -e ".[dev]" pytest tests/ -v ``` -## Path C: Full Self-Hosted Deployment +## Path C: Run Local Web And Control Plane + +Use this when developing or debugging behavior that requires the web app and control plane to run +together locally, while still using real GitHub App credentials and real Modal sandboxes. + +This path is most useful for control-plane and session-flow changes that need real end-to-end +behavior: auth, session creation, Durable Objects, WebSockets, Modal sandbox creation, bridge +callbacks, and streamed events. + +The detailed guide is here [LOCAL_WEB_CONTROL_PLANE.md](./LOCAL_WEB_CONTROL_PLANE.md) + +Because Modal sandboxes run remotely, this path requires a tunnel from Modal back to the local +control plane. The detailed guide covers the required `CONTROL_PLANE_URL`, `NEXT_PUBLIC_WS_URL`, and +`WORKER_URL` settings. + +## Path D: Full Self-Hosted Deployment For full infrastructure setup, use: @@ -223,5 +251,6 @@ Control plane cannot reach Modal (or Modal is not properly configured/deployed). - Architecture and internals: [docs/HOW_IT_WORKS.md](./HOW_IT_WORKS.md) - Full production deployment: [docs/GETTING_STARTED.md](./GETTING_STARTED.md) - Debugging and observability: [docs/DEBUGGING_PLAYBOOK.md](./DEBUGGING_PLAYBOOK.md) +- Local web and control plane: [docs/LOCAL_WEB_CONTROL_PLANE.md](./LOCAL_WEB_CONTROL_PLANE.md) - OpenAI model setup: [docs/OPENAI_MODELS.md](./OPENAI_MODELS.md) - Contribution workflow: [CONTRIBUTING.md](../CONTRIBUTING.md) diff --git a/package.json b/package.json index ca360b791..4de41da5b 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,11 @@ "packages/*" ], "scripts": { + "dev:env": "node scripts/local-env.mjs", + "dev:modal-secrets": "node scripts/modal-secrets.mjs", + "dev:db:local": "bash scripts/d1-migrate-local.sh", + "dev:control-plane": "npm run dev -w @open-inspect/control-plane", + "dev:web": "npm run dev -w @open-inspect/web", "lint": "eslint .", "lint:fix": "eslint . --fix", "format": "prettier --write .", diff --git a/packages/control-plane/package.json b/packages/control-plane/package.json index 4e7f6f019..81cf9725f 100644 --- a/packages/control-plane/package.json +++ b/packages/control-plane/package.json @@ -4,6 +4,7 @@ "private": true, "type": "module", "scripts": { + "dev": "wrangler dev --port 8787", "build": "esbuild src/index.ts --bundle --format=esm --outfile=dist/index.js --platform=browser --target=es2022 --external:cloudflare:* --external:node:*", "test": "vitest run", "test:coverage": "vitest run --coverage", diff --git a/scripts/d1-migrate-local.sh b/scripts/d1-migrate-local.sh new file mode 100755 index 000000000..150134d49 --- /dev/null +++ b/scripts/d1-migrate-local.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +DATABASE_NAME="${1:-open-inspect-test}" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MIGRATIONS_DIR="${2:-$SCRIPT_DIR/../terraform/d1/migrations}" +CONFIG_PATH="$SCRIPT_DIR/../packages/control-plane/wrangler.jsonc" + +WRANGLER="npx wrangler" + +$WRANGLER d1 execute "$DATABASE_NAME" --local --config "$CONFIG_PATH" \ + --command "CREATE TABLE IF NOT EXISTS _schema_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + )" + +APPLIED=$($WRANGLER d1 execute "$DATABASE_NAME" --local --config "$CONFIG_PATH" \ + --command "SELECT version FROM _schema_migrations ORDER BY version" \ + --json | jq -r '.[0].results[].version // empty' 2>/dev/null || true) + +COUNT=0 +for file in "$MIGRATIONS_DIR"/*.sql; do + [ -f "$file" ] || continue + FILENAME=$(basename "$file") + VERSION=$(printf "%s" "$FILENAME" | grep -oE '^[0-9]+') + + if printf "%s\n" "$APPLIED" | grep -qxF "$VERSION"; then + printf "Skip (already applied): %s\n" "$FILENAME" + continue + fi + + printf "Applying: %s\n" "$FILENAME" + $WRANGLER d1 execute "$DATABASE_NAME" --local --config "$CONFIG_PATH" --file "$file" + + SAFE_FILENAME=$(printf "%s" "$FILENAME" | sed "s/'/''/g") + $WRANGLER d1 execute "$DATABASE_NAME" --local --config "$CONFIG_PATH" \ + --command "INSERT INTO _schema_migrations (version, name) VALUES ('$VERSION', '$SAFE_FILENAME')" + + COUNT=$((COUNT + 1)) +done + +printf "Done. Applied %d migration(s).\n" "$COUNT" diff --git a/scripts/local-env.mjs b/scripts/local-env.mjs new file mode 100755 index 000000000..95c67e5c0 --- /dev/null +++ b/scripts/local-env.mjs @@ -0,0 +1,100 @@ +#!/usr/bin/env node +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; + +const root = resolve(new URL("..", import.meta.url).pathname); +const sourcePath = resolve(root, ".env.local"); + +function parseEnv(contents) { + const env = {}; + + for (const rawLine of contents.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith("#")) continue; + + const index = line.indexOf("="); + if (index === -1) continue; + + const key = line.slice(0, index).trim(); + let value = line.slice(index + 1).trim(); + + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + + env[key] = value.replace(/\\n/g, "\n"); + } + + return env; +} + +function serializeEnv(env, keys) { + return `${keys + .map((key) => { + const value = env[key] ?? ""; + const serialized = value.includes("\n") ? JSON.stringify(value) : value; + return `${key}=${serialized}`; + }) + .join("\n")}\n`; +} + +function writeEnv(path, contents) { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, contents); + console.log(`wrote ${path.replace(`${root}/`, "")}`); +} + +let source; +try { + source = readFileSync(sourcePath, "utf8"); +} catch { + console.error("Missing .env.local. Copy .env.example to .env.local and fill in real values."); + process.exit(1); +} + +const env = parseEnv(source); + +const webKeys = [ + "GITHUB_CLIENT_ID", + "GITHUB_CLIENT_SECRET", + "NEXTAUTH_URL", + "NEXTAUTH_SECRET", + "CONTROL_PLANE_URL", + "NEXT_PUBLIC_WS_URL", + "NEXT_PUBLIC_SANDBOX_PROVIDER", + "NEXT_PUBLIC_SCM_PROVIDER", + "INTERNAL_CALLBACK_SECRET", + "ALLOWED_USERS", + "ALLOWED_EMAIL_DOMAINS", + "UNSAFE_ALLOW_ALL_USERS", +]; + +const controlPlaneKeys = [ + "GITHUB_CLIENT_ID", + "GITHUB_CLIENT_SECRET", + "TOKEN_ENCRYPTION_KEY", + "REPO_SECRETS_ENCRYPTION_KEY", + "INTERNAL_CALLBACK_SECRET", + "GITHUB_APP_ID", + "GITHUB_APP_PRIVATE_KEY", + "GITHUB_APP_INSTALLATION_ID", + "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET", + "MODAL_API_SECRET", + "MODAL_WORKSPACE", + "DEPLOYMENT_NAME", + "SCM_PROVIDER", + "SANDBOX_PROVIDER", + "WORKER_URL", + "WEB_APP_URL", + "LOG_LEVEL", +]; + +writeEnv(resolve(root, "packages/web/.env.local"), serializeEnv(env, webKeys)); +writeEnv(resolve(root, "packages/control-plane/.dev.vars"), serializeEnv(env, controlPlaneKeys)); + +console.log("\nNext, sync Modal secrets from .env.local:"); +console.log(" npm run dev:modal-secrets"); diff --git a/scripts/modal-secrets.mjs b/scripts/modal-secrets.mjs new file mode 100755 index 000000000..bf7a80956 --- /dev/null +++ b/scripts/modal-secrets.mjs @@ -0,0 +1,122 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +const root = resolve(new URL("..", import.meta.url).pathname); +const sourcePath = resolve(root, ".env.local"); + +function parseEnv(contents) { + const env = {}; + + for (const rawLine of contents.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith("#")) continue; + + const index = line.indexOf("="); + if (index === -1) continue; + + const key = line.slice(0, index).trim(); + let value = line.slice(index + 1).trim(); + + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + + env[key] = value.replace(/\\n/g, "\n"); + } + + return env; +} + +function requireEnv(env, keys) { + const missing = keys.filter((key) => !env[key]); + if (missing.length > 0) { + console.error(`Missing required values in .env.local: ${missing.join(", ")}`); + process.exit(1); + } +} + +function shellWords(command) { + return command.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g)?.map((part) => { + if ( + (part.startsWith('"') && part.endsWith('"')) || + (part.startsWith("'") && part.endsWith("'")) + ) { + return part.slice(1, -1); + } + return part; + }) ?? [command]; +} + +function createSecret(modalCommand, secretName, values) { + const [command, ...baseArgs] = modalCommand; + const args = [ + ...baseArgs, + "secret", + "create", + secretName, + ...Object.entries(values).map(([key, value]) => `${key}=${value}`), + "--force", + ]; + + console.log(`creating/updating Modal secret: ${secretName}`); + const result = spawnSync(command, args, { + stdio: "inherit", + cwd: root, + }); + + if (result.error) { + console.error(result.error.message); + process.exit(1); + } + + if (result.status !== 0) { + process.exit(result.status ?? 1); + } +} + +let source; +try { + source = readFileSync(sourcePath, "utf8"); +} catch { + console.error("Missing .env.local. Copy .env.example to .env.local and fill in real values."); + process.exit(1); +} + +const env = parseEnv(source); +requireEnv(env, [ + "ANTHROPIC_API_KEY", + "GITHUB_APP_ID", + "GITHUB_APP_PRIVATE_KEY", + "GITHUB_APP_INSTALLATION_ID", + "MODAL_API_SECRET", + "INTERNAL_CALLBACK_SECRET", + "CONTROL_PLANE_URL", +]); + +const controlPlaneUrl = new URL(env.CONTROL_PLANE_URL); +const allowedHosts = env.ALLOWED_CONTROL_PLANE_HOSTS || controlPlaneUrl.host; +const modalCommand = shellWords(process.env.MODAL_CLI || "modal"); + +createSecret(modalCommand, "llm-api-keys", { + ANTHROPIC_API_KEY: env.ANTHROPIC_API_KEY, +}); + +createSecret(modalCommand, "github-app", { + GITHUB_APP_ID: env.GITHUB_APP_ID, + GITHUB_APP_PRIVATE_KEY: env.GITHUB_APP_PRIVATE_KEY, + GITHUB_APP_INSTALLATION_ID: env.GITHUB_APP_INSTALLATION_ID, +}); + +createSecret(modalCommand, "internal-api", { + MODAL_API_SECRET: env.MODAL_API_SECRET, + INTERNAL_CALLBACK_SECRET: env.INTERNAL_CALLBACK_SECRET, + ALLOWED_CONTROL_PLANE_HOSTS: allowedHosts, + CONTROL_PLANE_URL: env.CONTROL_PLANE_URL, +}); + +console.log("Modal secrets are up to date."); From 6db8c5cbf60e172b6db170a8560c573c9146217c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CSid?= Date: Tue, 28 Apr 2026 10:14:57 -0400 Subject: [PATCH 02/10] consolidate d1 migration scripts --- package.json | 2 +- scripts/d1-migrate-local.sh | 43 ------------------- scripts/d1-migrate.sh | 55 ++++++++++++++++++------- terraform/environments/production/d1.tf | 2 +- 4 files changed, 41 insertions(+), 61 deletions(-) delete mode 100755 scripts/d1-migrate-local.sh diff --git a/package.json b/package.json index 4de41da5b..8d477e2e0 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "scripts": { "dev:env": "node scripts/local-env.mjs", "dev:modal-secrets": "node scripts/modal-secrets.mjs", - "dev:db:local": "bash scripts/d1-migrate-local.sh", + "dev:db:local": "bash scripts/d1-migrate.sh local", "dev:control-plane": "npm run dev -w @open-inspect/control-plane", "dev:web": "npm run dev -w @open-inspect/web", "lint": "eslint .", diff --git a/scripts/d1-migrate-local.sh b/scripts/d1-migrate-local.sh deleted file mode 100755 index 150134d49..000000000 --- a/scripts/d1-migrate-local.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -DATABASE_NAME="${1:-open-inspect-test}" -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -MIGRATIONS_DIR="${2:-$SCRIPT_DIR/../terraform/d1/migrations}" -CONFIG_PATH="$SCRIPT_DIR/../packages/control-plane/wrangler.jsonc" - -WRANGLER="npx wrangler" - -$WRANGLER d1 execute "$DATABASE_NAME" --local --config "$CONFIG_PATH" \ - --command "CREATE TABLE IF NOT EXISTS _schema_migrations ( - version TEXT PRIMARY KEY, - name TEXT NOT NULL, - applied_at TEXT NOT NULL DEFAULT (datetime('now')) - )" - -APPLIED=$($WRANGLER d1 execute "$DATABASE_NAME" --local --config "$CONFIG_PATH" \ - --command "SELECT version FROM _schema_migrations ORDER BY version" \ - --json | jq -r '.[0].results[].version // empty' 2>/dev/null || true) - -COUNT=0 -for file in "$MIGRATIONS_DIR"/*.sql; do - [ -f "$file" ] || continue - FILENAME=$(basename "$file") - VERSION=$(printf "%s" "$FILENAME" | grep -oE '^[0-9]+') - - if printf "%s\n" "$APPLIED" | grep -qxF "$VERSION"; then - printf "Skip (already applied): %s\n" "$FILENAME" - continue - fi - - printf "Applying: %s\n" "$FILENAME" - $WRANGLER d1 execute "$DATABASE_NAME" --local --config "$CONFIG_PATH" --file "$file" - - SAFE_FILENAME=$(printf "%s" "$FILENAME" | sed "s/'/''/g") - $WRANGLER d1 execute "$DATABASE_NAME" --local --config "$CONFIG_PATH" \ - --command "INSERT INTO _schema_migrations (version, name) VALUES ('$VERSION', '$SAFE_FILENAME')" - - COUNT=$((COUNT + 1)) -done - -printf "Done. Applied %d migration(s).\n" "$COUNT" diff --git a/scripts/d1-migrate.sh b/scripts/d1-migrate.sh index 41d8cd498..b4dea7608 100755 --- a/scripts/d1-migrate.sh +++ b/scripts/d1-migrate.sh @@ -1,45 +1,68 @@ #!/usr/bin/env bash set -euo pipefail -DATABASE_NAME="${1:?Usage: d1-migrate.sh [migrations-dir]}" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -MIGRATIONS_DIR="${2:-$SCRIPT_DIR/../terraform/d1/migrations}" +MODE="${1:-}" WRANGLER="npx wrangler" -# 1. Ensure tracking table exists -$WRANGLER d1 execute "$DATABASE_NAME" --remote \ +usage() { + printf "Usage:\n" + printf " d1-migrate.sh local [database-name] [migrations-dir]\n" + printf " d1-migrate.sh remote [migrations-dir]\n" +} + +case "$MODE" in + local) + DATABASE_NAME="${2:-open-inspect-test}" + MIGRATIONS_DIR="${3:-$SCRIPT_DIR/../terraform/d1/migrations}" + D1_OPTIONS=(--local --config "$SCRIPT_DIR/../packages/control-plane/wrangler.jsonc") + ;; + remote) + if [ -z "${2:-}" ]; then + usage + exit 1 + fi + DATABASE_NAME="$2" + MIGRATIONS_DIR="${3:-$SCRIPT_DIR/../terraform/d1/migrations}" + D1_OPTIONS=(--remote) + ;; + *) + usage + exit 1 + ;; +esac + +$WRANGLER d1 execute "$DATABASE_NAME" "${D1_OPTIONS[@]}" \ --command "CREATE TABLE IF NOT EXISTS _schema_migrations ( version TEXT PRIMARY KEY, name TEXT NOT NULL, applied_at TEXT NOT NULL DEFAULT (datetime('now')) )" -# 2. Get applied versions (parse JSON output) -APPLIED=$($WRANGLER d1 execute "$DATABASE_NAME" --remote \ +APPLIED=$($WRANGLER d1 execute "$DATABASE_NAME" "${D1_OPTIONS[@]}" \ --command "SELECT version FROM _schema_migrations ORDER BY version" \ - --json | jq -r '.[0].results[].version // empty' 2>/dev/null || echo "") + --json | jq -r '.[0].results[].version // empty' 2>/dev/null || true) -# 3. Apply pending migrations in order COUNT=0 for file in "$MIGRATIONS_DIR"/*.sql; do [ -f "$file" ] || continue FILENAME=$(basename "$file") - VERSION=$(echo "$FILENAME" | grep -oE '^[0-9]+') + VERSION=$(printf "%s" "$FILENAME" | grep -oE '^[0-9]+') - if echo "$APPLIED" | grep -qxF "$VERSION"; then - echo "Skip (already applied): $FILENAME" + if printf "%s\n" "$APPLIED" | grep -qxF "$VERSION"; then + printf "Skip (already applied): %s\n" "$FILENAME" continue fi - echo "Applying: $FILENAME" - $WRANGLER d1 execute "$DATABASE_NAME" --remote --file "$file" + printf "Applying: %s\n" "$FILENAME" + $WRANGLER d1 execute "$DATABASE_NAME" "${D1_OPTIONS[@]}" --file "$file" - SAFE_FILENAME=$(echo "$FILENAME" | sed "s/'/''/g") - $WRANGLER d1 execute "$DATABASE_NAME" --remote \ + SAFE_FILENAME=$(printf "%s" "$FILENAME" | sed "s/'/''/g") + $WRANGLER d1 execute "$DATABASE_NAME" "${D1_OPTIONS[@]}" \ --command "INSERT INTO _schema_migrations (version, name) VALUES ('$VERSION', '$SAFE_FILENAME')" COUNT=$((COUNT + 1)) done -echo "Done. Applied $COUNT migration(s)." +printf "Done. Applied %d migration(s).\n" "$COUNT" diff --git a/terraform/environments/production/d1.tf b/terraform/environments/production/d1.tf index ea8f2c508..0e8e0e260 100644 --- a/terraform/environments/production/d1.tf +++ b/terraform/environments/production/d1.tf @@ -23,7 +23,7 @@ resource "null_resource" "d1_migrations" { } provisioner "local-exec" { - command = "bash scripts/d1-migrate.sh ${cloudflare_d1_database.main.name} terraform/d1/migrations" + command = "bash scripts/d1-migrate.sh remote ${cloudflare_d1_database.main.name} terraform/d1/migrations" working_dir = var.project_root environment = { From c0ed02c8ae8980eb3c61c7886b8e7fe5c1b1077d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CSid?= Date: Tue, 28 Apr 2026 10:58:57 -0400 Subject: [PATCH 03/10] refactor: dedupe D1 migration directory default --- scripts/d1-migrate.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/d1-migrate.sh b/scripts/d1-migrate.sh index b4dea7608..7396fc7f4 100755 --- a/scripts/d1-migrate.sh +++ b/scripts/d1-migrate.sh @@ -3,6 +3,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" MODE="${1:-}" +MIGRATIONS_DIR="${3:-$SCRIPT_DIR/../terraform/d1/migrations}" WRANGLER="npx wrangler" @@ -15,7 +16,6 @@ usage() { case "$MODE" in local) DATABASE_NAME="${2:-open-inspect-test}" - MIGRATIONS_DIR="${3:-$SCRIPT_DIR/../terraform/d1/migrations}" D1_OPTIONS=(--local --config "$SCRIPT_DIR/../packages/control-plane/wrangler.jsonc") ;; remote) @@ -24,7 +24,6 @@ case "$MODE" in exit 1 fi DATABASE_NAME="$2" - MIGRATIONS_DIR="${3:-$SCRIPT_DIR/../terraform/d1/migrations}" D1_OPTIONS=(--remote) ;; *) From d57e7d0f6686dbdf898d233d378fa47fe4d22d5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CSid?= Date: Tue, 28 Apr 2026 21:49:47 -0400 Subject: [PATCH 04/10] chore: centralize local env generation --- .env.example | 15 ++- docs/LOCAL_WEB_CONTROL_PLANE.md | 57 ++++++++++- eslint.config.js | 14 +++ package-lock.json | 26 +---- package.json | 1 + scripts/env-utils.mjs | 168 ++++++++++++++++++++++++++++++++ scripts/local-env.mjs | 116 ++++------------------ scripts/modal-secrets.mjs | 84 ++++------------ 8 files changed, 287 insertions(+), 194 deletions(-) create mode 100644 scripts/env-utils.mjs diff --git a/.env.example b/.env.example index 89a3d537b..b669c2245 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,4 @@ -# Local full-stack development against real GitHub + Modal services. +# Local full-stack development against real GitHub + sandbox services. # Copy to .env.local, fill values, then run `npm run dev:env`. # GitHub App OAuth @@ -12,12 +12,20 @@ GITHUB_APP_ID= GITHUB_APP_PRIVATE_KEY= GITHUB_APP_INSTALLATION_ID= -# Modal +# Sandbox provider. Supported values: modal, daytona. +SANDBOX_PROVIDER=modal + +# Modal sandbox provider. Required when SANDBOX_PROVIDER=modal. MODAL_TOKEN_ID= MODAL_TOKEN_SECRET= MODAL_WORKSPACE= MODAL_API_SECRET= +# Daytona sandbox provider. Required when SANDBOX_PROVIDER=daytona. +DAYTONA_API_KEY= +DAYTONA_API_URL= +DAYTONA_BASE_SNAPSHOT= + # LLM keys injected by Modal ANTHROPIC_API_KEY= @@ -39,7 +47,6 @@ NEXT_PUBLIC_WS_URL=ws://localhost:8787 WEB_APP_URL=http://localhost:3000 WORKER_URL=http://localhost:8787 DEPLOYMENT_NAME=local -SANDBOX_PROVIDER=modal # Access control. Prefer ALLOWED_USERS for local real-service runs. ALLOWED_USERS= @@ -48,6 +55,4 @@ UNSAFE_ALLOW_ALL_USERS=false # Optional SCM_PROVIDER=github -NEXT_PUBLIC_SCM_PROVIDER=github -NEXT_PUBLIC_SANDBOX_PROVIDER=modal LOG_LEVEL=debug diff --git a/docs/LOCAL_WEB_CONTROL_PLANE.md b/docs/LOCAL_WEB_CONTROL_PLANE.md index 11682c041..9e65dbc24 100644 --- a/docs/LOCAL_WEB_CONTROL_PLANE.md +++ b/docs/LOCAL_WEB_CONTROL_PLANE.md @@ -1,13 +1,13 @@ # Local Web And Control Plane Run the web app and control plane on your machine while using real GitHub App credentials and real -Modal sandboxes. +sandbox services. ## Services - Web: `http://localhost:3000` - Control plane: `http://localhost:8787` -- Modal: deployed `open-inspect` app in your Modal workspace +- Modal or Daytona: deployed sandbox backend - GitHub: real GitHub App OAuth and installation APIs ## Setup @@ -20,7 +20,7 @@ Complete [Step 0 in SETUP_GUIDE.md](./SETUP_GUIDE.md#step-0-bootstrap-the-repo) cp .env.example .env.local ``` -2. Fill in `.env.local` with the real GitHub App, Modal, and app secrets. +2. Fill in `.env.local` with the real GitHub App, sandbox provider, and app secrets. Use escaped newlines for `GITHUB_APP_PRIVATE_KEY` because `.env.local` is a single-line env file: @@ -75,6 +75,8 @@ For remote Modal callbacks, `ALLOWED_CONTROL_PLANE_HOSTS` must include the tunne `dev:modal-secrets` script derives this from `CONTROL_PLANE_URL` unless you set `ALLOWED_CONTROL_PLANE_HOSTS` explicitly. +Skip this step when `SANDBOX_PROVIDER=daytona`; `dev:modal-secrets` is only needed for Modal. + 6. Deploy Modal: ```bash @@ -111,6 +113,55 @@ Your real GitHub App must include this callback URL for local web auth: http://localhost:3000/api/auth/callback/github ``` +## Env File Management + +Local full-stack development uses the root `.env.local` as the only editable env file. Do not edit +the generated service-specific files directly. + +Source file: + +- `.env.local` — copied from `.env.example` and edited by you + +Generated files: + +- `packages/web/.env.local` — consumed by Next.js +- `packages/control-plane/.dev.vars` — consumed by Wrangler + +Run this after changing `.env.local`: + +```bash +npm run dev:env +``` + +`dev:env` validates required values, writes the generated files, and stamps them with: + +```text +# Generated by npm run dev:env. Do not edit directly. +``` + +Validation is provider-aware: + +- `SANDBOX_PROVIDER=modal` requires Modal credentials and Modal callback secrets +- `SANDBOX_PROVIDER=daytona` requires Daytona credentials instead and does not require Modal tokens + +The generator also derives public provider values for the web app: + +- `NEXT_PUBLIC_SANDBOX_PROVIDER` is derived from `SANDBOX_PROVIDER` +- `NEXT_PUBLIC_SCM_PROVIDER` is derived from `SCM_PROVIDER` + +Only set the `NEXT_PUBLIC_*` provider values in `.env.local` if you intentionally need to override +the derived value. + +Modal secrets are synced separately because Modal stores them in its own secret manager: + +```bash +npm run dev:modal-secrets +``` + +This command reads `.env.local`, validates Modal-specific values, and creates or updates the Modal +secrets used by the deployed Modal app. It exits without making changes when `SANDBOX_PROVIDER` is +not `modal`. + ## Notes - The control plane uses local Wrangler storage for D1, KV, R2, and Durable Objects. diff --git a/eslint.config.js b/eslint.config.js index 964decef8..db15d5a49 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -108,6 +108,20 @@ export default tseslint.config( }, }, + // Root maintenance scripts run in Node.js. + { + files: ["scripts/**/*.{js,mjs,cjs,ts}"], + languageOptions: { + globals: { + ...globals.node, + ...globals.es2022, + }, + }, + rules: { + "no-console": "off", + }, + }, + // Test files configuration { files: ["**/*.test.{ts,tsx}", "**/*.spec.{ts,tsx}"], diff --git a/package-lock.json b/package-lock.json index 0e942d5b5..8ea2c576b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "devDependencies": { "@eslint/js": "^9.18.0", "@types/node": "^22.10.5", + "dotenv": "^16.6.1", "eslint": "^9.18.0", "eslint-config-prettier": "^10.0.1", "eslint-plugin-react": "^7.37.4", @@ -2679,7 +2680,6 @@ "version": "1.8.1", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", "integrity": "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -3410,7 +3410,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3433,7 +3432,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3456,7 +3454,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3473,7 +3470,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3490,7 +3486,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3507,7 +3502,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3524,7 +3518,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3541,7 +3534,6 @@ "cpu": [ "riscv64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3558,7 +3550,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3575,7 +3566,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3592,7 +3582,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3609,7 +3598,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3626,7 +3614,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3649,7 +3636,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3672,7 +3658,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3695,7 +3680,6 @@ "cpu": [ "riscv64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3718,7 +3702,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3741,7 +3724,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3764,7 +3746,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3787,7 +3768,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3810,7 +3790,6 @@ "cpu": [ "wasm32" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", "optional": true, "dependencies": { @@ -3830,7 +3809,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -3850,7 +3828,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -3870,7 +3847,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ diff --git a/package.json b/package.json index 8d477e2e0..d2cf734b5 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "devDependencies": { "@eslint/js": "^9.18.0", "@types/node": "^22.10.5", + "dotenv": "^16.6.1", "eslint": "^9.18.0", "eslint-config-prettier": "^10.0.1", "eslint-plugin-react": "^7.37.4", diff --git a/scripts/env-utils.mjs b/scripts/env-utils.mjs new file mode 100644 index 000000000..f406d924c --- /dev/null +++ b/scripts/env-utils.mjs @@ -0,0 +1,168 @@ +import dotenv from "dotenv"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; + +export const root = resolve(new URL("..", import.meta.url).pathname); +export const sourcePath = resolve(root, ".env.local"); + +export const GENERATED_ENV_HEADER = "# Generated by npm run dev:env. Do not edit directly."; + +export const serviceEnvKeys = { + web: [ + "GITHUB_CLIENT_ID", + "GITHUB_CLIENT_SECRET", + "NEXTAUTH_URL", + "NEXTAUTH_SECRET", + "CONTROL_PLANE_URL", + "NEXT_PUBLIC_WS_URL", + "NEXT_PUBLIC_SANDBOX_PROVIDER", + "NEXT_PUBLIC_SCM_PROVIDER", + "INTERNAL_CALLBACK_SECRET", + "ALLOWED_USERS", + "ALLOWED_EMAIL_DOMAINS", + "UNSAFE_ALLOW_ALL_USERS", + ], + controlPlane: [ + "GITHUB_CLIENT_ID", + "GITHUB_CLIENT_SECRET", + "TOKEN_ENCRYPTION_KEY", + "REPO_SECRETS_ENCRYPTION_KEY", + "INTERNAL_CALLBACK_SECRET", + "GITHUB_APP_ID", + "GITHUB_APP_PRIVATE_KEY", + "GITHUB_APP_INSTALLATION_ID", + "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET", + "MODAL_API_SECRET", + "MODAL_WORKSPACE", + "DAYTONA_API_KEY", + "DAYTONA_API_URL", + "DAYTONA_BASE_SNAPSHOT", + "DEPLOYMENT_NAME", + "SCM_PROVIDER", + "SANDBOX_PROVIDER", + "WORKER_URL", + "WEB_APP_URL", + "LOG_LEVEL", + ], + modalSecrets: [ + "ANTHROPIC_API_KEY", + "GITHUB_APP_ID", + "GITHUB_APP_PRIVATE_KEY", + "GITHUB_APP_INSTALLATION_ID", + "MODAL_API_SECRET", + "INTERNAL_CALLBACK_SECRET", + "CONTROL_PLANE_URL", + ], +}; + +export const requiredEnvKeys = { + web: [ + "GITHUB_CLIENT_ID", + "GITHUB_CLIENT_SECRET", + "NEXTAUTH_URL", + "NEXTAUTH_SECRET", + "CONTROL_PLANE_URL", + "NEXT_PUBLIC_WS_URL", + "INTERNAL_CALLBACK_SECRET", + ], + controlPlane: [ + "GITHUB_CLIENT_ID", + "GITHUB_CLIENT_SECRET", + "TOKEN_ENCRYPTION_KEY", + "REPO_SECRETS_ENCRYPTION_KEY", + "INTERNAL_CALLBACK_SECRET", + "GITHUB_APP_ID", + "GITHUB_APP_PRIVATE_KEY", + "GITHUB_APP_INSTALLATION_ID", + "DEPLOYMENT_NAME", + "WORKER_URL", + "WEB_APP_URL", + ], +}; + +const modalControlPlaneKeys = [ + "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET", + "MODAL_API_SECRET", + "MODAL_WORKSPACE", +]; + +const daytonaControlPlaneKeys = ["DAYTONA_API_KEY", "DAYTONA_API_URL", "DAYTONA_BASE_SNAPSHOT"]; + +export function getRequiredEnvKeys(env) { + const sandboxProvider = normalizeProvider(env.SANDBOX_PROVIDER, "modal"); + + if (sandboxProvider === "modal") { + return { + ...requiredEnvKeys, + controlPlane: [...requiredEnvKeys.controlPlane, ...modalControlPlaneKeys], + modalSecrets: serviceEnvKeys.modalSecrets, + }; + } + + if (sandboxProvider === "daytona") { + return { + ...requiredEnvKeys, + controlPlane: [...requiredEnvKeys.controlPlane, ...daytonaControlPlaneKeys], + modalSecrets: [], + }; + } + + console.error(`Unsupported SANDBOX_PROVIDER in .env.local: ${env.SANDBOX_PROVIDER}`); + process.exit(1); +} + +function normalizeProvider(value, defaultValue) { + return value?.trim().toLowerCase() || defaultValue; +} + +export function readRootEnv() { + let source; + try { + source = readFileSync(sourcePath, "utf8"); + } catch { + console.error("Missing .env.local. Copy .env.example to .env.local and fill in real values."); + process.exit(1); + } + + return withDerivedEnv(normalizeEnv(dotenv.parse(source))); +} + +export function normalizeEnv(env) { + return Object.fromEntries( + Object.entries(env).map(([key, value]) => [key, value.replace(/\\n/g, "\n")]) + ); +} + +export function withDerivedEnv(env) { + return { + ...env, + NEXT_PUBLIC_SANDBOX_PROVIDER: env.NEXT_PUBLIC_SANDBOX_PROVIDER || env.SANDBOX_PROVIDER, + NEXT_PUBLIC_SCM_PROVIDER: env.NEXT_PUBLIC_SCM_PROVIDER || env.SCM_PROVIDER, + }; +} + +export function requireEnv(env, keys, context = ".env.local") { + const missing = [...new Set(keys)].filter((key) => !env[key]); + if (missing.length > 0) { + console.error(`Missing required values in ${context}: ${missing.join(", ")}`); + process.exit(1); + } +} + +export function serializeEnv(env, keys) { + return `${GENERATED_ENV_HEADER}\n${keys + .map((key) => { + const value = env[key] ?? ""; + const serialized = value.includes("\n") ? JSON.stringify(value) : value; + return `${key}=${serialized}`; + }) + .join("\n")}\n`; +} + +export function writeEnv(path, contents) { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, contents); + console.log(`wrote ${path.replace(`${root}/`, "")}`); +} diff --git a/scripts/local-env.mjs b/scripts/local-env.mjs index 95c67e5c0..c13fe0e22 100755 --- a/scripts/local-env.mjs +++ b/scripts/local-env.mjs @@ -1,100 +1,24 @@ #!/usr/bin/env node -import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { dirname, resolve } from "node:path"; - -const root = resolve(new URL("..", import.meta.url).pathname); -const sourcePath = resolve(root, ".env.local"); - -function parseEnv(contents) { - const env = {}; - - for (const rawLine of contents.split(/\r?\n/)) { - const line = rawLine.trim(); - if (!line || line.startsWith("#")) continue; - - const index = line.indexOf("="); - if (index === -1) continue; - - const key = line.slice(0, index).trim(); - let value = line.slice(index + 1).trim(); - - if ( - (value.startsWith('"') && value.endsWith('"')) || - (value.startsWith("'") && value.endsWith("'")) - ) { - value = value.slice(1, -1); - } - - env[key] = value.replace(/\\n/g, "\n"); - } - - return env; -} - -function serializeEnv(env, keys) { - return `${keys - .map((key) => { - const value = env[key] ?? ""; - const serialized = value.includes("\n") ? JSON.stringify(value) : value; - return `${key}=${serialized}`; - }) - .join("\n")}\n`; -} - -function writeEnv(path, contents) { - mkdirSync(dirname(path), { recursive: true }); - writeFileSync(path, contents); - console.log(`wrote ${path.replace(`${root}/`, "")}`); -} - -let source; -try { - source = readFileSync(sourcePath, "utf8"); -} catch { - console.error("Missing .env.local. Copy .env.example to .env.local and fill in real values."); - process.exit(1); -} - -const env = parseEnv(source); - -const webKeys = [ - "GITHUB_CLIENT_ID", - "GITHUB_CLIENT_SECRET", - "NEXTAUTH_URL", - "NEXTAUTH_SECRET", - "CONTROL_PLANE_URL", - "NEXT_PUBLIC_WS_URL", - "NEXT_PUBLIC_SANDBOX_PROVIDER", - "NEXT_PUBLIC_SCM_PROVIDER", - "INTERNAL_CALLBACK_SECRET", - "ALLOWED_USERS", - "ALLOWED_EMAIL_DOMAINS", - "UNSAFE_ALLOW_ALL_USERS", -]; - -const controlPlaneKeys = [ - "GITHUB_CLIENT_ID", - "GITHUB_CLIENT_SECRET", - "TOKEN_ENCRYPTION_KEY", - "REPO_SECRETS_ENCRYPTION_KEY", - "INTERNAL_CALLBACK_SECRET", - "GITHUB_APP_ID", - "GITHUB_APP_PRIVATE_KEY", - "GITHUB_APP_INSTALLATION_ID", - "MODAL_TOKEN_ID", - "MODAL_TOKEN_SECRET", - "MODAL_API_SECRET", - "MODAL_WORKSPACE", - "DEPLOYMENT_NAME", - "SCM_PROVIDER", - "SANDBOX_PROVIDER", - "WORKER_URL", - "WEB_APP_URL", - "LOG_LEVEL", -]; - -writeEnv(resolve(root, "packages/web/.env.local"), serializeEnv(env, webKeys)); -writeEnv(resolve(root, "packages/control-plane/.dev.vars"), serializeEnv(env, controlPlaneKeys)); +import { resolve } from "node:path"; +import { + getRequiredEnvKeys, + readRootEnv, + requireEnv, + root, + serializeEnv, + serviceEnvKeys, + writeEnv, +} from "./env-utils.mjs"; + +const env = readRootEnv(); +const requiredEnvKeys = getRequiredEnvKeys(env); +requireEnv(env, [...requiredEnvKeys.web, ...requiredEnvKeys.controlPlane]); + +writeEnv(resolve(root, "packages/web/.env.local"), serializeEnv(env, serviceEnvKeys.web)); +writeEnv( + resolve(root, "packages/control-plane/.dev.vars"), + serializeEnv(env, serviceEnvKeys.controlPlane) +); console.log("\nNext, sync Modal secrets from .env.local:"); console.log(" npm run dev:modal-secrets"); diff --git a/scripts/modal-secrets.mjs b/scripts/modal-secrets.mjs index bf7a80956..a44d25549 100755 --- a/scripts/modal-secrets.mjs +++ b/scripts/modal-secrets.mjs @@ -1,55 +1,19 @@ #!/usr/bin/env node import { spawnSync } from "node:child_process"; -import { readFileSync } from "node:fs"; -import { resolve } from "node:path"; - -const root = resolve(new URL("..", import.meta.url).pathname); -const sourcePath = resolve(root, ".env.local"); - -function parseEnv(contents) { - const env = {}; - - for (const rawLine of contents.split(/\r?\n/)) { - const line = rawLine.trim(); - if (!line || line.startsWith("#")) continue; - - const index = line.indexOf("="); - if (index === -1) continue; - - const key = line.slice(0, index).trim(); - let value = line.slice(index + 1).trim(); - - if ( - (value.startsWith('"') && value.endsWith('"')) || - (value.startsWith("'") && value.endsWith("'")) - ) { - value = value.slice(1, -1); - } - - env[key] = value.replace(/\\n/g, "\n"); - } - - return env; -} - -function requireEnv(env, keys) { - const missing = keys.filter((key) => !env[key]); - if (missing.length > 0) { - console.error(`Missing required values in .env.local: ${missing.join(", ")}`); - process.exit(1); - } -} +import { getRequiredEnvKeys, readRootEnv, requireEnv, root } from "./env-utils.mjs"; function shellWords(command) { - return command.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g)?.map((part) => { - if ( - (part.startsWith('"') && part.endsWith('"')) || - (part.startsWith("'") && part.endsWith("'")) - ) { - return part.slice(1, -1); - } - return part; - }) ?? [command]; + return ( + command.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g)?.map((part) => { + if ( + (part.startsWith('"') && part.endsWith('"')) || + (part.startsWith("'") && part.endsWith("'")) + ) { + return part.slice(1, -1); + } + return part; + }) ?? [command] + ); } function createSecret(modalCommand, secretName, values) { @@ -79,24 +43,14 @@ function createSecret(modalCommand, secretName, values) { } } -let source; -try { - source = readFileSync(sourcePath, "utf8"); -} catch { - console.error("Missing .env.local. Copy .env.example to .env.local and fill in real values."); - process.exit(1); -} +const env = readRootEnv(); +const requiredEnvKeys = getRequiredEnvKeys(env); +requireEnv(env, requiredEnvKeys.modalSecrets); -const env = parseEnv(source); -requireEnv(env, [ - "ANTHROPIC_API_KEY", - "GITHUB_APP_ID", - "GITHUB_APP_PRIVATE_KEY", - "GITHUB_APP_INSTALLATION_ID", - "MODAL_API_SECRET", - "INTERNAL_CALLBACK_SECRET", - "CONTROL_PLANE_URL", -]); +if (requiredEnvKeys.modalSecrets.length === 0) { + console.log("Modal secrets are not required for SANDBOX_PROVIDER=daytona."); + process.exit(0); +} const controlPlaneUrl = new URL(env.CONTROL_PLANE_URL); const allowedHosts = env.ALLOWED_CONTROL_PLANE_HOSTS || controlPlaneUrl.host; From d274fae97c2c9a26dbcb68bb867ddbcb7c29563d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CSid?= Date: Tue, 28 Apr 2026 21:56:49 -0400 Subject: [PATCH 05/10] docs: clean up local env guide wrapping --- docs/LOCAL_WEB_CONTROL_PLANE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/LOCAL_WEB_CONTROL_PLANE.md b/docs/LOCAL_WEB_CONTROL_PLANE.md index 9e65dbc24..4f80739ab 100644 --- a/docs/LOCAL_WEB_CONTROL_PLANE.md +++ b/docs/LOCAL_WEB_CONTROL_PLANE.md @@ -31,7 +31,7 @@ GITHUB_APP_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-- 3. Expose the local control plane to Modal. Modal sandboxes run remotely, so they cannot connect back to `localhost:8787` on your machine. Start -a tunnel to the local control plane port: +a tunnel to expose the local control plane port: ```bash ngrok http 8787 From 55eec005678cd5bf5755d94ce02e62dfa9483ecc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CSid?= Date: Tue, 28 Apr 2026 21:57:27 -0400 Subject: [PATCH 06/10] docs: preserve local env guide prose lines --- docs/LOCAL_WEB_CONTROL_PLANE.md | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/docs/LOCAL_WEB_CONTROL_PLANE.md b/docs/LOCAL_WEB_CONTROL_PLANE.md index 4f80739ab..c3ebbf441 100644 --- a/docs/LOCAL_WEB_CONTROL_PLANE.md +++ b/docs/LOCAL_WEB_CONTROL_PLANE.md @@ -1,7 +1,7 @@ # Local Web And Control Plane -Run the web app and control plane on your machine while using real GitHub App credentials and real -sandbox services. + +Run the web app and control plane on your machine while using real GitHub App credentials and real sandbox services. ## Services @@ -30,8 +30,8 @@ GITHUB_APP_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-- 3. Expose the local control plane to Modal. -Modal sandboxes run remotely, so they cannot connect back to `localhost:8787` on your machine. Start -a tunnel to expose the local control plane port: + +Modal sandboxes run remotely, so they cannot connect back to `localhost:8787` on your machine. Start a tunnel to expose the local control plane port: ```bash ngrok http 8787 @@ -71,9 +71,8 @@ At minimum Modal needs: - `internal-api`: `MODAL_API_SECRET`, `INTERNAL_CALLBACK_SECRET`, `ALLOWED_CONTROL_PLANE_HOSTS`, `CONTROL_PLANE_URL` -For remote Modal callbacks, `ALLOWED_CONTROL_PLANE_HOSTS` must include the tunnel host. The -`dev:modal-secrets` script derives this from `CONTROL_PLANE_URL` unless you set -`ALLOWED_CONTROL_PLANE_HOSTS` explicitly. + +For remote Modal callbacks, `ALLOWED_CONTROL_PLANE_HOSTS` must include the tunnel host. The `dev:modal-secrets` script derives this from `CONTROL_PLANE_URL` unless you set `ALLOWED_CONTROL_PLANE_HOSTS` explicitly. Skip this step when `SANDBOX_PROVIDER=daytona`; `dev:modal-secrets` is only needed for Modal. @@ -115,8 +114,8 @@ http://localhost:3000/api/auth/callback/github ## Env File Management -Local full-stack development uses the root `.env.local` as the only editable env file. Do not edit -the generated service-specific files directly. + +Local full-stack development uses the root `.env.local` as the only editable env file. Do not edit the generated service-specific files directly. Source file: @@ -149,8 +148,8 @@ The generator also derives public provider values for the web app: - `NEXT_PUBLIC_SANDBOX_PROVIDER` is derived from `SANDBOX_PROVIDER` - `NEXT_PUBLIC_SCM_PROVIDER` is derived from `SCM_PROVIDER` -Only set the `NEXT_PUBLIC_*` provider values in `.env.local` if you intentionally need to override -the derived value. + +Only set the `NEXT_PUBLIC_*` provider values in `.env.local` if you intentionally need to override the derived value. Modal secrets are synced separately because Modal stores them in its own secret manager: @@ -158,12 +157,12 @@ Modal secrets are synced separately because Modal stores them in its own secret npm run dev:modal-secrets ``` -This command reads `.env.local`, validates Modal-specific values, and creates or updates the Modal -secrets used by the deployed Modal app. It exits without making changes when `SANDBOX_PROVIDER` is -not `modal`. + +This command reads `.env.local`, validates Modal-specific values, and creates or updates the Modal secrets used by the deployed Modal app. It exits without making changes when `SANDBOX_PROVIDER` is not `modal`. ## Notes - The control plane uses local Wrangler storage for D1, KV, R2, and Durable Objects. + - Modal remains remote, so sandboxes must be able to call back through the tunnel. If the ngrok URL changes, update `.env.local`, then rerun `npm run dev:env` and `npm run dev:modal-secrets`. From 39cb1f533cb405f4b485dc028ed01d1df284702f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CSid?= Date: Tue, 28 Apr 2026 21:58:36 -0400 Subject: [PATCH 07/10] Revert "docs: preserve local env guide prose lines" This reverts commit 55eec005678cd5bf5755d94ce02e62dfa9483ecc. --- docs/LOCAL_WEB_CONTROL_PLANE.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/docs/LOCAL_WEB_CONTROL_PLANE.md b/docs/LOCAL_WEB_CONTROL_PLANE.md index c3ebbf441..4f80739ab 100644 --- a/docs/LOCAL_WEB_CONTROL_PLANE.md +++ b/docs/LOCAL_WEB_CONTROL_PLANE.md @@ -1,7 +1,7 @@ # Local Web And Control Plane - -Run the web app and control plane on your machine while using real GitHub App credentials and real sandbox services. +Run the web app and control plane on your machine while using real GitHub App credentials and real +sandbox services. ## Services @@ -30,8 +30,8 @@ GITHUB_APP_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-- 3. Expose the local control plane to Modal. - -Modal sandboxes run remotely, so they cannot connect back to `localhost:8787` on your machine. Start a tunnel to expose the local control plane port: +Modal sandboxes run remotely, so they cannot connect back to `localhost:8787` on your machine. Start +a tunnel to expose the local control plane port: ```bash ngrok http 8787 @@ -71,8 +71,9 @@ At minimum Modal needs: - `internal-api`: `MODAL_API_SECRET`, `INTERNAL_CALLBACK_SECRET`, `ALLOWED_CONTROL_PLANE_HOSTS`, `CONTROL_PLANE_URL` - -For remote Modal callbacks, `ALLOWED_CONTROL_PLANE_HOSTS` must include the tunnel host. The `dev:modal-secrets` script derives this from `CONTROL_PLANE_URL` unless you set `ALLOWED_CONTROL_PLANE_HOSTS` explicitly. +For remote Modal callbacks, `ALLOWED_CONTROL_PLANE_HOSTS` must include the tunnel host. The +`dev:modal-secrets` script derives this from `CONTROL_PLANE_URL` unless you set +`ALLOWED_CONTROL_PLANE_HOSTS` explicitly. Skip this step when `SANDBOX_PROVIDER=daytona`; `dev:modal-secrets` is only needed for Modal. @@ -114,8 +115,8 @@ http://localhost:3000/api/auth/callback/github ## Env File Management - -Local full-stack development uses the root `.env.local` as the only editable env file. Do not edit the generated service-specific files directly. +Local full-stack development uses the root `.env.local` as the only editable env file. Do not edit +the generated service-specific files directly. Source file: @@ -148,8 +149,8 @@ The generator also derives public provider values for the web app: - `NEXT_PUBLIC_SANDBOX_PROVIDER` is derived from `SANDBOX_PROVIDER` - `NEXT_PUBLIC_SCM_PROVIDER` is derived from `SCM_PROVIDER` - -Only set the `NEXT_PUBLIC_*` provider values in `.env.local` if you intentionally need to override the derived value. +Only set the `NEXT_PUBLIC_*` provider values in `.env.local` if you intentionally need to override +the derived value. Modal secrets are synced separately because Modal stores them in its own secret manager: @@ -157,12 +158,12 @@ Modal secrets are synced separately because Modal stores them in its own secret npm run dev:modal-secrets ``` - -This command reads `.env.local`, validates Modal-specific values, and creates or updates the Modal secrets used by the deployed Modal app. It exits without making changes when `SANDBOX_PROVIDER` is not `modal`. +This command reads `.env.local`, validates Modal-specific values, and creates or updates the Modal +secrets used by the deployed Modal app. It exits without making changes when `SANDBOX_PROVIDER` is +not `modal`. ## Notes - The control plane uses local Wrangler storage for D1, KV, R2, and Durable Objects. - - Modal remains remote, so sandboxes must be able to call back through the tunnel. If the ngrok URL changes, update `.env.local`, then rerun `npm run dev:env` and `npm run dev:modal-secrets`. From 990afc58744d612b221be9efdf9fc1c7e09a5ee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CSid?= Date: Tue, 28 Apr 2026 21:58:37 -0400 Subject: [PATCH 08/10] Revert "docs: clean up local env guide wrapping" This reverts commit d274fae97c2c9a26dbcb68bb867ddbcb7c29563d. --- docs/LOCAL_WEB_CONTROL_PLANE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/LOCAL_WEB_CONTROL_PLANE.md b/docs/LOCAL_WEB_CONTROL_PLANE.md index 4f80739ab..9e65dbc24 100644 --- a/docs/LOCAL_WEB_CONTROL_PLANE.md +++ b/docs/LOCAL_WEB_CONTROL_PLANE.md @@ -31,7 +31,7 @@ GITHUB_APP_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-- 3. Expose the local control plane to Modal. Modal sandboxes run remotely, so they cannot connect back to `localhost:8787` on your machine. Start -a tunnel to expose the local control plane port: +a tunnel to the local control plane port: ```bash ngrok http 8787 From 0e7ad2db5c05a82fa4040e145347786742110335 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CSid?= Date: Tue, 28 Apr 2026 22:17:52 -0400 Subject: [PATCH 09/10] docs: restore D1 migration script comments --- scripts/d1-migrate.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/d1-migrate.sh b/scripts/d1-migrate.sh index 7396fc7f4..f37def7cf 100755 --- a/scripts/d1-migrate.sh +++ b/scripts/d1-migrate.sh @@ -32,6 +32,7 @@ case "$MODE" in ;; esac +# 1. Ensure tracking table exists. $WRANGLER d1 execute "$DATABASE_NAME" "${D1_OPTIONS[@]}" \ --command "CREATE TABLE IF NOT EXISTS _schema_migrations ( version TEXT PRIMARY KEY, @@ -39,10 +40,12 @@ $WRANGLER d1 execute "$DATABASE_NAME" "${D1_OPTIONS[@]}" \ applied_at TEXT NOT NULL DEFAULT (datetime('now')) )" +# 2. Get applied versions from the tracking table. APPLIED=$($WRANGLER d1 execute "$DATABASE_NAME" "${D1_OPTIONS[@]}" \ --command "SELECT version FROM _schema_migrations ORDER BY version" \ --json | jq -r '.[0].results[].version // empty' 2>/dev/null || true) +# 3. Apply pending migrations in order. COUNT=0 for file in "$MIGRATIONS_DIR"/*.sql; do [ -f "$file" ] || continue From fda40651c993465876ae97269bdea6fba1857cf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CSid?= Date: Tue, 28 Apr 2026 22:37:48 -0400 Subject: [PATCH 10/10] fix: harden local dev helper scripts --- scripts/d1-migrate.sh | 9 +++++++-- scripts/env-utils.mjs | 3 ++- scripts/modal-secrets.mjs | 9 ++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/d1-migrate.sh b/scripts/d1-migrate.sh index f37def7cf..7cedbd4aa 100755 --- a/scripts/d1-migrate.sh +++ b/scripts/d1-migrate.sh @@ -43,14 +43,19 @@ $WRANGLER d1 execute "$DATABASE_NAME" "${D1_OPTIONS[@]}" \ # 2. Get applied versions from the tracking table. APPLIED=$($WRANGLER d1 execute "$DATABASE_NAME" "${D1_OPTIONS[@]}" \ --command "SELECT version FROM _schema_migrations ORDER BY version" \ - --json | jq -r '.[0].results[].version // empty' 2>/dev/null || true) + --json | jq -r '.[0].results // [] | .[].version') # 3. Apply pending migrations in order. COUNT=0 for file in "$MIGRATIONS_DIR"/*.sql; do [ -f "$file" ] || continue FILENAME=$(basename "$file") - VERSION=$(printf "%s" "$FILENAME" | grep -oE '^[0-9]+') + if [[ "$FILENAME" =~ ^([0-9]+) ]]; then + VERSION="${BASH_REMATCH[1]}" + else + printf "Invalid migration filename: VERSION is empty for FILENAME=%s; expected numeric prefix.\n" "$FILENAME" >&2 + exit 1 + fi if printf "%s\n" "$APPLIED" | grep -qxF "$VERSION"; then printf "Skip (already applied): %s\n" "$FILENAME" diff --git a/scripts/env-utils.mjs b/scripts/env-utils.mjs index f406d924c..6191577e3 100644 --- a/scripts/env-utils.mjs +++ b/scripts/env-utils.mjs @@ -1,8 +1,9 @@ import dotenv from "dotenv"; import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; -export const root = resolve(new URL("..", import.meta.url).pathname); +export const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); export const sourcePath = resolve(root, ".env.local"); export const GENERATED_ENV_HEADER = "# Generated by npm run dev:env. Do not edit directly."; diff --git a/scripts/modal-secrets.mjs b/scripts/modal-secrets.mjs index a44d25549..a1a41c851 100755 --- a/scripts/modal-secrets.mjs +++ b/scripts/modal-secrets.mjs @@ -52,7 +52,14 @@ if (requiredEnvKeys.modalSecrets.length === 0) { process.exit(0); } -const controlPlaneUrl = new URL(env.CONTROL_PLANE_URL); +let controlPlaneUrl; +try { + controlPlaneUrl = new URL(env.CONTROL_PLANE_URL); +} catch { + console.error(`Invalid CONTROL_PLANE_URL in .env.local: ${env.CONTROL_PLANE_URL}`); + process.exit(1); +} + const allowedHosts = env.ALLOWED_CONTROL_PLANE_HOSTS || controlPlaneUrl.host; const modalCommand = shellWords(process.env.MODAL_CLI || "modal");