diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..b669c2245 --- /dev/null +++ b/.env.example @@ -0,0 +1,58 @@ +# Local full-stack development against real GitHub + sandbox services. +# Copy to .env.local, fill values, then run `npm run dev:env`. + +# GitHub App OAuth +GITHUB_CLIENT_ID= +GITHUB_CLIENT_SECRET= + +# GitHub App installation access +GITHUB_APP_ID= +# Use escaped newlines for this single-line env file, for example: +# GITHUB_APP_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----" +GITHUB_APP_PRIVATE_KEY= +GITHUB_APP_INSTALLATION_ID= + +# Sandbox provider. Supported values: modal, daytona. +SANDBOX_PROVIDER=modal + +# Modal sandbox provider. Required when SANDBOX_PROVIDER=modal. +MODAL_TOKEN_ID= +MODAL_TOKEN_SECRET= +MODAL_WORKSPACE= +MODAL_API_SECRET= + +# Daytona sandbox provider. Required when SANDBOX_PROVIDER=daytona. +DAYTONA_API_KEY= +DAYTONA_API_URL= +DAYTONA_BASE_SNAPSHOT= + +# LLM keys injected by Modal +ANTHROPIC_API_KEY= + +# Shared app secrets +TOKEN_ENCRYPTION_KEY= +REPO_SECRETS_ENCRYPTION_KEY= +INTERNAL_CALLBACK_SECRET= +NEXTAUTH_SECRET= + +# Local service URLs +NEXTAUTH_URL=http://localhost:3000 +# Modal sandboxes run remotely and cannot call back to localhost on your machine. +# For Path C, expose the local control plane with a tunnel (for example `ngrok http 8787`) and set: +# CONTROL_PLANE_URL=https:// +# NEXT_PUBLIC_WS_URL=wss:// +# WORKER_URL=https:// +CONTROL_PLANE_URL=http://localhost:8787 +NEXT_PUBLIC_WS_URL=ws://localhost:8787 +WEB_APP_URL=http://localhost:3000 +WORKER_URL=http://localhost:8787 +DEPLOYMENT_NAME=local + +# Access control. Prefer ALLOWED_USERS for local real-service runs. +ALLOWED_USERS= +ALLOWED_EMAIL_DOMAINS= +UNSAFE_ALLOW_ALL_USERS=false + +# Optional +SCM_PROVIDER=github +LOG_LEVEL=debug diff --git a/docs/LOCAL_WEB_CONTROL_PLANE.md b/docs/LOCAL_WEB_CONTROL_PLANE.md new file mode 100644 index 000000000..9e65dbc24 --- /dev/null +++ b/docs/LOCAL_WEB_CONTROL_PLANE.md @@ -0,0 +1,169 @@ +# Local Web And Control Plane + +Run the web app and control plane on your machine while using real GitHub App credentials and real +sandbox services. + +## Services + +- Web: `http://localhost:3000` +- Control plane: `http://localhost:8787` +- Modal or Daytona: deployed sandbox backend +- GitHub: real GitHub App OAuth and installation APIs + +## Setup + +Complete [Step 0 in SETUP_GUIDE.md](./SETUP_GUIDE.md#step-0-bootstrap-the-repo) first. + +1. Copy the root env template: + +```bash +cp .env.example .env.local +``` + +2. Fill in `.env.local` with the real GitHub App, sandbox provider, and app secrets. + +Use escaped newlines for `GITHUB_APP_PRIVATE_KEY` because `.env.local` is a single-line env file: + +```bash +GITHUB_APP_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----" +``` + +3. Expose the local control plane to Modal. + +Modal sandboxes run remotely, so they cannot connect back to `localhost:8787` on your machine. Start +a tunnel to the local control plane port: + +```bash +ngrok http 8787 +``` + +Update `.env.local` with the tunnel URL: + +```bash +CONTROL_PLANE_URL=https:// +NEXT_PUBLIC_WS_URL=wss:// +WORKER_URL=https:// +``` + +Keep `NEXTAUTH_URL=http://localhost:3000`; the web app still runs locally. + +4. Generate package env files: + +```bash +npm run dev:env +``` + +This writes: + +- `packages/web/.env.local` +- `packages/control-plane/.dev.vars` + +5. Configure Modal secrets from `.env.local`: + +```bash +npm run dev:modal-secrets +``` + +At minimum Modal needs: + +- `llm-api-keys`: `ANTHROPIC_API_KEY` +- `github-app`: `GITHUB_APP_ID`, `GITHUB_APP_PRIVATE_KEY`, `GITHUB_APP_INSTALLATION_ID` +- `internal-api`: `MODAL_API_SECRET`, `INTERNAL_CALLBACK_SECRET`, `ALLOWED_CONTROL_PLANE_HOSTS`, + `CONTROL_PLANE_URL` + +For remote Modal callbacks, `ALLOWED_CONTROL_PLANE_HOSTS` must include the tunnel host. The +`dev:modal-secrets` script derives this from `CONTROL_PLANE_URL` unless you set +`ALLOWED_CONTROL_PLANE_HOSTS` explicitly. + +Skip this step when `SANDBOX_PROVIDER=daytona`; `dev:modal-secrets` is only needed for Modal. + +6. Deploy Modal: + +```bash +cd packages/modal-infra +uv sync --frozen --extra dev +uv run modal deploy deploy.py +``` + +7. Apply local D1 migrations: + +```bash +npm run dev:db:local +``` + +8. Run the control plane: + +```bash +npm run dev:control-plane +``` + +9. In another terminal, run the web app: + +```bash +npm run dev:web +``` + +10. Open `http://localhost:3000`. + +## GitHub App Callback + +Your real GitHub App must include this callback URL for local web auth: + +```text +http://localhost:3000/api/auth/callback/github +``` + +## Env File Management + +Local full-stack development uses the root `.env.local` as the only editable env file. Do not edit +the generated service-specific files directly. + +Source file: + +- `.env.local` — copied from `.env.example` and edited by you + +Generated files: + +- `packages/web/.env.local` — consumed by Next.js +- `packages/control-plane/.dev.vars` — consumed by Wrangler + +Run this after changing `.env.local`: + +```bash +npm run dev:env +``` + +`dev:env` validates required values, writes the generated files, and stamps them with: + +```text +# Generated by npm run dev:env. Do not edit directly. +``` + +Validation is provider-aware: + +- `SANDBOX_PROVIDER=modal` requires Modal credentials and Modal callback secrets +- `SANDBOX_PROVIDER=daytona` requires Daytona credentials instead and does not require Modal tokens + +The generator also derives public provider values for the web app: + +- `NEXT_PUBLIC_SANDBOX_PROVIDER` is derived from `SANDBOX_PROVIDER` +- `NEXT_PUBLIC_SCM_PROVIDER` is derived from `SCM_PROVIDER` + +Only set the `NEXT_PUBLIC_*` provider values in `.env.local` if you intentionally need to override +the derived value. + +Modal secrets are synced separately because Modal stores them in its own secret manager: + +```bash +npm run dev:modal-secrets +``` + +This command reads `.env.local`, validates Modal-specific values, and creates or updates the Modal +secrets used by the deployed Modal app. It exits without making changes when `SANDBOX_PROVIDER` is +not `modal`. + +## Notes + +- The control plane uses local Wrangler storage for D1, KV, R2, and Durable Objects. +- Modal remains remote, so sandboxes must be able to call back through the tunnel. If the ngrok URL + changes, update `.env.local`, then rerun `npm run dev:env` and `npm run dev:modal-secrets`. diff --git a/docs/SETUP_GUIDE.md b/docs/SETUP_GUIDE.md index 6d6188ae6..6014b7e64 100644 --- a/docs/SETUP_GUIDE.md +++ b/docs/SETUP_GUIDE.md @@ -4,11 +4,12 @@ This is the primary setup guide for users and contributors. It is organized by goal so you can pick the fastest path: -| Path | Best For | Time | -| ------ | -------------------------------------------------------- | ---------- | -| Path A | Run the web app locally against an existing backend | ~10-20 min | -| Path B | Contribute code locally (lint/typecheck/tests) | ~15-30 min | -| Path C | Deploy your own full stack (Cloudflare + Modal + Vercel) | ~1-3 hours | +| Path | Best For | Time | +| ------ | --------------------------------------------------------- | ---------- | +| Path A | Run the web app locally against an existing backend | ~10-20 min | +| Path B | Contribute code locally (lint/typecheck/tests) | ~15-30 min | +| Path C | Run local web + control plane against real GitHub + Modal | ~30-60 min | +| Path D | Deploy your own full stack (Cloudflare + Modal + Vercel) | ~1-3 hours | ## Important Context @@ -30,6 +31,12 @@ Optional (needed for `modal-infra` development): - `uv` (recommended) or `pip` - Modal CLI (`modal`) +Optional (needed for Path C local web + control plane): + +- Wrangler CLI (`npx wrangler` from repo dependencies is fine) +- `jq` for local D1 migration scripts +- A tunnel tool such as `ngrok` so remote Modal sandboxes can call back to the local control plane + Optional (needed for full deployment): - Terraform `1.6+` @@ -41,6 +48,12 @@ Quick check: node -v npm -v git --version +python --version +uv --version +modal --version +npx wrangler --version +jq --version +ngrok --version ``` ## Step 0: Bootstrap the Repo @@ -183,7 +196,22 @@ pip install -e ".[dev]" pytest tests/ -v ``` -## Path C: Full Self-Hosted Deployment +## Path C: Run Local Web And Control Plane + +Use this when developing or debugging behavior that requires the web app and control plane to run +together locally, while still using real GitHub App credentials and real Modal sandboxes. + +This path is most useful for control-plane and session-flow changes that need real end-to-end +behavior: auth, session creation, Durable Objects, WebSockets, Modal sandbox creation, bridge +callbacks, and streamed events. + +The detailed guide is here [LOCAL_WEB_CONTROL_PLANE.md](./LOCAL_WEB_CONTROL_PLANE.md) + +Because Modal sandboxes run remotely, this path requires a tunnel from Modal back to the local +control plane. The detailed guide covers the required `CONTROL_PLANE_URL`, `NEXT_PUBLIC_WS_URL`, and +`WORKER_URL` settings. + +## Path D: Full Self-Hosted Deployment For full infrastructure setup, use: @@ -223,5 +251,6 @@ Control plane cannot reach Modal (or Modal is not properly configured/deployed). - Architecture and internals: [docs/HOW_IT_WORKS.md](./HOW_IT_WORKS.md) - Full production deployment: [docs/GETTING_STARTED.md](./GETTING_STARTED.md) - Debugging and observability: [docs/DEBUGGING_PLAYBOOK.md](./DEBUGGING_PLAYBOOK.md) +- Local web and control plane: [docs/LOCAL_WEB_CONTROL_PLANE.md](./LOCAL_WEB_CONTROL_PLANE.md) - OpenAI model setup: [docs/OPENAI_MODELS.md](./OPENAI_MODELS.md) - Contribution workflow: [CONTRIBUTING.md](../CONTRIBUTING.md) diff --git a/eslint.config.js b/eslint.config.js index 964decef8..db15d5a49 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -108,6 +108,20 @@ export default tseslint.config( }, }, + // Root maintenance scripts run in Node.js. + { + files: ["scripts/**/*.{js,mjs,cjs,ts}"], + languageOptions: { + globals: { + ...globals.node, + ...globals.es2022, + }, + }, + rules: { + "no-console": "off", + }, + }, + // Test files configuration { files: ["**/*.test.{ts,tsx}", "**/*.spec.{ts,tsx}"], diff --git a/package-lock.json b/package-lock.json index 0e942d5b5..8ea2c576b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "devDependencies": { "@eslint/js": "^9.18.0", "@types/node": "^22.10.5", + "dotenv": "^16.6.1", "eslint": "^9.18.0", "eslint-config-prettier": "^10.0.1", "eslint-plugin-react": "^7.37.4", @@ -2679,7 +2680,6 @@ "version": "1.8.1", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", "integrity": "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -3410,7 +3410,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3433,7 +3432,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3456,7 +3454,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3473,7 +3470,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3490,7 +3486,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3507,7 +3502,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3524,7 +3518,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3541,7 +3534,6 @@ "cpu": [ "riscv64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3558,7 +3550,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3575,7 +3566,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3592,7 +3582,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3609,7 +3598,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3626,7 +3614,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3649,7 +3636,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3672,7 +3658,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3695,7 +3680,6 @@ "cpu": [ "riscv64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3718,7 +3702,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3741,7 +3724,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3764,7 +3746,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3787,7 +3768,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -3810,7 +3790,6 @@ "cpu": [ "wasm32" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", "optional": true, "dependencies": { @@ -3830,7 +3809,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -3850,7 +3828,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -3870,7 +3847,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ diff --git a/package.json b/package.json index ca360b791..d2cf734b5 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,11 @@ "packages/*" ], "scripts": { + "dev:env": "node scripts/local-env.mjs", + "dev:modal-secrets": "node scripts/modal-secrets.mjs", + "dev:db:local": "bash scripts/d1-migrate.sh local", + "dev:control-plane": "npm run dev -w @open-inspect/control-plane", + "dev:web": "npm run dev -w @open-inspect/web", "lint": "eslint .", "lint:fix": "eslint . --fix", "format": "prettier --write .", @@ -21,6 +26,7 @@ "devDependencies": { "@eslint/js": "^9.18.0", "@types/node": "^22.10.5", + "dotenv": "^16.6.1", "eslint": "^9.18.0", "eslint-config-prettier": "^10.0.1", "eslint-plugin-react": "^7.37.4", diff --git a/packages/control-plane/package.json b/packages/control-plane/package.json index 4e7f6f019..81cf9725f 100644 --- a/packages/control-plane/package.json +++ b/packages/control-plane/package.json @@ -4,6 +4,7 @@ "private": true, "type": "module", "scripts": { + "dev": "wrangler dev --port 8787", "build": "esbuild src/index.ts --bundle --format=esm --outfile=dist/index.js --platform=browser --target=es2022 --external:cloudflare:* --external:node:*", "test": "vitest run", "test:coverage": "vitest run --coverage", diff --git a/scripts/d1-migrate.sh b/scripts/d1-migrate.sh index 41d8cd498..7cedbd4aa 100755 --- a/scripts/d1-migrate.sh +++ b/scripts/d1-migrate.sh @@ -1,45 +1,75 @@ #!/usr/bin/env bash set -euo pipefail -DATABASE_NAME="${1:?Usage: d1-migrate.sh [migrations-dir]}" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -MIGRATIONS_DIR="${2:-$SCRIPT_DIR/../terraform/d1/migrations}" +MODE="${1:-}" +MIGRATIONS_DIR="${3:-$SCRIPT_DIR/../terraform/d1/migrations}" WRANGLER="npx wrangler" -# 1. Ensure tracking table exists -$WRANGLER d1 execute "$DATABASE_NAME" --remote \ +usage() { + printf "Usage:\n" + printf " d1-migrate.sh local [database-name] [migrations-dir]\n" + printf " d1-migrate.sh remote [migrations-dir]\n" +} + +case "$MODE" in + local) + DATABASE_NAME="${2:-open-inspect-test}" + D1_OPTIONS=(--local --config "$SCRIPT_DIR/../packages/control-plane/wrangler.jsonc") + ;; + remote) + if [ -z "${2:-}" ]; then + usage + exit 1 + fi + DATABASE_NAME="$2" + D1_OPTIONS=(--remote) + ;; + *) + usage + exit 1 + ;; +esac + +# 1. Ensure tracking table exists. +$WRANGLER d1 execute "$DATABASE_NAME" "${D1_OPTIONS[@]}" \ --command "CREATE TABLE IF NOT EXISTS _schema_migrations ( version TEXT PRIMARY KEY, name TEXT NOT NULL, applied_at TEXT NOT NULL DEFAULT (datetime('now')) )" -# 2. Get applied versions (parse JSON output) -APPLIED=$($WRANGLER d1 execute "$DATABASE_NAME" --remote \ +# 2. Get applied versions from the tracking table. +APPLIED=$($WRANGLER d1 execute "$DATABASE_NAME" "${D1_OPTIONS[@]}" \ --command "SELECT version FROM _schema_migrations ORDER BY version" \ - --json | jq -r '.[0].results[].version // empty' 2>/dev/null || echo "") + --json | jq -r '.[0].results // [] | .[].version') -# 3. Apply pending migrations in order +# 3. Apply pending migrations in order. COUNT=0 for file in "$MIGRATIONS_DIR"/*.sql; do [ -f "$file" ] || continue FILENAME=$(basename "$file") - VERSION=$(echo "$FILENAME" | grep -oE '^[0-9]+') + if [[ "$FILENAME" =~ ^([0-9]+) ]]; then + VERSION="${BASH_REMATCH[1]}" + else + printf "Invalid migration filename: VERSION is empty for FILENAME=%s; expected numeric prefix.\n" "$FILENAME" >&2 + exit 1 + fi - if echo "$APPLIED" | grep -qxF "$VERSION"; then - echo "Skip (already applied): $FILENAME" + if printf "%s\n" "$APPLIED" | grep -qxF "$VERSION"; then + printf "Skip (already applied): %s\n" "$FILENAME" continue fi - echo "Applying: $FILENAME" - $WRANGLER d1 execute "$DATABASE_NAME" --remote --file "$file" + printf "Applying: %s\n" "$FILENAME" + $WRANGLER d1 execute "$DATABASE_NAME" "${D1_OPTIONS[@]}" --file "$file" - SAFE_FILENAME=$(echo "$FILENAME" | sed "s/'/''/g") - $WRANGLER d1 execute "$DATABASE_NAME" --remote \ + SAFE_FILENAME=$(printf "%s" "$FILENAME" | sed "s/'/''/g") + $WRANGLER d1 execute "$DATABASE_NAME" "${D1_OPTIONS[@]}" \ --command "INSERT INTO _schema_migrations (version, name) VALUES ('$VERSION', '$SAFE_FILENAME')" COUNT=$((COUNT + 1)) done -echo "Done. Applied $COUNT migration(s)." +printf "Done. Applied %d migration(s).\n" "$COUNT" diff --git a/scripts/env-utils.mjs b/scripts/env-utils.mjs new file mode 100644 index 000000000..6191577e3 --- /dev/null +++ b/scripts/env-utils.mjs @@ -0,0 +1,169 @@ +import dotenv from "dotenv"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +export const sourcePath = resolve(root, ".env.local"); + +export const GENERATED_ENV_HEADER = "# Generated by npm run dev:env. Do not edit directly."; + +export const serviceEnvKeys = { + web: [ + "GITHUB_CLIENT_ID", + "GITHUB_CLIENT_SECRET", + "NEXTAUTH_URL", + "NEXTAUTH_SECRET", + "CONTROL_PLANE_URL", + "NEXT_PUBLIC_WS_URL", + "NEXT_PUBLIC_SANDBOX_PROVIDER", + "NEXT_PUBLIC_SCM_PROVIDER", + "INTERNAL_CALLBACK_SECRET", + "ALLOWED_USERS", + "ALLOWED_EMAIL_DOMAINS", + "UNSAFE_ALLOW_ALL_USERS", + ], + controlPlane: [ + "GITHUB_CLIENT_ID", + "GITHUB_CLIENT_SECRET", + "TOKEN_ENCRYPTION_KEY", + "REPO_SECRETS_ENCRYPTION_KEY", + "INTERNAL_CALLBACK_SECRET", + "GITHUB_APP_ID", + "GITHUB_APP_PRIVATE_KEY", + "GITHUB_APP_INSTALLATION_ID", + "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET", + "MODAL_API_SECRET", + "MODAL_WORKSPACE", + "DAYTONA_API_KEY", + "DAYTONA_API_URL", + "DAYTONA_BASE_SNAPSHOT", + "DEPLOYMENT_NAME", + "SCM_PROVIDER", + "SANDBOX_PROVIDER", + "WORKER_URL", + "WEB_APP_URL", + "LOG_LEVEL", + ], + modalSecrets: [ + "ANTHROPIC_API_KEY", + "GITHUB_APP_ID", + "GITHUB_APP_PRIVATE_KEY", + "GITHUB_APP_INSTALLATION_ID", + "MODAL_API_SECRET", + "INTERNAL_CALLBACK_SECRET", + "CONTROL_PLANE_URL", + ], +}; + +export const requiredEnvKeys = { + web: [ + "GITHUB_CLIENT_ID", + "GITHUB_CLIENT_SECRET", + "NEXTAUTH_URL", + "NEXTAUTH_SECRET", + "CONTROL_PLANE_URL", + "NEXT_PUBLIC_WS_URL", + "INTERNAL_CALLBACK_SECRET", + ], + controlPlane: [ + "GITHUB_CLIENT_ID", + "GITHUB_CLIENT_SECRET", + "TOKEN_ENCRYPTION_KEY", + "REPO_SECRETS_ENCRYPTION_KEY", + "INTERNAL_CALLBACK_SECRET", + "GITHUB_APP_ID", + "GITHUB_APP_PRIVATE_KEY", + "GITHUB_APP_INSTALLATION_ID", + "DEPLOYMENT_NAME", + "WORKER_URL", + "WEB_APP_URL", + ], +}; + +const modalControlPlaneKeys = [ + "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET", + "MODAL_API_SECRET", + "MODAL_WORKSPACE", +]; + +const daytonaControlPlaneKeys = ["DAYTONA_API_KEY", "DAYTONA_API_URL", "DAYTONA_BASE_SNAPSHOT"]; + +export function getRequiredEnvKeys(env) { + const sandboxProvider = normalizeProvider(env.SANDBOX_PROVIDER, "modal"); + + if (sandboxProvider === "modal") { + return { + ...requiredEnvKeys, + controlPlane: [...requiredEnvKeys.controlPlane, ...modalControlPlaneKeys], + modalSecrets: serviceEnvKeys.modalSecrets, + }; + } + + if (sandboxProvider === "daytona") { + return { + ...requiredEnvKeys, + controlPlane: [...requiredEnvKeys.controlPlane, ...daytonaControlPlaneKeys], + modalSecrets: [], + }; + } + + console.error(`Unsupported SANDBOX_PROVIDER in .env.local: ${env.SANDBOX_PROVIDER}`); + process.exit(1); +} + +function normalizeProvider(value, defaultValue) { + return value?.trim().toLowerCase() || defaultValue; +} + +export function readRootEnv() { + let source; + try { + source = readFileSync(sourcePath, "utf8"); + } catch { + console.error("Missing .env.local. Copy .env.example to .env.local and fill in real values."); + process.exit(1); + } + + return withDerivedEnv(normalizeEnv(dotenv.parse(source))); +} + +export function normalizeEnv(env) { + return Object.fromEntries( + Object.entries(env).map(([key, value]) => [key, value.replace(/\\n/g, "\n")]) + ); +} + +export function withDerivedEnv(env) { + return { + ...env, + NEXT_PUBLIC_SANDBOX_PROVIDER: env.NEXT_PUBLIC_SANDBOX_PROVIDER || env.SANDBOX_PROVIDER, + NEXT_PUBLIC_SCM_PROVIDER: env.NEXT_PUBLIC_SCM_PROVIDER || env.SCM_PROVIDER, + }; +} + +export function requireEnv(env, keys, context = ".env.local") { + const missing = [...new Set(keys)].filter((key) => !env[key]); + if (missing.length > 0) { + console.error(`Missing required values in ${context}: ${missing.join(", ")}`); + process.exit(1); + } +} + +export function serializeEnv(env, keys) { + return `${GENERATED_ENV_HEADER}\n${keys + .map((key) => { + const value = env[key] ?? ""; + const serialized = value.includes("\n") ? JSON.stringify(value) : value; + return `${key}=${serialized}`; + }) + .join("\n")}\n`; +} + +export function writeEnv(path, contents) { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, contents); + console.log(`wrote ${path.replace(`${root}/`, "")}`); +} diff --git a/scripts/local-env.mjs b/scripts/local-env.mjs new file mode 100755 index 000000000..c13fe0e22 --- /dev/null +++ b/scripts/local-env.mjs @@ -0,0 +1,24 @@ +#!/usr/bin/env node +import { resolve } from "node:path"; +import { + getRequiredEnvKeys, + readRootEnv, + requireEnv, + root, + serializeEnv, + serviceEnvKeys, + writeEnv, +} from "./env-utils.mjs"; + +const env = readRootEnv(); +const requiredEnvKeys = getRequiredEnvKeys(env); +requireEnv(env, [...requiredEnvKeys.web, ...requiredEnvKeys.controlPlane]); + +writeEnv(resolve(root, "packages/web/.env.local"), serializeEnv(env, serviceEnvKeys.web)); +writeEnv( + resolve(root, "packages/control-plane/.dev.vars"), + serializeEnv(env, serviceEnvKeys.controlPlane) +); + +console.log("\nNext, sync Modal secrets from .env.local:"); +console.log(" npm run dev:modal-secrets"); diff --git a/scripts/modal-secrets.mjs b/scripts/modal-secrets.mjs new file mode 100755 index 000000000..a1a41c851 --- /dev/null +++ b/scripts/modal-secrets.mjs @@ -0,0 +1,83 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { getRequiredEnvKeys, readRootEnv, requireEnv, root } from "./env-utils.mjs"; + +function shellWords(command) { + return ( + command.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g)?.map((part) => { + if ( + (part.startsWith('"') && part.endsWith('"')) || + (part.startsWith("'") && part.endsWith("'")) + ) { + return part.slice(1, -1); + } + return part; + }) ?? [command] + ); +} + +function createSecret(modalCommand, secretName, values) { + const [command, ...baseArgs] = modalCommand; + const args = [ + ...baseArgs, + "secret", + "create", + secretName, + ...Object.entries(values).map(([key, value]) => `${key}=${value}`), + "--force", + ]; + + console.log(`creating/updating Modal secret: ${secretName}`); + const result = spawnSync(command, args, { + stdio: "inherit", + cwd: root, + }); + + if (result.error) { + console.error(result.error.message); + process.exit(1); + } + + if (result.status !== 0) { + process.exit(result.status ?? 1); + } +} + +const env = readRootEnv(); +const requiredEnvKeys = getRequiredEnvKeys(env); +requireEnv(env, requiredEnvKeys.modalSecrets); + +if (requiredEnvKeys.modalSecrets.length === 0) { + console.log("Modal secrets are not required for SANDBOX_PROVIDER=daytona."); + process.exit(0); +} + +let controlPlaneUrl; +try { + controlPlaneUrl = new URL(env.CONTROL_PLANE_URL); +} catch { + console.error(`Invalid CONTROL_PLANE_URL in .env.local: ${env.CONTROL_PLANE_URL}`); + process.exit(1); +} + +const allowedHosts = env.ALLOWED_CONTROL_PLANE_HOSTS || controlPlaneUrl.host; +const modalCommand = shellWords(process.env.MODAL_CLI || "modal"); + +createSecret(modalCommand, "llm-api-keys", { + ANTHROPIC_API_KEY: env.ANTHROPIC_API_KEY, +}); + +createSecret(modalCommand, "github-app", { + GITHUB_APP_ID: env.GITHUB_APP_ID, + GITHUB_APP_PRIVATE_KEY: env.GITHUB_APP_PRIVATE_KEY, + GITHUB_APP_INSTALLATION_ID: env.GITHUB_APP_INSTALLATION_ID, +}); + +createSecret(modalCommand, "internal-api", { + MODAL_API_SECRET: env.MODAL_API_SECRET, + INTERNAL_CALLBACK_SECRET: env.INTERNAL_CALLBACK_SECRET, + ALLOWED_CONTROL_PLANE_HOSTS: allowedHosts, + CONTROL_PLANE_URL: env.CONTROL_PLANE_URL, +}); + +console.log("Modal secrets are up to date."); diff --git a/terraform/environments/production/d1.tf b/terraform/environments/production/d1.tf index ea8f2c508..0e8e0e260 100644 --- a/terraform/environments/production/d1.tf +++ b/terraform/environments/production/d1.tf @@ -23,7 +23,7 @@ resource "null_resource" "d1_migrations" { } provisioner "local-exec" { - command = "bash scripts/d1-migrate.sh ${cloudflare_d1_database.main.name} terraform/d1/migrations" + command = "bash scripts/d1-migrate.sh remote ${cloudflare_d1_database.main.name} terraform/d1/migrations" working_dir = var.project_root environment = {