CTX7-1764: lazy authentication on /mcp

.changeset/lazy-auth-mcp-endpoint.md

@@ -0,0 +1,9 @@
+---
+"@upstash/context7-mcp": minor
+---
+
+Lazy authentication on the public `/mcp` endpoint. Anonymous clients can still connect, list tools and call public tools; the server now issues an OAuth challenge only when an unauthenticated caller invokes a protected tool or spends its anonymous allowance (`CONTEXT7_ANON_FREE_CALLS`, default 5).
+
+The challenge is delivered in whichever shape the calling client acts on: an HTTP 401 with `WWW-Authenticate` for spec-compliant clients (Claude, VS Code, Cursor, Cline, Zed), or a `CallToolResult` carrying `_meta["mcp/www_authenticate"]` for ChatGPT and Codex, which do not raise their link-account UI from a bare 401. Tools also advertise `securitySchemes` in `tools/list` so OpenAI clients know they are callable before an account is linked.
+
+This replaces the anonymous sign-in elicitation, which nudged the user with a `ctx7 setup` command instead of driving the client's own OAuth flow.

packages/mcp/scripts/docker-compose.redis.yml

@@ -0,0 +1,22 @@
+# Local Upstash-compatible Redis for testing the MCP HTTP server.
+# `@upstash/redis` speaks an HTTP REST protocol; serverless-redis-http (srh)
+# exposes a plain Redis over that protocol so getRedis() works locally.
+#
+#   docker compose -f scripts/docker-compose.redis.yml up -d
+#
+# Then point the server at it:
+#   UPSTASH_REDIS_REST_URL=http://localhost:8079
+#   UPSTASH_REDIS_REST_TOKEN=local_token
+services:
+  redis:
+    image: redis:7-alpine
+  srh:
+    image: hiett/serverless-redis-http:latest
+    environment:
+      SRH_MODE: env
+      SRH_TOKEN: local_token
+      SRH_CONNECTION_STRING: redis://redis:6379
+    ports:
+      - "8079:80"
+    depends_on:
+      - redis

packages/mcp/scripts/lazy-auth-probe.mjs

@@ -0,0 +1,151 @@
+// Drives the lazy-auth gate end-to-end over raw HTTP and prints what each
+// client family actually sees. Raw fetch rather than the MCP SDK client on
+// purpose: the status code, the WWW-Authenticate header and the `_meta`
+// challenge are the whole contract, and an SDK client hides all three.
+//
+//   # terminal 1 — small allowance so the challenge fires quickly
+//   CONTEXT7_ANON_FREE_CALLS=3 node dist/index.js --transport http --port 3000
+//
+//   # terminal 2
+//   node scripts/lazy-auth-probe.mjs
+//
+// Env: MCP_URL (default http://localhost:3000/mcp), MAX_CALLS (default 8).
+const url = process.env.MCP_URL ?? "http://localhost:3000/mcp";
+const MAX_CALLS = Number(process.env.MAX_CALLS ?? 8);
+
+// User-Agents that select each challenge shape. See challengeTransportFor().
+const CLIENTS = [
+  { label: "spec client (Claude, VS Code, Cursor, …)", ua: "claude-code/2.1.0" },
+  { label: "OpenAI client (ChatGPT, Codex)", ua: "codex_cli_rs/0.104.0" },
+];
+
+async function rpc(method, params, { ua, sessionId }) {
+  const headers = {
+    "Content-Type": "application/json",
+    Accept: "application/json, text/event-stream",
+    "User-Agent": ua,
+  };
+  if (sessionId) headers["mcp-session-id"] = sessionId;
+
+  const res = await fetch(url, {
+    method: "POST",
+    headers,
+    body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }),
+  });
+
+  const raw = await res.text();
+  // Tool calls come back as SSE so headers flush before the tool finishes.
+  const payload = raw.startsWith("event:") || raw.startsWith("data:") ? sseData(raw) : raw;
+  let body;
+  try {
+    body = JSON.parse(payload);
+  } catch {
+    body = payload;
+  }
+  return {
+    status: res.status,
+    wwwAuthenticate: res.headers.get("www-authenticate"),
+    sessionId: res.headers.get("mcp-session-id"),
+    body,
+  };
+}
+
+function sseData(raw) {
+  return raw
+    .split("\n")
+    .filter((line) => line.startsWith("data:"))
+    .map((line) => line.slice(5).trim())
+    .join("");
+}
+
+function challengeOf(res) {
+  if (res.status === 401 || res.status === 403) {
+    return { shape: `HTTP ${res.status}`, header: res.wwwAuthenticate };
+  }
+  const meta = res.body?.result?._meta?.["mcp/www_authenticate"];
+  if (Array.isArray(meta) && meta.length > 0) {
+    return { shape: "200 + _meta[mcp/www_authenticate]", header: meta[0] };
+  }
+  return null;
+}
+
+async function probe({ label, ua }) {
+  console.log(`\n=== ${label}`);
+  console.log(`    User-Agent: ${ua}`);
+
+  const init = await rpc(
+    "initialize",
+    {
+      protocolVersion: "2025-06-18",
+      capabilities: {},
+      clientInfo: { name: "lazy-auth-probe", version: "0.0.0" },
+    },
+    { ua }
+  );
+  if (init.status !== 200) {
+    console.log(`    initialize FAILED with ${init.status} — lazy auth must let this through`);
+    return false;
+  }
+  const sessionId = init.sessionId;
+  console.log(`    initialize: 200 anonymously (session ${sessionId ?? "none"})`);
+
+  const list = await rpc("tools/list", {}, { ua, sessionId });
+  const tools = list.body?.result?.tools ?? [];
+  console.log(`    tools/list: 200 anonymously, ${tools.length} tool(s)`);
+  for (const tool of tools) {
+    const schemes = (tool.securitySchemes ?? [])
+      .map((s) => (s.type === "oauth2" ? `oauth2(${s.scopes?.join(" ")})` : s.type))
+      .join(" + ");
+    console.log(`      - ${tool.name}: ${schemes || "no securitySchemes advertised!"}`);
+  }
+
+  for (let i = 1; i <= MAX_CALLS; i++) {
+    const res = await rpc(
+      "tools/call",
+      { name: "resolve-library-id", arguments: { query: "routing", libraryName: "react" } },
+      { ua, sessionId }
+    );
+    const challenge = challengeOf(res);
+    if (!challenge) {
+      console.log(`    call ${i}: allowed`);
+      continue;
+    }
+    console.log(`    call ${i}: CHALLENGED as ${challenge.shape}`);
+    console.log(`      ${challenge.header}`);
+    console.log("      A real client now runs OAuth and retries this same call.");
+    return true;
+  }
+
+  console.log(`    no challenge in ${MAX_CALLS} calls — is CONTEXT7_ANON_FREE_CALLS low enough?`);
+  return false;
+}
+
+async function checkDiscovery() {
+  const origin = new URL(url).origin;
+  const paths = [
+    "/.well-known/oauth-protected-resource",
+    "/.well-known/oauth-protected-resource/mcp",
+  ];
+  console.log("\n=== OAuth discovery documents");
+  for (const path of paths) {
+    const res = await fetch(origin + path);
+    const doc = res.ok ? await res.json() : null;
+    const servers = doc
+      ? ` -> authorization_servers=${JSON.stringify(doc.authorization_servers)}`
+      : "";
+    console.log(`    ${path}: ${res.status}${servers}`);
+  }
+}
+
+let ok = true;
+for (const client of CLIENTS) {
+  ok = (await probe(client)) && ok;
+}
+await checkDiscovery();
+
+console.log(
+  ok
+    ? "\nBoth client families were challenged in the shape they understand."
+    : "\nAt least one probe did not reach a challenge — see above."
+);
+process.exit(ok ? 0 : 1);

packages/mcp/src/index.ts

@@ -26,8 +26,16 @@ import {
   AUTH_SERVER_URL,
   OPENAI_APPS_CHALLENGE_TOKEN,
 } from "./lib/constants.js";
-import { maybeElicitAuthSignIn } from "./lib/auth/auth-prompt.js";
 import { getClientIp } from "./lib/client-ip.js";
+import {
+  resolveAuthState,
+  evaluateLazyAuth,
+  buildWwwAuthenticate,
+  buildHttpChallenge,
+  buildToolResultChallenge,
+  challengeTransportFor,
+} from "./lib/auth/lazy-auth.js";
+import { advertiseToolSecuritySchemes } from "./lib/auth/tool-security.js";
 
 /** Default HTTP server port */
 const DEFAULT_PORT = 3000;
@@ -193,7 +201,6 @@ IMPORTANT: Do not call this tool more than 3 times per question. If you cannot f
 
       if (!searchResponse.results || searchResponse.results.length === 0) {
         const text = searchResponse.error ?? "No libraries found matching the provided name.";
-        maybeElicitAuthSignIn(server, ctx);
         return {
           content: [
             {
@@ -206,7 +213,6 @@ IMPORTANT: Do not call this tool more than 3 times per question. If you cannot f
 
       const resultsText = formatSearchResults(searchResponse);
       const responseText = `Available Libraries:\n\n${resultsText}`;
-      maybeElicitAuthSignIn(server, ctx);
       return {
         content: [
           {
@@ -249,7 +255,6 @@ Do not call this tool more than 3 times per question.`,
     async ({ query, libraryId }: { query: string; libraryId: string }) => {
       const ctx = getClientContext();
       const response = await fetchLibraryContext({ query, libraryId }, ctx);
-      maybeElicitAuthSignIn(server, ctx);
       return {
         content: [
           {
@@ -261,6 +266,8 @@ Do not call this tool more than 3 times per question.`,
     }
   );
 
+  advertiseToolSecuritySchemes(server);
+
   server.server.registerCapabilities({ prompts: {}, resources: {} });
   server.server.setRequestHandler(ListPromptsRequestSchema, async () => ({ prompts: [] }));
   server.server.setRequestHandler(ListResourcesRequestSchema, async () => ({
@@ -383,7 +390,7 @@ async function main() {
     const handleMcpRequest = async (
       req: express.Request,
       res: express.Response,
-      requireAuth: boolean
+      mode: "lazy" | "required"
     ) => {
       // Reject GET requests — sessions are tracked in Redis, but this server does not send
       // server-initiated notifications, so SSE streams serve no purpose and cause mass NGINX
@@ -402,36 +409,61 @@ async function main() {
         const baseUrl = new URL(resourceUrl).origin;
 
         // OAuth discovery info header, used by MCP clients to discover the authorization server
-        res.set(
-          "WWW-Authenticate",
-          `Bearer resource_metadata="${baseUrl}/.well-known/oauth-protected-resource"`
-        );
+        res.set("WWW-Authenticate", buildWwwAuthenticate(baseUrl));
 
-        if (requireAuth) {
+        if (mode === "required") {
+          // Eager auth: challenge on every request, including initialize/tools/list.
           if (!apiKey) {
+            const message = "Authentication required. Please authenticate to use this MCP server.";
+            res.set("WWW-Authenticate", buildWwwAuthenticate(baseUrl, "invalid_token", message));
             return res.status(401).json({
               jsonrpc: "2.0",
-              error: {
-                code: -32001,
-                message: "Authentication required. Please authenticate to use this MCP server.",
-              },
+              error: { code: -32001, message },
               id: null,
             });
           }
 
           if (isJWT(apiKey)) {
             const validationResult = await validateJWT(apiKey);
             if (!validationResult.valid) {
+              const message = validationResult.error || "Invalid token. Please re-authenticate.";
+              res.set("WWW-Authenticate", buildWwwAuthenticate(baseUrl, "invalid_token", message));
               return res.status(401).json({
                 jsonrpc: "2.0",
-                error: {
-                  code: -32001,
-                  message: validationResult.error || "Invalid token. Please re-authenticate.",
-                },
+                error: { code: -32001, message },
                 id: null,
               });
             }
           }
+        } else {
+          // Lazy auth: connect/list anonymously; challenge only when an
+          // unauthenticated caller hits a protected tool or exhausts its
+          // anonymous quota. Decided here, before the transport streams a 200.
+          const auth = await resolveAuthState(apiKey);
+          const { challenge } = await evaluateLazyAuth({
+            body: req.body,
+            auth,
+            clientIp: getClientIp(req),
+            sessionId: extractHeaderValue(req.headers["mcp-session-id"]),
+          });
+          if (challenge) {
+            res.set(
+              "WWW-Authenticate",
+              buildWwwAuthenticate(baseUrl, challenge.error, challenge.message)
+            );
+            // ChatGPT and Codex only raise their link-account UI for a
+            // challenge carried in the tool result; everyone else needs the
+            // transport-level 401. Same challenge string either way.
+            // A batch would need one result per message to stay well-formed;
+            // the 401 refuses the whole request, which is correct either way.
+            const challengeTransport = Array.isArray(req.body)
+              ? "http-401"
+              : challengeTransportFor(extractHeaderValue(req.headers["user-agent"]));
+            if (challengeTransport === "tool-result") {
+              return res.status(200).json(buildToolResultChallenge(challenge, baseUrl));
+            }
+            return res.status(challenge.status).json(buildHttpChallenge(challenge));
+          }
         }
 
         const context: ClientContext = {
@@ -522,14 +554,15 @@ async function main() {
       }
     };
 
-    // Anonymous access endpoint - no authentication required
+    // Lazy-auth endpoint: anonymous connect/list and public tool calls, with a
+    // 401 challenge only for protected tools or once the anonymous quota is spent.
     app.all("/mcp", async (req, res) => {
-      await handleMcpRequest(req, res, false);
+      await handleMcpRequest(req, res, "lazy");
     });
 
-    // OAuth-protected endpoint - requires authentication
+    // OAuth-protected endpoint - requires authentication on every request
     app.all("/mcp/oauth", async (req, res) => {
-      await handleMcpRequest(req, res, true);
+      await handleMcpRequest(req, res, "required");
     });
 
     app.get("/ping", (_req: express.Request, res: express.Response) => {
@@ -538,17 +571,23 @@ async function main() {
 
     // OAuth 2.0 Protected Resource Metadata (RFC 9728)
     // Used by MCP clients to discover the authorization server
-    app.get(
-      "/.well-known/oauth-protected-resource",
-      (_req: express.Request, res: express.Response) => {
-        res.json({
-          resource: RESOURCE_URL,
-          authorization_servers: [AUTH_SERVER_URL],
-          scopes_supported: ["profile", "email"],
-          bearer_methods_supported: ["header"],
-        });
-      }
-    );
+    const protectedResourceMetadata = (_req: express.Request, res: express.Response) => {
+      res.json({
+        resource: RESOURCE_URL,
+        authorization_servers: [AUTH_SERVER_URL],
+        scopes_supported: ["profile", "email"],
+        bearer_methods_supported: ["header"],
+        resource_documentation: `${AUTH_SERVER_URL}/docs/howto/oauth`,
+      });
+    };
+
+    app.get("/.well-known/oauth-protected-resource", protectedResourceMetadata);
+
+    // Path-suffixed variants (RFC 9728 section 3.1). Clients connecting to
+    // /mcp or /mcp/oauth try these before falling back to the root document,
+    // so serving them avoids a 404 round-trip on every OAuth discovery.
+    app.get("/.well-known/oauth-protected-resource/mcp", protectedResourceMetadata);
+    app.get("/.well-known/oauth-protected-resource/mcp/oauth", protectedResourceMetadata);
 
     app.get(
       "/.well-known/oauth-authorization-server",