feat: Multi-Step README Generation Pipeline - Fixes Token Limit Issues (#101)

package.json

@@ -14,7 +14,8 @@
     "@radix-ui/react-slot": "^1.2.4",
     "@vercel/analytics": "^2.0.1",
     "lucide-react": "^0.577.0",
-    "next": "16.2.3",
+    "next": "^16.2.4",
+    "next-auth": "^4.24.14",
     "octokit": "^5.0.5",
     "react": "19.2.4",
     "react-dom": "19.2.4",

public/robots.txt

@@ -0,0 +1,9 @@
+# *
+User-agent: *
+Allow: /
+
+# Host
+Host: https://readmegen-ai.vercel.app
+
+# Sitemaps
+Sitemap: https://readmegen-ai.vercel.app/sitemap.xml

public/sitemap-0.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:mobile="http://www.google.com/schemas/sitemap-mobile/1.0" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1">
+<url><loc>https://readmegen-ai.vercel.app</loc><lastmod>2026-04-04T09:31:17.224Z</lastmod><changefreq>weekly</changefreq><priority>1</priority></url>
+<url><loc>https://readmegen-ai.vercel.app/docs</loc><lastmod>2026-04-04T09:31:17.224Z</lastmod><changefreq>monthly</changefreq><priority>0.8</priority></url>
+<url><loc>https://readmegen-ai.vercel.app/examples</loc><lastmod>2026-04-04T09:31:17.224Z</lastmod><changefreq>monthly</changefreq><priority>0.8</priority></url>
+<url><loc>https://readmegen-ai.vercel.app/features</loc><lastmod>2026-04-04T09:31:17.224Z</lastmod><changefreq>monthly</changefreq><priority>0.8</priority></url>
+<url><loc>https://readmegen-ai.vercel.app/generate</loc><lastmod>2026-04-04T09:31:17.224Z</lastmod><changefreq>weekly</changefreq><priority>0.9</priority></url>
+</urlset>

public/sitemap.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+<sitemap><loc>https://readmegen-ai.vercel.app/sitemap-0.xml</loc></sitemap>
+</sitemapindex>

src/app/api/auth/[...nextauth]/route.ts

@@ -0,0 +1,6 @@
+import NextAuth from "next-auth";
+import { authOptions } from "@/lib/auth";
+
+const handler = NextAuth(authOptions);
+
+export { handler as GET, handler as POST };

src/app/api/generate/route.ts

@@ -1,40 +1,41 @@
-import { NextResponse } from "next/server";
+import { getToken } from "next-auth/jwt";
+import { NextRequest, NextResponse } from "next/server";
 import { getGeminiModel } from "@/lib/gemini";
-import { getRepoData, getRepoContents } from "@/lib/octokit";
-import { SUPPORTED_LANGUAGES } from "@/constants/languages";
+import { getRepoSnapshot, RepoAccessError } from "@/lib/octokit";
 
 export const dynamic = "force-dynamic";
 
 /**
- * AI README Generation Endpoint
- * Optimized for data accuracy, clean prompt interpolation, and multi-language support.
+ * Enhanced Multi-Step README Generation Endpoint
  *
- * @param {Request} req - The incoming Next.js/standard Web API Request object containing the repo URL and optional language.
+ * @param {NextRequest} req - The incoming Next.js request object containing the repo URL and optional language.
  * @returns {Promise<NextResponse>} A JSON response containing the generated Markdown or an error message.
  */
-export async function POST(req: Request) {
+export async function POST(req: NextRequest) {
   let rawUrl: string;
   let language: string;
+  let ackPrivateRepo = false;
   try {
     const body = await req.json();
     rawUrl = body.url;
     language = body.language || "English";
+    ackPrivateRepo = Boolean(body.ackPrivateRepo);
   } catch {
     return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 });
   }
 
-  try {
-    const trimmedUrl = rawUrl?.trim();
-    if (!trimmedUrl) {
+    // Validate required fields
+    if (!githubUrl) {
       return NextResponse.json(
         { error: "GitHub URL is required" },
         { status: 400 },
       );
     }
 
+    // Validate GitHub URL format
     let parsedUrl: URL;
     try {
-      parsedUrl = new URL(trimmedUrl);
+      parsedUrl = new URL(githubUrl.trim());
     } catch {
       return NextResponse.json(
         { error: "Please provide a valid URL" },
@@ -63,13 +64,34 @@ export async function POST(req: Request) {
       );
     }
 
-    const [repoInfo, repoContents] = await Promise.all([
-      getRepoData(owner, repo),
-      getRepoContents(owner, repo),
-    ]);
+    const token = await getToken({
+      req,
+      secret: process.env.NEXTAUTH_SECRET,
+    });
+    const accessToken =
+      typeof token?.accessToken === "string" ? token.accessToken : undefined;
+
+    const { repoInfo, repoContents } = await getRepoSnapshot(
+      owner,
+      repo,
+      accessToken,
+    );
+
+    const isPrivateRepo = Boolean(repoInfo?.private);
+    if (isPrivateRepo && !ackPrivateRepo) {
+      return NextResponse.json(
+        {
+          error: "private_repo_consent_required",
+          message:
+            "This repository appears to be private. Confirm consent to send private repository data to the AI model by re-submitting with { ackPrivateRepo: true }.",
+          authRequired: Boolean(accessToken),
+        },
+        { status: 403 },
+      );
+    }
 
     const files = Array.isArray(repoContents)
-      ? repoContents.map((f: { name: string }) => f.name)
+      ? repoContents.map((f: { path: string }) => f.path)
       : [];
     const fileListString =
       files.length > 0 ? files.join(", ") : "Standard repository structure";
@@ -158,12 +180,50 @@ export async function POST(req: Request) {
 
     return NextResponse.json({ markdown: cleanMarkdown });
   } catch (error: unknown) {
+    if (error instanceof RepoAccessError) {
+      return NextResponse.json(
+        {
+          error: error.message,
+          authRequired: error.code === "AUTH_REQUIRED",
+        },
+        { status: error.status },
+      );
+    }
+
     const message =
       error instanceof Error ? error.message : "Internal Server Error";
     console.error("README Generation Failed:", message);
 
+    // Return successful result with enhanced metadata
+    return NextResponse.json({
+      success: true,
+      markdown: result.readme, // Keep 'markdown' key for compatibility with existing frontend
+      stats: {
+        sectionsGenerated: result.stats.sectionsGenerated,
+        sectionsTotal: result.stats.sectionsTotal,
+        tokensUsed: result.stats.tokensUsed,
+        timeElapsed: result.stats.timeElapsed,
+        generationMethod: "multi-step", // Indicate the method used
+      },
+      metadata: {
+        name: result.metadata?.name,
+        description: result.metadata?.description,
+        language: result.metadata?.language,
+        stars: result.metadata?.stars,
+        license: result.metadata?.license,
+        projectType: result.structure?.projectType,
+        techStack: result.structure?.techStack.primary,
+        frameworks: result.structure?.techStack.frameworks,
+      },
+      warnings: result.errors.length > 0 ? result.errors : undefined,
+    });
+  } catch (error) {
+    console.error("Multi-step README generation API error:", error);
     return NextResponse.json(
-      { error: "Failed to generate README. Check your URL and try again." },
+      {
+        error: "Internal server error in multi-step README generation",
+        message: error instanceof Error ? error.message : "Unknown error",
+      },
       { status: 500 },
     );
   }

src/app/api/generate/route.ts.backup

@@ -0,0 +1,170 @@
+import { NextResponse } from "next/server";
+import { getGeminiModel } from "@/lib/gemini";
+import { getRepoData, getRepoContents } from "@/lib/octokit";
+import { SUPPORTED_LANGUAGES } from "@/constants/languages";
+
+export const dynamic = "force-dynamic";
+
+/**
+ * AI README Generation Endpoint
+ * Optimized for data accuracy, clean prompt interpolation, and multi-language support.
+ *
+ * @param {Request} req - The incoming Next.js/standard Web API Request object containing the repo URL and optional language.
+ * @returns {Promise<NextResponse>} A JSON response containing the generated Markdown or an error message.
+ */
+export async function POST(req: Request) {
+  let rawUrl: string;
+  let language: string;
+  try {
+    const body = await req.json();
+    rawUrl = body.url;
+    language = body.language || "English";
+  } catch {
+    return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 });
+  }
+
+  try {
+    const trimmedUrl = rawUrl?.trim();
+    if (!trimmedUrl) {
+      return NextResponse.json(
+        { error: "GitHub URL is required" },
+        { status: 400 },
+      );
+    }
+
+    let parsedUrl: URL;
+    try {
+      parsedUrl = new URL(trimmedUrl);
+    } catch {
+      return NextResponse.json(
+        { error: "Please provide a valid URL" },
+        { status: 400 },
+      );
+    }
+
+    if (
+      parsedUrl.hostname !== "github.com" &&
+      parsedUrl.hostname !== "www.github.com"
+    ) {
+      return NextResponse.json(
+        { error: "Only GitHub URLs are supported" },
+        { status: 400 },
+      );
+    }
+
+    const pathSegments = parsedUrl.pathname.split("/").filter(Boolean);
+    const owner = pathSegments[0];
+    const repo = pathSegments[1];
+
+    if (!owner || !repo) {
+      return NextResponse.json(
+        { error: "URL must include owner and repository name" },
+        { status: 400 },
+      );
+    }
+
+    const [repoInfo, repoContents] = await Promise.all([
+      getRepoData(owner, repo),
+      getRepoContents(owner, repo),
+    ]);
+
+    const files = Array.isArray(repoContents)
+      ? repoContents.map((f: { name: string }) => f.name)
+      : [];
+    const fileListString =
+      files.length > 0 ? files.join(", ") : "Standard repository structure";
+
+    // Tech Stack detection logic
+    const hasNode = files.includes("package.json");
+    const hasPython =
+      files.includes("requirements.txt") || files.includes("setup.py");
+    const hasDocker =
+      files.includes("Dockerfile") || files.includes("docker-compose.yml");
+
+    // Fix: Cleanly joined Tech Stack labels
+    const stackLabels =
+      [
+        hasNode && "Node.js Environment",
+        hasPython && "Python Environment",
+        hasDocker && "Containerized",
+      ]
+        .filter(Boolean)
+        .join(", ") || "Generic Software Environment";
+
+    // Fix: Dynamic License detection
+    const licenseName =
+      repoInfo?.license?.name ||
+      repoInfo?.license?.spdx_id ||
+      "the repository's license file";
+
+    const model = getGeminiModel();
+
+    // Fix: Prompt updated with neutral fallbacks and dynamic license
+    const prompt = `
+**Role**: You are a Principal Solutions Architect and World-Class Technical Writer. 
+**Task**: Generate a professional, high-conversion README.md for the GitHub repository: "${repo}" in the following language: **${language}**.
+
+---
+### 1. PROJECT CONTEXT (VERIFIED DATA)
+- **Project Name**: ${repo}
+- **Description**: ${repoInfo?.description || "No description provided."}
+- **Primary Language**: ${repoInfo?.language || "Language unknown"}
+- **Detected Root Files**: ${fileListString}
+- **Tech Stack Context**: ${stackLabels}
+
+---
+### 2. STRICT README STRUCTURE REQUIREMENTS
+
+1. **Visual Header**:
+   - Center-aligned H1 with project name.
+   - A compelling 1-sentence tagline describing the **Value Proposition**.
+   - A centered row of Shields.io badges (Build, License, PRs Welcome, Stars).
+
+2. **The Strategic "Why" (Overview)**:
+   - **The Problem**: Use a blockquote to describe the real-world pain point this project solves.
+   - **The Solution**: Explain how this project provides a superior outcome for the user.
+
+3. **Key Features**:
+   - Minimum 5 features. Use emojis and focus on **User Benefits**.
+
+4. **Technical Architecture**:
+   - Provide a table of the tech stack: | Technology | Purpose | Key Benefit |.
+   - Create a tree-style directory structure code block using 📁 for folders and 📄 for files based on the file manifest provided.
+
+5. **Operational Setup**:
+   - **Prerequisites**: List required runtimes.
+   - **Installation**: Provide step-by-step terminal commands. 
+     ${hasNode ? "- Use npm/yarn/pnpm since package.json was detected." : ""}
+     ${hasPython ? "- Use pip/venv since Python markers were detected." : ""}
+   - **Environment**: If any .env or config files are in the manifest, include a configuration section.
+
+6. **Community & Governance**:
+   - Professional "Contributing" section (Fork -> Branch -> PR).
+   - Detailed "License" section: Reference ${licenseName} and provide a summary of permissions.
+
+---
+### 3. TONE & STYLE
+- **Tone**: Authoritative, polished, and developer-centric.
+- **Visuals**: Extensive use of Markdown formatting.
+- **Constraint**: Return ONLY the raw Markdown. No conversational filler.
+    `;
+
+    const result = await model.generateContent(prompt);
+    const response = await result.response;
+    const markdown = response.text().trim();
+    const cleanMarkdown = markdown
+      .replace(/^```(markdown|md)?\n/, "")
+      .replace(/\n```$/, "");
+
+    return NextResponse.json({ markdown: cleanMarkdown });
+  } catch (error: unknown) {
+    const message =
+      error instanceof Error ? error.message : "Internal Server Error";
+    console.error("README Generation Failed:", message);
+
+    return NextResponse.json(
+      { error: "Failed to generate README. Check your URL and try again." },
+      { status: 500 },
+    );
+  }
+}