diff --git a/app/api/generate/scene-outlines-stream/route.ts b/app/api/generate/scene-outlines-stream/route.ts index f0d599e8fc..b7ea020ba2 100644 --- a/app/api/generate/scene-outlines-stream/route.ts +++ b/app/api/generate/scene-outlines-stream/route.ts @@ -337,6 +337,8 @@ export async function POST(req: NextRequest) { mediaEnabled: mediaGenerationEnabled, teacherContext, userProfile: userProfileText, + // Design Brief mode: emit a natural-language `brief` + structured `media[]` per slide. + designBriefMode: requirements.designBriefMode === true, }); if (!prompts) { diff --git a/lib/generation/outline-generator.ts b/lib/generation/outline-generator.ts index 1c90feab60..7e938d445c 100644 --- a/lib/generation/outline-generator.ts +++ b/lib/generation/outline-generator.ts @@ -100,6 +100,7 @@ export async function generateSceneOutlinesFromRequirements( imageEnabled, videoEnabled, mediaEnabled, + designBriefMode: requirements.designBriefMode === true, researchContext: options?.researchContext || 'None', // Server-side generation populates this via options; client-side populates via formatTeacherPersonaForPrompt teacherContext: options?.teacherContext || '', diff --git a/lib/generation/scene-builder.ts b/lib/generation/scene-builder.ts index 478e179fdc..309d5fb25b 100644 --- a/lib/generation/scene-builder.ts +++ b/lib/generation/scene-builder.ts @@ -33,10 +33,34 @@ const log = createLogger('Generation'); * contamination on the homepage. Using nanoid-based IDs ensures global uniqueness. */ export function uniquifyMediaElementIds(outlines: SceneOutline[]): SceneOutline[] { + // Design Brief mode bridge: the outliner emits a structured `media[]` manifest + // (parallel to `brief`). Derive the existing `mediaGenerations` (generate items) + // and `suggestedImageIds` (asset items) from it so the existing media-orchestrator + // dispatches generation. Skipped when mediaGenerations is already populated. + const bridged = outlines.map((outline) => { + if (!outline.media || outline.media.length === 0) return outline; + const gen = outline.media.filter((m) => m.source === 'generate'); + const assetIds = outline.media.filter((m) => m.source === 'asset').map((m) => m.id); + const mediaGenerations = + outline.mediaGenerations && outline.mediaGenerations.length > 0 + ? outline.mediaGenerations + : gen.map((m) => ({ + type: m.type, + prompt: m.prompt ?? m.caption ?? '', + elementId: m.id, + aspectRatio: m.aspectRatio, + })); + const suggestedImageIds = + assetIds.length > 0 + ? Array.from(new Set([...(outline.suggestedImageIds ?? []), ...assetIds])) + : outline.suggestedImageIds; + return { ...outline, mediaGenerations, suggestedImageIds }; + }); + const idMap = new Map(); // First pass: collect all sequential media IDs and assign unique replacements - for (const outline of outlines) { + for (const outline of bridged) { if (!outline.mediaGenerations) continue; for (const mg of outline.mediaGenerations) { if (!idMap.has(mg.elementId)) { @@ -46,17 +70,25 @@ export function uniquifyMediaElementIds(outlines: SceneOutline[]): SceneOutline[ } } - if (idMap.size === 0) return outlines; + if (idMap.size === 0) return bridged; + + // Rewrite a brief's inline id references (e.g. "right column shows gen_img_1") to + // keep them in sync with the uniquified ids. `\d+\b` captures the full number so + // gen_img_1 and gen_img_10 stay distinct. + const rewriteBrief = (brief: string | undefined): string | undefined => + brief?.replace(/\b(?:gen_(?:img|vid)|img)_\d+\b/g, (token) => idMap.get(token) ?? token); - // Second pass: replace IDs in mediaGenerations - return outlines.map((outline) => { - if (!outline.mediaGenerations) return outline; + // Second pass: replace IDs in mediaGenerations, media[], and the brief together + return bridged.map((outline) => { + if (!outline.mediaGenerations && !outline.media && !outline.brief) return outline; return { ...outline, - mediaGenerations: outline.mediaGenerations.map((mg) => ({ + mediaGenerations: outline.mediaGenerations?.map((mg) => ({ ...mg, - elementId: idMap.get(mg.elementId) || mg.elementId, + elementId: idMap.get(mg.elementId) ?? mg.elementId, })), + media: outline.media?.map((m) => ({ ...m, id: idMap.get(m.id) ?? m.id })), + brief: rewriteBrief(outline.brief), }; }); } diff --git a/lib/generation/scene-generator.ts b/lib/generation/scene-generator.ts index 0a86a7157e..8cdd1c0b9a 100644 --- a/lib/generation/scene-generator.ts +++ b/lib/generation/scene-generator.ts @@ -738,6 +738,9 @@ async function generateSlideContent( title: outline.title, description: outline.description, keyPoints: (outline.keyPoints || []).map((p, i) => `${i + 1}. ${p}`).join('\n'), + // Design Brief mode: when the outline carries a natural-language `brief`, it is + // the authoritative layout spec (the prompt prefers it over title/keyPoints). + brief: outline.brief?.trim() || '', elements: '(根据要点自动生成)', assignedImages: assignedImagesText, canvas_width: canvasWidth, diff --git a/lib/prompts/templates/requirements-to-outlines/system.md b/lib/prompts/templates/requirements-to-outlines/system.md index 096dba2582..153bb029dc 100644 --- a/lib/prompts/templates/requirements-to-outlines/system.md +++ b/lib/prompts/templates/requirements-to-outlines/system.md @@ -279,6 +279,10 @@ Rules: | title | string | ✅ | Scene title, concise and clear | | description | string | ✅ | 1-2 sentences describing teaching purpose | | keyPoints | string[] | ✅ | 3-5 core points | +{{#if designBriefMode}} +| brief | string | ✅ (for slide) | Detailed natural-language design brief for `slide` scenes — see "Slide Design Brief" below | +| media | SlideMediaItem[] | ❌ (for slide) | Structured manifest of the images/videos this slide actually needs; the brief references each by id — see "Slide Design Brief" below. Omit when the slide needs no media. | +{{/if}} | teachingObjective | string | ❌ | Corresponding learning objective | | estimatedDuration | number | ❌ | Estimated duration (seconds) | | order | number | ✅ | Sort order, starting from 1 | @@ -294,6 +298,65 @@ Rules: | widgetOutline | object | ✅ (for interactive) | Widget-specific configuration (see Widget Type Selection) | | pblConfig | object | ❌ | Required for pbl type, contains projectTopic/projectDescription/targetSkills/issueCount/language | +{{#if designBriefMode}} +### Slide Design Brief (required for every `slide` scene) + +For every scene with `"type": "slide"`, add a `brief` field: a detailed natural-language design brief that a layout designer could turn directly into a finished slide. This is the PRIMARY input to the slide renderer — write rich, specific, flowing prose (roughly 120–280 words), not a bullet list. Keep `description` and `keyPoints` short as before; invest the real detail in `brief`. + +Each `brief` must cover, in prose: + +1. **Page goal** — the single concept this one slide must land and what the viewer grasps at a glance. One slide = one idea; never cram a whole lesson. +2. **Overall visual style** — the look plus a simple color palette in words (e.g. "white background, deep-blue title, teal accent line, light-gray table"), and whether to avoid images or use them. +3. **Layout in regions, not pixels** — e.g. a top title band, then a left/right column split with rough proportions ("left column ~60% holds the table, right ~40% the diagram"), and grouping/spacing. NEVER give pixel coordinates, px sizes, or font sizes. +4. **The actual content of each block, written out** — do NOT say "a table comparing X"; write the real rows, numbers, and formulas. Wrap anything that must appear exactly (formulas, numbers, proper nouns, technical terms) in double quotes, e.g. the formula `"3x + 5 = 17"`, the value `"42%"`, the term `"SYN-ACK"`. On-slide text must be in the course language from `languageDirective`. +5. **Visual emphasis** — name the focal element and what is secondary. +6. **Media discipline** — do NOT add decorative images or per-card icons just to fill space; **most slides need ZERO images**. Draw dividers, badges and simple icons with `shape`/`line`, and prefer `table`/`chart`/`latex` to carry information. Only request a real image/video when it carries irreplaceable visual information (an actual diagram, a photograph, an illustrative scene). When you do, put it in the structured `media` manifest below — **never just write "add an icon/picture" in the prose.** + +#### The `media` manifest (structured, parallel to `brief`) + +When — and only when — a slide genuinely needs an image or video, also output a `media` array listing each one, and have the `brief` refer to each item **by its `id`** (e.g. `right column shows gen_img_1 (a GAD-vs-human scatter plot)`) rather than vaguely saying "add a figure". Each item: + +- `id`: `gen_img_1`, `gen_img_2`, … for to-be-generated images (`gen_vid_1`, … for video). Use a real `img_N` id **only** when that image is listed under Available Images. +- `source`: `"generate"` (AI will create it) or `"asset"` (a real listed `img_N` already exists). +- `type`: `"image"` or `"video"`. +- `prompt`: concise English description of what to generate — **REQUIRED for `source:"generate"`**, omit for `"asset"`. +- `caption`: one short phrase for what it depicts / its role. +- `aspectRatio`: `"16:9"` | `"4:3"` | `"1:1"` | `"9:16"` (optional, default 16:9). + +**Consistency is mandatory**: every id referenced in `brief` MUST appear in `media`, and every id in `media` MUST be referenced in `brief`. If the slide needs no media, omit `media` and do not mention any image id in the brief. + +**Example A — a content slide that needs NO media (the common case):** + +```json +{ + "id": "scene_4", + "type": "slide", + "title": "Special-Angle Trig Values", + "description": "Help students quickly look up trig values for special angles.", + "keyPoints": ["quick-reference table", "unit circle", "memory steps"], + "brief": "A clean reference slide titled \"Special-Angle Trig Values\". White background, deep-blue title, teal accent line, light blue-gray table; no images. Top ~12% is the title band with a small right-aligned tag \"radians · unit circle · quick table\", and a thin teal rule under it. The body is a left/right split: the left column (~62%) holds the focal table — 6 rows × 5 columns with headers \"angle, radians, sinθ, cosθ, tanθ\" and data \"0°|0|0|1|0\", \"30°|π/6|1/2|√3/2|√3/3\", \"45°|π/4|√2/2|√2/2|1\", \"60°|π/3|√3/2|1/2|√3\", \"90°|π/2|1|0|undefined\"; deep-blue header row, with a light-gray note below it: \"tanθ = sinθ / cosθ, so tanθ is undefined when cosθ = 0\". The right column (~38%) draws a simple unit circle marking the \"30°\", \"45°\", \"60°\" radii, and below it three memory-step cards \"1 convert to radians\", \"2 find the reference angle\", \"3 fix the quadrant sign\" with teal borders.", + "order": 4 +} +``` + +**Example B — a slide that genuinely needs ONE figure (structured `media` + id reference in the brief):** + +```json +{ + "id": "scene_6", + "type": "slide", + "title": "GAD: Right Density, No Overlap, No Fragmentation", + "description": "Use a scatter plot to show GAD correlates strongly with human ratings.", + "keyPoints": ["GAD=OM+FR", "correlates with human score", "RMSE/ρ"], + "media": [ + { "id": "gen_img_1", "source": "generate", "type": "image", "prompt": "scatter plot showing strong positive correlation between a GAD layout metric on the x-axis and human ratings on the y-axis, clean academic style", "caption": "GAD vs human-rating scatter plot", "aspectRatio": "4:3" } + ], + "brief": "An academic slide titled \"GAD: Right Density, No Overlap, No Fragmentation\". White background, deep-blue title, teal accents, with a thin rule under the title band. Centered near the top is the formula \"GAD = OM + FR\". Below it a left/right split: the left column explains the three terms in words (Occupancy Matching OM, Fragmentation Reward FR, overall density), and the right column shows gen_img_1 (the GAD-vs-human-rating scatter plot) with a one-line caption beneath it. A centered light-yellow stat box at the bottom reads \"RMSE = 0.580 ρ = 0.820\". No other images or icons besides that scatter plot.", + "order": 6 +} +``` +{{/if}} + ### quizConfig Structure ```json diff --git a/lib/prompts/templates/requirements-to-outlines/user.md b/lib/prompts/templates/requirements-to-outlines/user.md index a00c1ff1a0..1186cf6914 100644 --- a/lib/prompts/templates/requirements-to-outlines/user.md +++ b/lib/prompts/templates/requirements-to-outlines/user.md @@ -86,6 +86,10 @@ Never return a bare array. Never omit `languageDirective`. Both keys are require {{#if hasSourceImages}} - **If source images are available**, add `suggestedImageIds` to relevant slide scenes. Only use image IDs listed under Available Images. {{/if}} +{{#if designBriefMode}} +- **Every `slide` scene MUST include a `brief`** — a detailed natural-language design brief (~120–280 words) following the "Slide Design Brief" section in the system prompt: page goal, visual style + color palette in words, region-based layout (no pixels), the actual written-out content (real rows/numbers/formulas, verbatim parts in quotes), and focal emphasis. `description`/`keyPoints` stay short; the real detail goes in `brief`. +- **Media discipline**: most slides need NO images — draw icons/dividers with shapes and carry information with `table`/`chart`/`latex`. Do NOT add decorative images or per-card icons. ONLY when a slide genuinely needs a figure/photo/video, add a structured `media` array (each `{id, source, type, prompt?, caption, aspectRatio?}`) and reference each item in the `brief` by its `id` (e.g. `gen_img_1`). Every brief id ⇔ a `media` entry (both directions). No media → omit `media` and mention no image id. +{{/if}} - **Interactive scenes**: If a concept benefits from hands-on simulation/visualization, use `"type": "interactive"` with `widgetType` and `widgetOutline` fields. Limit to 1-2 per course. - Select widgetType based on concept: simulation (physics/chem), diagram (processes), code (programming), game (practice), visualization3d (3D models) - Provide appropriate widgetOutline for the widget type diff --git a/lib/prompts/templates/slide-content/user.md b/lib/prompts/templates/slide-content/user.md index 844139b9c3..2d38fbae54 100644 --- a/lib/prompts/templates/slide-content/user.md +++ b/lib/prompts/templates/slide-content/user.md @@ -7,6 +7,14 @@ - **Key Points**: {{keyPoints}} +{{#if brief}} +## Design Brief (authoritative) + +A detailed design brief is provided below. Treat it as the **authoritative** layout and content spec for this slide — the Title/Description/Key Points above are only a summary. Realize the brief faithfully: preserve its visual hierarchy, regions and proportions, and render the content it spells out (tables, formulas, lists, media). When the brief references a media id (e.g. `gen_img_1`, `img_2`), use that exact id as the element `src`. + +{{brief}} +{{/if}} + {{teacherContext}} ## Available Resources diff --git a/lib/types/generation.ts b/lib/types/generation.ts index 845846fe8b..b6bee8b63c 100644 --- a/lib/types/generation.ts +++ b/lib/types/generation.ts @@ -53,6 +53,7 @@ export interface UserRequirements { webSearch?: boolean; // Enable web search for richer context interactiveMode?: boolean; // Enable Interactive Mode for interactive-first generation taskEngineMode?: boolean; // Enable vocational task-engine generation path + designBriefMode?: boolean; // Outliner emits a natural-language design `brief` + structured `media[]` per slide scene } // ==================== Stage 1 Output: Scene Outlines (Simplified) ==================== @@ -85,6 +86,28 @@ export interface WidgetOutline { challengeType?: string; // code - type of coding challenge } +/** + * Structured media item for a slide outline (Design Brief mode). + * + * Unifies real source images (e.g. PDF-extracted) and to-be-generated media into + * ONE manifest that sits parallel to `brief`. The brief references each item by + * `id`, so the slide-content model is told exactly which media exist (and whether + * each is a real asset or a placeholder) instead of inferring image intent from + * prose — which avoids over-emitting decorative image placeholders. + */ +export interface SlideMediaItem { + /** Id used both in the brief reference and as the slide element `src`. */ + id: string; // 'img_1' (source asset) | 'gen_img_1' / 'gen_vid_1' (to generate) + /** 'asset' = a real image already available; 'generate' = AI-generated placeholder. */ + source: 'asset' | 'generate'; + type: 'image' | 'video'; + /** Generation prompt — required for source:'generate'; omit for 'asset'. */ + prompt?: string; + /** What it depicts / its role on the page; shown to the slide-content model. */ + caption?: string; + aspectRatio?: '16:9' | '4:3' | '1:1' | '9:16'; +} + /** * Simplified scene outline * Gives AI more freedom, only requiring intent description and key points @@ -95,6 +118,12 @@ export interface SceneOutline { title: string; description: string; // 1-2 sentences describing the purpose keyPoints: string[]; // 3-5 core key points + // Design Brief mode (UserRequirements.designBriefMode): a detailed natural-language + // design brief for `slide` scenes, used as the primary slide-content input when present. + brief?: string; + // Design Brief mode: structured manifest of the images/videos this slide actually + // needs; the brief references each by id. Unifies source (asset) and generate. + media?: SlideMediaItem[]; teachingObjective?: string; estimatedDuration?: number; // seconds order: number;