From f3ef055cdfe37c58792376bba934e28d68f6776a Mon Sep 17 00:00:00 2001 From: ly-wang19 Date: Wed, 10 Jun 2026 14:55:26 +0800 Subject: [PATCH] fix(pbl): explain the tool-calling requirement on immediate rejection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PBL is the only generation path that drives the model through native tool/function calls, and providers whose chat template lacks tools support reject such requests outright — surfacing as a raw provider error (e.g. an instant 500 from LM Studio with Gemma 4 in #664) with no hint that the model, not the course, is the problem. Track agentic-loop progress and, when the request fails before the first step completes — the signature of missing tool support — wrap the error with an actionable message naming the requirement and the way out (pick a tool-capable model or change the scene type). Failures after steps have completed rethrow unchanged so mid-loop network or provider errors are not misdiagnosed. Regression tests cover both branches. Refs #664. --- lib/pbl/generate-pbl.ts | 58 ++++++++++++------- tests/pbl/generate-pbl-tool-rejection.test.ts | 58 +++++++++++++++++++ 2 files changed, 96 insertions(+), 20 deletions(-) create mode 100644 tests/pbl/generate-pbl-tool-rejection.test.ts diff --git a/lib/pbl/generate-pbl.ts b/lib/pbl/generate-pbl.ts index a0c47be2db..d6e28a36db 100644 --- a/lib/pbl/generate-pbl.ts +++ b/lib/pbl/generate-pbl.ts @@ -285,28 +285,46 @@ export async function generatePBLContent( // Run the agentic loop const systemPrompt = buildPBLSystemPrompt(config); - const _result = await callLLM( - { - model, - system: systemPrompt, - prompt: `Design a PBL project. Start in project_info mode by setting the project title and description.`, - tools: pblTools, - stopWhen: stepCountIs(30), - onStepFinish: ({ toolCalls, text }) => { - if (text) { - callbacks?.onProgress?.(`Thinking: ${text.slice(0, 100)}...`); - } - if (toolCalls) { - for (const tc of toolCalls) { - callbacks?.onProgress?.(`Tool: ${tc.toolName}`); + // PBL is the only generation path that requires native tool calling, and + // providers whose chat template lacks tools support reject such requests + // outright (e.g. LM Studio templates without a tools section, see #664). + // Track loop progress so a rejection before the first step — the signature + // of missing tool support — gets an actionable message instead of a raw + // provider error, while mid-loop failures keep their original error. + let completedSteps = 0; + try { + await callLLM( + { + model, + system: systemPrompt, + prompt: `Design a PBL project. Start in project_info mode by setting the project title and description.`, + tools: pblTools, + stopWhen: stepCountIs(30), + onStepFinish: ({ toolCalls, text }) => { + completedSteps += 1; + if (text) { + callbacks?.onProgress?.(`Thinking: ${text.slice(0, 100)}...`); } - } + if (toolCalls) { + for (const tc of toolCalls) { + callbacks?.onProgress?.(`Tool: ${tc.toolName}`); + } + } + }, }, - }, - 'pbl-generate', - undefined, - thinkingConfig, - ); + 'pbl-generate', + undefined, + thinkingConfig, + ); + } catch (error) { + if (completedSteps === 0) { + const message = error instanceof Error ? error.message : String(error); + throw new Error( + `PBL generation was rejected before the first step. PBL scenes drive the model through tool/function calls, so the selected model must support tool calling; models without it reject the request immediately. Pick a tool-capable model for PBL scenes, or change this scene's type. Original error: ${message}`, + ); + } + throw error; + } // Check if mode reached idle; if not, the LLM may have stopped early if (modeMCP.getCurrentMode() !== 'idle') { diff --git a/tests/pbl/generate-pbl-tool-rejection.test.ts b/tests/pbl/generate-pbl-tool-rejection.test.ts new file mode 100644 index 0000000000..e74addc7d4 --- /dev/null +++ b/tests/pbl/generate-pbl-tool-rejection.test.ts @@ -0,0 +1,58 @@ +/** + * Regression tests for issue #664: PBL generation is the only path that + * requires native tool calling, and models without tool support reject the + * request outright with a raw provider error (e.g. an immediate 500 from + * LM Studio when the chat template has no tools section). + * + * A rejection before the first agentic step must be wrapped with an + * actionable message naming the tool-calling requirement; a failure after + * steps have completed must keep its original error (no misdiagnosis). + */ +import { describe, expect, it, vi, beforeEach } from 'vitest'; + +vi.mock('@/lib/ai/llm', () => ({ + callLLM: vi.fn(), +})); + +import { callLLM } from '@/lib/ai/llm'; +import { generatePBLContent, type GeneratePBLConfig } from '@/lib/pbl/generate-pbl'; +import type { LanguageModel } from 'ai'; + +type StepCallback = (step: { toolCalls: unknown[]; text: string }) => void; + +const config: GeneratePBLConfig = { + projectTopic: 'Bridge building', + projectDescription: 'Design and stress-test a model bridge', + targetSkills: ['statics'], + languageDirective: 'Respond in English.', +}; + +const model = {} as unknown as LanguageModel; + +describe('generatePBLContent tool-rejection context (issue #664)', () => { + beforeEach(() => { + vi.mocked(callLLM).mockReset(); + }); + + it('wraps a rejection before the first step with the tool-calling requirement', async () => { + vi.mocked(callLLM).mockRejectedValue( + new Error('500 status: this model template does not support tools'), + ); + + await expect(generatePBLContent(config, model)).rejects.toThrow( + /must support tool calling[\s\S]*Original error: 500 status: this model template does not support tools/, + ); + }); + + it('keeps the original error when the loop fails after completing a step', async () => { + vi.mocked(callLLM).mockImplementation(async (params) => { + // Simulate one completed agentic step, then a mid-loop failure. + (params as { onStepFinish?: StepCallback }).onStepFinish?.({ toolCalls: [], text: 'plan' }); + throw new Error('network reset mid-loop'); + }); + + const failure = generatePBLContent(config, model); + await expect(failure).rejects.toThrow('network reset mid-loop'); + await expect(failure).rejects.not.toThrow(/must support tool calling/); + }); +});