refactor: route all markdown parsing through mdq() (plan 022)

src/ai/experience-compactor.ts

@@ -1,12 +1,12 @@
 import { unlinkSync } from 'node:fs';
 import { basename } from 'node:path';
 import dedent from 'dedent';
-import { type Tokens, marked } from 'marked';
 import { z } from 'zod';
 import { type ExperienceFile, type ExperienceTracker, RECENT_WINDOW_DAYS } from '../experience-tracker.js';
 import { Observability } from '../observability.js';
 import { createDebug, log, tag } from '../utils/logger.js';
 import { mdq } from '../utils/markdown-query.js';
+import { isNonReusableCode } from '../utils/step-analyzer.js';
 import { generalizeUrl, hasDynamicUrlSegment } from '../utils/url-matcher.js';
 import type { Agent } from './agent.js';
 import type { Provider } from './provider.js';
@@ -438,28 +438,9 @@ export class ExperienceCompactor implements Agent {
 }
 
 function listSections(content: string): { title: string; raw: string }[] {
-  const tokens = marked.lexer(content);
-  const sections: { title: string; raw: string }[] = [];
-
-  let currentHeading: string | null = null;
-  let currentRaw = '';
-
-  const flush = () => {
-    if (currentHeading !== null) sections.push({ title: currentHeading, raw: currentRaw });
-  };
-
-  for (const token of tokens) {
-    const raw = (token as any).raw || '';
-    if (token.type === 'heading' && (token as Tokens.Heading).depth === 2) {
-      flush();
-      currentHeading = (token as Tokens.Heading).text.trim();
-      currentRaw = raw;
-      continue;
-    }
-    if (currentHeading !== null) currentRaw += raw;
-  }
-  flush();
-  return sections;
+  const sections = mdq(content).query('section2').each();
+  const metas = mdq(content).query('section2').meta();
+  return sections.map((q, i) => ({ title: metas[i].text.trim(), raw: q.text() }));
 }
 
 function dropEmptySections(content: string): string {
@@ -483,7 +464,7 @@ function dropNonReusableSections(content: string): string {
 
   for (const section of sections) {
     const raw = section.text();
-    if (!/\bI\.clickXY\s*\(/.test(raw)) continue;
+    if (!isNonReusableCode(raw)) continue;
     result = result.replace(raw, '');
   }
 

src/ai/planner.ts

@@ -280,29 +280,36 @@ export class Planner extends PlannerBase implements Agent {
   }
 
   private cleanExperienceFlows(text: string): string | null {
-    const seenTitles = new Set<string>();
     let result = text;
+    const seenTitles = new Set<string>();
+    for (const selector of ['section2', 'section3']) {
+      result = mdq(result)
+        .query(selector)
+        .replaceEach((section) => {
+          const heading = section.query('heading').text().trim();
+          const withoutHeadings = mdq(section.text()).query('heading').replace('');
+          const body = mdq(withoutHeadings).query('hr').replace('').trim();
+          if (body && !seenTitles.has(heading)) {
+            seenTitles.add(heading);
+            return section.text();
+          }
+          return '';
+        });
+    }
 
-    for (const section of [...mdq(text).query('section2').each(), ...mdq(text).query('section3').each()]) {
-      const heading = section.query('heading').text().trim();
-      const body = mdq(section.text())
-        .query('heading')
-        .replace('')
-        .replace(/^---\s*$/gm, '')
-        .trim();
-
-      if (!body || seenTitles.has(heading)) {
-        result = result.replace(section.text(), '');
-        continue;
-      }
-      seenTitles.add(heading);
-
-      const blockquotes = section.query('blockquote').each();
-      if (blockquotes.length <= 10) continue;
-      for (const bq of blockquotes.slice(10)) {
-        result = result.replace(bq.text(), '');
-      }
-      result = result.replace(section.text().trim(), `${section.text().trim()}\n> ... and ${blockquotes.length - 10} more discoveries`);
+    const trimmedTitles = new Set<string>();
+    for (const selector of ['section2', 'section3']) {
+      result = mdq(result)
+        .query(selector)
+        .replaceEach((section) => {
+          const heading = section.query('heading').text().trim();
+          if (trimmedTitles.has(heading)) return section.text();
+          const count = section.query('blockquote').count();
+          if (count <= 10) return section.text();
+          const kept = mdq(section.text()).query('blockquote[10:]').replace('');
+          trimmedTitles.add(heading);
+          return `${kept.trimEnd()}\n> ... and ${count - 10} more discoveries\n`;
+        });
     }
 
     return result.trim() || null;
@@ -382,16 +389,17 @@ export class Planner extends PlannerBase implements Agent {
       deep: true,
     });
     let plannerResearch = mdq(research).query('code').replace('');
-    for (const table of mdq(plannerResearch).query('table').each()) {
-      const rawTable = table.text();
-      const rows = table.toJson();
-      if (rows.length === 0 || !rows[0].Element) continue;
-      const elementWithType = rows.map((r) => ({
-        Element: r.Element,
-        Type: r.Type || '',
-      }));
-      plannerResearch = plannerResearch.replace(rawTable, jsonToTable(elementWithType, ['Element', 'Type']));
-    }
+    plannerResearch = mdq(plannerResearch)
+      .query('table')
+      .replaceEach((table) => {
+        const rows = table.toJson();
+        if (rows.length === 0 || !rows[0].Element) return table.text();
+        const elementWithType = rows.map((r) => ({
+          Element: r.Element,
+          Type: r.Type || '',
+        }));
+        return jsonToTable(elementWithType, ['Element', 'Type']);
+      });
 
     const hasFocusedOverlay = hasFocusedSection(plannerResearch);
     const focusNote = hasFocusedOverlay ? "IMPORTANT: One section is marked as **Focused** — this is the user's current focus area. Concentrate testing on the Focused section FIRST — test all interactions inside it before planning tests for the rest of the page." : '';

src/ai/researcher.ts

@@ -314,11 +314,9 @@ export class Researcher extends ResearcherBase implements Agent {
         researchFile = saveResearch(stateHash, result.text, combinedHtml);
       }
 
-      const summaryMatch = result.text.match(/## Summary\s*\n+([\s\S]*?)(?=\n##|$)/i);
-      if (summaryMatch) {
-        const summaryLine = summaryMatch[1].trim().split('\n')[0].trim().slice(0, 200);
-        if (summaryLine) this.experienceTracker.updateSummary(this.actionResult!, summaryLine);
-      }
+      const summaryText = mdq(result.text).query('section2(/^summary/)').query('paragraph[0]').text().trim();
+      const summaryLine = summaryText.split('\n')[0]?.trim().slice(0, 200);
+      if (summaryLine) this.experienceTracker.updateSummary(this.actionResult!, summaryLine);
 
       tag('multiline').log(formatResearchSummary(result.text, { visionUsed: this.hasScreenshotToAnalyze }));
       tag('success').log('Research complete');

src/ai/researcher/parser.ts

@@ -94,7 +94,7 @@ export function extractContainerFromBlockquote(sectionMarkdown: string): string
 }
 
 export function parseResearchSections(markdown: string): ResearchSection[] {
-  const hasExtendedResearch = markdown.includes('\n# Extended Research') || markdown.startsWith('# Extended Research');
+  const hasExtendedResearch = mdq(markdown).query('section1(~"Extended Research")').count() > 0;
 
   return parseSections(markdown)
     .filter((s) => !SKIP_SECTIONS.has(s.name.toLowerCase()) && !s.name.toLowerCase().includes('data:'))

src/experience-tracker.ts

@@ -1,7 +1,6 @@
 import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
 import { basename, join } from 'node:path';
 import matter from 'gray-matter';
-import { type Tokens, marked } from 'marked';
 import type { ActionResult } from './action-result.js';
 import { ConfigParser } from './config.js';
 import { KnowledgeTracker } from './knowledge-tracker.js';
@@ -159,7 +158,12 @@ export class ExperienceTracker {
     this.ensureExperienceFile(state);
     const stateHash = state.getStateHash();
     const { content, data } = this.readExperienceFile(stateHash);
-    if (content.includes(action.code)) {
+    if (
+      mdq(content)
+        .query('code')
+        .meta()
+        .some((m) => m.text.trim() === action.code.trim())
+    ) {
       debugLog('Skipping duplicate action', action.code);
       return;
     }
@@ -187,7 +191,12 @@ export class ExperienceTracker {
     const stateHash = state.getStateHash();
     const { content, data } = this.readExperienceFile(stateHash);
 
-    if (content.includes(body)) {
+    if (
+      mdq(content)
+        .query('section2')
+        .each()
+        .some((s) => s.text().trim() === body.trim())
+    ) {
       debugLog('Skipping duplicate flow body');
       return;
     }
@@ -244,9 +253,14 @@ export class ExperienceTracker {
         });
       })
       .map((experience) => {
-        const lines = experience.content.split('\n');
-        if (lines.length <= maxLines) return experience;
-        return { ...experience, content: lines.slice(0, maxLines).join('\n') };
+        if (experience.content.split('\n').length <= maxLines) return experience;
+        let content = experience.content;
+        while (content.split('\n').length > maxLines) {
+          const sections = mdq(content).query('section2').each();
+          if (sections.length <= 1) break;
+          content = sections[sections.length - 1].replace('');
+        }
+        return { ...experience, content };
       });
   }
 
@@ -393,49 +407,22 @@ export class ExperienceTracker {
 }
 
 function listTocHeadings(content: string): { index: number; level: 2 | 3; title: string }[] {
-  const tokens = marked.lexer(content);
-  const result: { index: number; level: 2 | 3; title: string }[] = [];
-  let index = 0;
-  for (const token of tokens) {
-    if (token.type !== 'heading') continue;
-    const heading = token as Tokens.Heading;
-    if (heading.depth !== 2 && heading.depth !== 3) continue;
-    index++;
-    result.push({ index, level: heading.depth as 2 | 3, title: heading.text });
-  }
-  return result;
+  return mdq(content)
+    .query('heading')
+    .meta()
+    .filter((m) => m.depth === 2 || m.depth === 3)
+    .map((m, i) => ({ index: i + 1, level: m.depth as 2 | 3, title: m.text }));
 }
 
 function extractHeadingSection(content: string, sectionIndex: number): { title: string; body: string } | null {
-  const tokens = marked.lexer(content);
-  const matching: { tokenIdx: number; depth: number; text: string }[] = [];
-
-  for (let i = 0; i < tokens.length; i++) {
-    const token = tokens[i];
-    if (token.type !== 'heading') continue;
-    const heading = token as Tokens.Heading;
-    if (heading.depth !== 2 && heading.depth !== 3) continue;
-    matching.push({ tokenIdx: i, depth: heading.depth, text: heading.text });
-  }
+  const sections = mdq(content).query('section').each();
+  const metas = mdq(content).query('section').meta();
+  const matching = metas.map((meta, i) => ({ meta, section: sections[i] })).filter((entry) => entry.meta.depth === 2 || entry.meta.depth === 3);
 
   if (sectionIndex < 1 || sectionIndex > matching.length) return null;
 
   const target = matching[sectionIndex - 1];
-  let endTokenIdx = tokens.length;
-  for (let j = target.tokenIdx + 1; j < tokens.length; j++) {
-    const token = tokens[j];
-    if (token.type !== 'heading') continue;
-    if ((token as Tokens.Heading).depth <= target.depth) {
-      endTokenIdx = j;
-      break;
-    }
-  }
-
-  const body = tokens
-    .slice(target.tokenIdx, endTokenIdx)
-    .map((t) => (t as any).raw || '')
-    .join('');
-  return { title: target.text, body };
+  return { title: target.meta.text, body: target.section.text() };
 }
 
 function indexToLetters(index: number): string {
@@ -504,23 +491,14 @@ function generateActionContent(title: string, code: string, explanation?: string
 }
 
 function renderAsHowTo(content: string): string {
-  const tokens = marked.lexer(content);
-  let result = '';
-  for (const token of tokens) {
-    if (token.type === 'heading' && (token as Tokens.Heading).depth === 2) {
-      const text = (token as Tokens.Heading).text.trim();
-      if (text.startsWith('FLOW:')) {
-        result += `## HOW to ${text.slice(5).trim()} (multi-step)\n\n`;
-        continue;
-      }
-      if (text.startsWith('ACTION:')) {
-        result += `## HOW to ${text.slice(7).trim()} (single-step)\n\n`;
-        continue;
-      }
-    }
-    result += (token as any).raw || '';
-  }
-  return result;
+  return mdq(content)
+    .query('h2')
+    .replaceEach((heading) => {
+      const text = heading.meta()[0]?.text.trim() || '';
+      if (text.startsWith('FLOW:')) return `## HOW to ${text.slice(5).trim()} (multi-step)\n\n`;
+      if (text.startsWith('ACTION:')) return `## HOW to ${text.slice(7).trim()} (single-step)\n\n`;
+      return heading.text();
+    });
 }
 
 export interface ExperienceFile {

src/knowledge-tracker.ts

@@ -7,6 +7,7 @@ import { ConfigParser } from './config.js';
 import { getCliName } from './utils/cli-name.ts';
 import { createDebug, pluralize, tag } from './utils/logger.js';
 import { loadMarkdownFiles } from './utils/markdown-files.js';
+import { mdq } from './utils/markdown-query.js';
 import { isSecretName, registerSecret } from './utils/secrets.js';
 import { slugify } from './utils/strings.js';
 
@@ -186,7 +187,7 @@ export class KnowledgeTracker {
 
     return this.knowledgeFiles.map((knowledge) => {
       const content = knowledge.content.trim();
-      const firstLine = content.split('\n')[0]?.trim() || '';
+      const firstLine = mdq(content).meta()[0]?.text.split('\n')[0]?.trim() || '';
       return {
         url: knowledge.url,
         firstLine,

src/utils/markdown-query.ts

@@ -408,6 +408,10 @@ export class MarkdownQuery {
   }
 
   replace(content: string): string {
+    return this.replaceEach(() => content);
+  }
+
+  replaceEach(replacer: (match: MarkdownQuery, index: number) => string): string {
     const sorted = [...this.matches].sort((a, b) => a.start - b.start);
 
     const kept: MatchedRange[] = [];
@@ -418,10 +422,11 @@ export class MarkdownQuery {
       lastEnd = range.start + range.length;
     }
 
+    const replacements = kept.map((range, index) => replacer(new MarkdownQuery(this.source, [range]), index));
     let result = this.source;
     for (let i = kept.length - 1; i >= 0; i--) {
       const range = kept[i];
-      result = result.slice(0, range.start) + content + result.slice(range.start + range.length);
+      result = result.slice(0, range.start) + replacements[i] + result.slice(range.start + range.length);
     }
 
     return result;
@@ -459,6 +464,15 @@ export class MarkdownQuery {
   each(): MarkdownQuery[] {
     return this.matches.map((m) => new MarkdownQuery(this.source, [m]));
   }
+
+  meta(): Array<{ type: string; depth: number | null; text: string }> {
+    return this.matches.map((range) => {
+      const token = range.token as any;
+      let depth: number | null = null;
+      if (token.type === 'heading') depth = token.depth;
+      return { type: token.type, depth, text: getTokenText(range.token) };
+    });
+  }
 }
 
 export function mdq(source: string): MarkdownQuery {

tests/unit/experience-tracker.test.ts

@@ -368,6 +368,32 @@ describe('ExperienceTracker', () => {
     });
   });
 
+  describe('getSuccessfulExperience rendering', () => {
+    it('rewrites FLOW/ACTION headings to HOW to phrasing', () => {
+      const state = new ActionResult({ html: '<html></html>', url: 'https://example.com/howto', title: 'HowTo' });
+      const body = ['## FLOW: create a test', '', '* Click create', '', '```js', 'I.click("Create")', '```', '', '---', '', '## ACTION: save the form', '', '```js', 'I.click("Save")', '```', ''].join('\n');
+      experienceTracker.writeExperienceFile(state.getStateHash(), body, { url: '/howto', title: 'HowTo' });
+
+      const combined = experienceTracker.getSuccessfulExperience(state).join('\n');
+
+      expect(combined).toContain('## HOW to create a test (multi-step)');
+      expect(combined).toContain('## HOW to save the form (single-step)');
+      expect(combined).not.toContain('## FLOW:');
+      expect(combined).not.toContain('## ACTION:');
+    });
+
+    it('rewrites and terminates when a title contains the marker', () => {
+      const state = new ActionResult({ html: '<html></html>', url: 'https://example.com/edge', title: 'Edge' });
+      const body = ['## FLOW: undo FLOW: steps', '', '* step', '', '---', ''].join('\n');
+      experienceTracker.writeExperienceFile(state.getStateHash(), body, { url: '/edge', title: 'Edge' });
+
+      const combined = experienceTracker.getSuccessfulExperience(state).join('\n');
+
+      expect(combined).toContain('## HOW to undo FLOW: steps (multi-step)');
+      expect(combined).not.toContain('## FLOW:');
+    });
+  });
+
   describe('table-of-contents lettering', () => {
     it('assigns contiguous fileTags when a zero-section file sorts first', () => {
       const url = '/lettering-page';