diff --git a/App/backend/src/adapters/outbound/memory-client/http-memory-client.ts b/App/backend/src/adapters/outbound/memory-client/http-memory-client.ts index 42e14d9c5..739d6876e 100644 --- a/App/backend/src/adapters/outbound/memory-client/http-memory-client.ts +++ b/App/backend/src/adapters/outbound/memory-client/http-memory-client.ts @@ -224,7 +224,8 @@ export function createHttpMemoryClient( priorityCohortOnly: input.priorityCohortOnly }, signal: input.signal, - timeoutMs: input.timeoutMs + timeoutMs: input.timeoutMs, + maxRetries: 0 }); }, diff --git a/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts b/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts index e4d12a3f7..75cb6e228 100644 --- a/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts +++ b/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts @@ -228,6 +228,19 @@ describe("HttpMemoryClient", () => { }); }); + it("does not replay a worker request after a server failure", async () => { + let calls = 0; + const baseUrl = await startServer(async (_request, response) => { + calls += 1; + response.writeHead(500, { "content-type": "application/json" }); + response.end("{}"); + }); + const client = createHttpMemoryClient({ baseUrl, token: "", timeoutMs: 500, maxRetries: 3 }); + + await expect(client.runWorker({ limit: 20 })).rejects.toThrow("memory layer 5xx"); + expect(calls).toBe(1); + }); + it("retries 5xx responses and succeeds before max retries is exhausted", async () => { let calls = 0; const baseUrl = await startServer(async (_request, response) => { diff --git a/App/backend/src/services/agent-source-service.ts b/App/backend/src/services/agent-source-service.ts index dc3d6a7f1..0ebb48411 100644 --- a/App/backend/src/services/agent-source-service.ts +++ b/App/backend/src/services/agent-source-service.ts @@ -47,9 +47,12 @@ import { export type { ScanProgress } from "../adapters/outbound/agent-source/types.js"; const SCAN_MESSAGE_YIELD_INTERVAL = 100; -const IMPORT_WORKER_BATCH_SIZE = 20; +const IMPORT_WORKER_BATCH_SIZE = 1; const IMPORT_PROCESSING_COHORT_SIZE = 100; -const IMPORT_WORKER_TIMEOUT_MS = 600_000; +// A targeted run leases one job so the timeout never depends on queue ordering. +// One summary can make three content attempts, each with four 180s HTTP attempts +// and backoff; keep a safety margin without replaying the worker request. +const IMPORT_WORKER_TIMEOUT_MS = 2_400_000; const IMPORT_PROGRESS_POLL_INTERVAL_MS = 250; const INITIAL_GLOBAL_MEMORY_LIMIT = 1_000; const INITIAL_ABSENT_SOURCE_MEMORY_LIMIT = 200; @@ -1077,44 +1080,36 @@ async function processPendingImportSummaries( while (pendingMemoryIds.size > 0) { scanOptions.signal?.throwIfAborted(); const targets = [...pendingMemoryIds]; - const result = await options.memoryClient.runWorker({ + const workerOutcome = options.memoryClient.runWorker({ limit: IMPORT_WORKER_BATCH_SIZE, targetMemoryIds: targets, priorityCohortOnly: true, signal: scanOptions.signal, timeoutMs: IMPORT_WORKER_TIMEOUT_MS - }); - - const refreshed = await options.memoryClient.getMemoryProcessingStatus(targets); - const processingByMemoryId = new Map(refreshed.items.map((item) => [item.memoryId, item])); - const activeMemoryIds = new Set(refreshed.items - .filter((item) => item.state === "summary_pending" || item.state === "summarizing" || - item.state === "embedding_pending" || item.state === "embedding") - .map((item) => item.memoryId)); - const previousPending = pendingMemoryIds.size; - for (const memoryId of pendingMemoryIds) { - if (activeMemoryIds.has(memoryId)) continue; - const processing = processingByMemoryId.get(memoryId); - if (!processing) { - failures.push({ memoryId, reason: "Memory processing state is missing" }); - } else if (processing.state === "failed") { - failures.push({ - memoryId, - reason: processing.errorMessage || "Memory processing failed" - }); + }).then((result) => ({ kind: "worker" as const, result })); + let result: Awaited> | undefined; + + while (!result) { + const outcome = await Promise.race([ + workerOutcome, + waitForWorkerProgress(IMPORT_PROGRESS_POLL_INTERVAL_MS, undefined, { signal: scanOptions.signal }) + .then(() => ({ kind: "poll" as const })) + ]); + if (outcome.kind === "worker") result = outcome.result; + const previousPending = pendingMemoryIds.size; + await reconcileImportProcessing(options.memoryClient, pendingMemoryIds, failures); + if (pendingMemoryIds.size < previousPending) lastProgressAt = Date.now(); + emitProgress(scanOptions, { + sourceId: progressSourceId, + phase: "summarize", + current: completedMemoryCount + cohort.length - pendingMemoryIds.size, + total: ownedMemoryIds.length, + message: "Summarizing and indexing latest memories" + }); + if (pendingMemoryIds.size === 0 && !result) { + result = (await workerOutcome).result; } - pendingMemoryIds.delete(memoryId); - } - if (pendingMemoryIds.size < previousPending) { - lastProgressAt = Date.now(); } - emitProgress(scanOptions, { - sourceId: progressSourceId, - phase: "summarize", - current: completedMemoryCount + cohort.length - pendingMemoryIds.size, - total: ownedMemoryIds.length, - message: "Summarizing and indexing latest memories" - }); if (pendingMemoryIds.size === 0) break; if (Date.now() - lastProgressAt >= IMPORT_WORKER_TIMEOUT_MS) { @@ -1130,6 +1125,29 @@ async function processPendingImportSummaries( return failures; } +async function reconcileImportProcessing( + memoryClient: Pick, + pendingMemoryIds: Set, + failures: ProcessingFailure[] +): Promise { + const refreshed = await memoryClient.getMemoryProcessingStatus([...pendingMemoryIds]); + const processingByMemoryId = new Map(refreshed.items.map((item) => [item.memoryId, item])); + const activeMemoryIds = new Set(refreshed.items + .filter((item) => item.state === "summary_pending" || item.state === "summarizing" || + item.state === "embedding_pending" || item.state === "embedding") + .map((item) => item.memoryId)); + for (const memoryId of pendingMemoryIds) { + if (activeMemoryIds.has(memoryId)) continue; + const processing = processingByMemoryId.get(memoryId); + if (!processing) { + failures.push({ memoryId, reason: "Memory processing state is missing" }); + } else if (processing.state === "failed") { + failures.push({ memoryId, reason: processing.errorMessage || "Memory processing failed" }); + } + pendingMemoryIds.delete(memoryId); + } +} + function appendProcessingFailures(result: ScanResult, failures: readonly ProcessingFailure[]): void { result.errors.push(...failures.map((failure) => ({ conversationId: failure.memoryId, diff --git a/App/backend/src/services/skill-distribution-service.ts b/App/backend/src/services/skill-distribution-service.ts index aca5a4b9e..a849bc27e 100644 --- a/App/backend/src/services/skill-distribution-service.ts +++ b/App/backend/src/services/skill-distribution-service.ts @@ -134,7 +134,13 @@ async function findSkillFiles(skillsRoot: string): Promise { if (entry.isFile() && entry.name.toLowerCase() === "skill.md") { files.push(entryPath); } else if (depth < 2 && (entry.isDirectory() || entry.isSymbolicLink())) { - const entryStat = await stat(entryPath); + let entryStat; + try { + entryStat = await stat(entryPath); + } catch (error) { + if (isNodeError(error) && error.code === "ENOENT") continue; + throw error; + } if (entryStat.isDirectory()) await visit(entryPath, depth + 1); } } diff --git a/App/backend/src/services/tests/agent-source-service.test.ts b/App/backend/src/services/tests/agent-source-service.test.ts index ab80615a7..2bbd25cd2 100644 --- a/App/backend/src/services/tests/agent-source-service.test.ts +++ b/App/backend/src/services/tests/agent-source-service.test.ts @@ -4,7 +4,7 @@ import { MANAGED_AGENT_DISCOVERY_PENDING_DATA_PATH } from "@memmy/local-api-cont import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { createSourceRegistry } from "../../adapters/outbound/agent-source/source-registry.js"; import type { ConversationMessage, @@ -610,9 +610,10 @@ describe("agent source service", () => { expect(enqueueCalls).toEqual([["memory-cursor", "memory-custom"]]); expect(workerCalls).toEqual([ expect.objectContaining({ - limit: 20, + limit: 1, targetMemoryIds: ["memory-cursor", "memory-custom"], - priorityCohortOnly: true + priorityCohortOnly: true, + timeoutMs: 2_400_000 }) ]); }); @@ -671,7 +672,7 @@ describe("agent source service", () => { })).resolves.toEqual([]); expect(workerTargets).toEqual([["memory-a", "memory-b"]]); - expect(workerLimits).toEqual([20]); + expect(workerLimits).toEqual([1]); expect(workerPriorityCohorts).toEqual([true]); expect(progress).toEqual([ { current: 0, total: 2 }, @@ -679,6 +680,60 @@ describe("agent source service", () => { ]); }); + it("emits persisted summary progress while the worker call remains pending", async () => { + const baseMemoryClient = createMockMemoryClient(); + let releaseWorker = () => undefined; + let workerSettled = false; + const workerGate = new Promise((resolve) => { + releaseWorker = resolve; + }); + const service = createService({ + memoryClient: { + ...baseMemoryClient, + async enqueueImportSummaries(memoryIds) { + return { enqueued: memoryIds?.length ?? 0, memoryIds: memoryIds ?? [], serverTime: "2026-05-28T10:00:00.000Z" }; + }, + async runWorker(input) { + await workerGate; + workerSettled = true; + return baseMemoryClient.runWorker(input); + }, + async getMemoryProcessingStatus(memoryIds) { + return { + items: memoryIds.map((memoryId) => ({ + memoryId, + state: "ready" as const, + stage: null, + activeJobId: null, + attemptCount: 1, + manualRetryCount: 0, + retryAction: "retry" as const, + errorCode: null, + errorMessage: null, + failedAt: null, + updatedAt: "2026-05-28T10:00:00.000Z" + })), + serverTime: "2026-05-28T10:00:00.000Z" + }; + } + } + }); + const progress: number[] = []; + const processing = service.processImportSummaries(["memory-a"], { + onProgress(event) { + if (event.phase === "summarize") progress.push(event.current); + } + }); + + try { + await vi.waitFor(() => expect(progress).toContain(1), { timeout: 1_000, interval: 25 }); + expect(workerSettled).toBe(false); + } finally { + releaseWorker(); + await processing; + } + }); + it("finishes an empty owned-memory batch without starting the worker", async () => { const baseMemoryClient = createMockMemoryClient(); const enqueued: string[][] = []; diff --git a/App/backend/src/services/tests/skill-distribution-service.test.ts b/App/backend/src/services/tests/skill-distribution-service.test.ts index 2ddaf12e0..b5abf7768 100644 --- a/App/backend/src/services/tests/skill-distribution-service.test.ts +++ b/App/backend/src/services/tests/skill-distribution-service.test.ts @@ -1,5 +1,5 @@ /** Skill distribution service tests. */ -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -40,6 +40,34 @@ describe("skill distribution service", () => { } }); + it("skips broken directory links while scanning valid sibling skills", async () => { + const rootDirectory = mkdtempSync(join(tmpdir(), "memmy-agent-broken-skill-link-")); + try { + mkdirSync(join(rootDirectory, "skills", "valid-skill"), { recursive: true }); + writeFileSync( + join(rootDirectory, "skills", "valid-skill", "SKILL.md"), + "---\nname: valid-skill\n---\nStill discoverable.\n", + "utf8" + ); + symlinkSync( + join(rootDirectory, "missing-skill-target"), + join(rootDirectory, "skills", "broken-skill"), + "junction" + ); + const service = createSkillDistributionService({ + targetRegistry: createSkillTargetRegistry([ + createFakeTarget({ resolveRootDirectory: () => rootDirectory }) + ]) + }); + + await expect(service.listSkills?.("cursor")).resolves.toEqual([ + expect.objectContaining({ sourceSkillId: "valid-skill" }) + ]); + } finally { + rmSync(rootDirectory, { recursive: true, force: true }); + } + }); + it("renders and installs the fixed Memmy skill manifest", async () => { let installed: SkillManifest | undefined; const service = createSkillDistributionService({ diff --git a/App/memmy-agent/src/providers/anthropic-provider.ts b/App/memmy-agent/src/providers/anthropic-provider.ts index 3a18ee512..151ec9f03 100644 --- a/App/memmy-agent/src/providers/anthropic-provider.ts +++ b/App/memmy-agent/src/providers/anthropic-provider.ts @@ -135,6 +135,10 @@ export class AnthropicProvider extends LLMProvider { return model.startsWith("anthropic/") ? model.slice("anthropic/".length) : model; } + static omitsTemperature(model: string): boolean { + return /(?:^|[./])claude-(?:sonnet-5|opus-(?:4-7|5))(?:[-.:@]|$)/i.test(model); + } + static toolResultBlock(msg: Record): Record { const content = msg.content; const block: Record = { @@ -364,7 +368,7 @@ export class AnthropicProvider extends LLMProvider { } const thinkingEnabled = Boolean(reasoningEffort) && String(reasoningEffort).toLowerCase() !== "none"; - const omitTemperature = modelName.includes("opus-4-7"); + const omitTemperature = AnthropicProvider.omitsTemperature(modelName); const kwargs: Record = { model: modelName, messages, @@ -393,6 +397,7 @@ export class AnthropicProvider extends LLMProvider { if (this.extraHeaders) kwargs.extra_headers = this.extraHeaders; if (this.extraBody) Object.assign(kwargs, this.extraBody); + if (omitTemperature) delete kwargs.temperature; return kwargs; } diff --git a/App/memmy-agent/tests/providers/anthropic-thinking.test.ts b/App/memmy-agent/tests/providers/anthropic-thinking.test.ts index 22389d216..29bb12e33 100644 --- a/App/memmy-agent/tests/providers/anthropic-thinking.test.ts +++ b/App/memmy-agent/tests/providers/anthropic-thinking.test.ts @@ -8,7 +8,11 @@ afterEach(() => { describe("Anthropic thinking", () => { function build(reasoningEffort: string | null, overrides: Record = {}): Record { - const provider = new AnthropicProvider({ apiKey: "key", defaultModel: overrides.defaultModel ?? "claude-sonnet-4-6" }); + const provider = new AnthropicProvider({ + apiKey: "key", + defaultModel: overrides.defaultModel ?? "claude-sonnet-4-6", + extraBody: overrides.extraBody ?? null, + }); return provider.buildKwargs({ messages: [{ role: "user", content: "hello" }], tools: null, @@ -21,6 +25,16 @@ describe("Anthropic thinking", () => { }); } + it("does not allow extra body options to reintroduce temperature for fixed-temperature models", () => { + const kwargs = build(null, { + defaultModel: "publishers/anthropic/models/claude-sonnet-5@20260801", + extraBody: { temperature: 0.2, metadata: { user_id: "u1" } }, + }); + + expect(kwargs).not.toHaveProperty("temperature"); + expect(kwargs.metadata).toEqual({ user_id: "u1" }); + }); + it("keeps dedicated thinking blocks separate from visible content", () => { expect(extractReasoning(null, [{ type: "thinking", thinking: "step 1" }], "hello")).toEqual(["step 1", "hello"]); }); @@ -195,4 +209,18 @@ describe("Anthropic thinking", () => { expect(none).not.toHaveProperty("temperature"); expect(none).not.toHaveProperty("thinking"); }); + + it.each([ + "claude-sonnet-5", + "anthropic/claude-sonnet-5-20260801", + "claude-opus-5", + "us.anthropic.claude-opus-5-20260801-v1:0" + ])("omits temperature for fixed-temperature Claude model %s", (model) => { + expect(build(null, { defaultModel: model })).not.toHaveProperty("temperature"); + expect(build("adaptive", { defaultModel: model })).not.toHaveProperty("temperature"); + }); + + it("keeps temperature for Claude models that accept it", () => { + expect(build(null, { defaultModel: "claude-sonnet-4-6" }).temperature).toBe(0.7); + }); }); diff --git a/Memory/package.json b/Memory/package.json index 043209a8a..ede0f6d54 100644 --- a/Memory/package.json +++ b/Memory/package.json @@ -44,6 +44,7 @@ "jsonc-parser": "^3.3.1", "sqlite-vec": "0.1.9", "smol-toml": "1.7.0", + "tiktoken": "^1.0.22", "typescript": "^6.0.3", "yaml": "^2.9.0", "zod": "^4.3.6" diff --git a/Memory/src/config/index.ts b/Memory/src/config/index.ts index c71a79635..60f829928 100644 --- a/Memory/src/config/index.ts +++ b/Memory/src/config/index.ts @@ -301,7 +301,7 @@ export const DEFAULT_MEMMY_CONFIG: MemmyConfig = { enableThinking: false, temperature: 0, maxTokens: MEMORY_SUMMARY_MAX_TOKENS, - timeoutMs: 45_000, + timeoutMs: 180_000, maxRetries: 3, malformedRetries: 1 }, diff --git a/Memory/src/model/embedder.ts b/Memory/src/model/embedder.ts index cf45b6cea..c4f35d47c 100644 --- a/Memory/src/model/embedder.ts +++ b/Memory/src/model/embedder.ts @@ -5,6 +5,10 @@ import type { EmbeddingConfig } from "../config/index.js"; import { createMemoryLogger, memoryErrorFields } from "../logging/logger.js"; import { stableHash } from "../utils/id.js"; import { bearer, postJsonWithRetry, trimTrailingSlash } from "./http.js"; +import { + aggregateOpenAiEmbeddingVectors, + planOpenAiEmbeddingInputs +} from "./openai-embedding-inputs.js"; import { HttpByokTokenUsageRecorder, extractModelTokenUsage } from "./token-usage.js"; import type { Embedder, ModelStatus } from "./types.js"; @@ -216,6 +220,29 @@ class HttpEmbedder implements Embedder { if (!this.config.apiKey && !this.config.endpoint) { throw new Error(`${provider} embedding provider requires apiKey or endpoint`); } + const plan = provider === "openai_compatible" + ? planOpenAiEmbeddingInputs(texts, this.config.model) + : null; + if (!plan) return this.requestOpenAiShape(texts, provider, url, role); + + const chunkVectors: number[][] = []; + for (const batch of plan.batches) { + chunkVectors.push(...await this.requestOpenAiShape( + batch.map((chunk) => chunk.tokens), + provider, + url, + role + )); + } + return aggregateOpenAiEmbeddingVectors(plan, chunkVectors); + } + + private async requestOpenAiShape( + inputs: Array, + provider: string, + url: string, + role: "query" | "document" + ): Promise { const response = await postJsonWithRetry({ actualModelContext: this.config.actualModelContext, provider: this.config.sourceProvider ?? provider, @@ -230,11 +257,11 @@ class HttpEmbedder implements Embedder { maxRetries: this.config.maxRetries, body: { model: this.config.model, - input: texts, + input: inputs, ...(this.config.extraBody ?? {}) } }); - const vectors = validateVectors(provider, response.data?.map((row) => row.embedding), texts.length); + const vectors = validateVectors(provider, response.data?.map((row) => row.embedding), inputs.length); this.recordEmbeddingUsage(response, provider, role); return vectors; } diff --git a/Memory/src/model/http.ts b/Memory/src/model/http.ts index 45da49ec3..6f55ffd5e 100644 --- a/Memory/src/model/http.ts +++ b/Memory/src/model/http.ts @@ -64,7 +64,7 @@ export async function postJsonWithRetry( } } catch (error) { lastError = error; - if (attempt < input.maxRetries) { + if (attempt < input.maxRetries && isRetryableModelRequestError(error)) { const delayMs = Math.min(1_000 * Math.pow(2, attempt), 8_000); logger.warn("request.retry_scheduled", { provider: input.provider, @@ -77,17 +77,18 @@ export async function postJsonWithRetry( ...memoryErrorFields(error) }); await sleep(delayMs); - } else { - logger.error("request.failed", { - provider: input.provider, - operation: input.operation, - model: input.model, - endpoint: safeEndpoint(input.url), - attempt: attempt + 1, - maxAttempts: input.maxRetries + 1, - ...memoryErrorFields(error) - }); + continue; } + logger.error("request.failed", { + provider: input.provider, + operation: input.operation, + model: input.model, + endpoint: safeEndpoint(input.url), + attempt: attempt + 1, + maxAttempts: input.maxRetries + 1, + ...memoryErrorFields(error) + }); + break; } } const normalized = lastError instanceof Error ? lastError : new Error(String(lastError)); @@ -109,6 +110,13 @@ function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } +function isRetryableModelRequestError(error: unknown): boolean { + if (error instanceof ModelHttpError) { + return error.httpStatus === 408 || error.httpStatus === 429 || error.httpStatus >= 500; + } + return error instanceof TypeError || (error instanceof Error && error.name === "AbortError"); +} + function clip(value: string, max: number): string { return value.length <= max ? value : `${value.slice(0, max)}...`; } diff --git a/Memory/src/model/openai-embedding-inputs.ts b/Memory/src/model/openai-embedding-inputs.ts new file mode 100644 index 000000000..7bd9cfc66 --- /dev/null +++ b/Memory/src/model/openai-embedding-inputs.ts @@ -0,0 +1,87 @@ +import { get_encoding } from "tiktoken"; + +const OPENAI_EMBEDDING_INPUT_TOKEN_BUDGET = 7_500; +const OPENAI_EMBEDDING_BATCH_TOKEN_BUDGET = 290_000; + +export interface OpenAiEmbeddingChunk { + originalIndex: number; + tokens: number[]; +} + +export interface OpenAiEmbeddingPlan { + batches: OpenAiEmbeddingChunk[][]; + chunks: OpenAiEmbeddingChunk[]; + originalCount: number; +} + +let encoder: ReturnType | undefined; + +export function planOpenAiEmbeddingInputs(texts: string[], model?: string): OpenAiEmbeddingPlan | null { + if (!isKnownOpenAiEmbeddingModel(model)) return null; + encoder ??= get_encoding("cl100k_base"); + const encoded = texts.map((text) => Array.from(encoder!.encode(text, [], []))); + const totalTokens = encoded.reduce((sum, tokens) => sum + tokens.length, 0); + if (totalTokens <= OPENAI_EMBEDDING_BATCH_TOKEN_BUDGET && + encoded.every((tokens) => tokens.length <= OPENAI_EMBEDDING_INPUT_TOKEN_BUDGET)) return null; + + const chunks = encoded.flatMap((tokens, originalIndex) => { + if (tokens.length === 0) return [{ originalIndex, tokens }]; + const items: OpenAiEmbeddingChunk[] = []; + for (let offset = 0; offset < tokens.length; offset += OPENAI_EMBEDDING_INPUT_TOKEN_BUDGET) { + items.push({ + originalIndex, + tokens: tokens.slice(offset, offset + OPENAI_EMBEDDING_INPUT_TOKEN_BUDGET) + }); + } + return items; + }); + return { + batches: batchChunks(chunks), + chunks, + originalCount: texts.length + }; +} + +export function aggregateOpenAiEmbeddingVectors(plan: OpenAiEmbeddingPlan, vectors: number[][]): number[][] { + if (vectors.length !== plan.chunks.length) { + throw new Error(`openai_compatible returned ${vectors.length} embeddings for ${plan.chunks.length} chunks`); + } + return Array.from({ length: plan.originalCount }, (_item, originalIndex) => { + const entries = plan.chunks + .map((chunk, index) => ({ chunk, vector: vectors[index]! })) + .filter((entry) => entry.chunk.originalIndex === originalIndex); + if (entries.length === 1) return entries[0]!.vector; + const dimensions = entries[0]?.vector.length ?? 0; + if (dimensions === 0 || entries.some((entry) => entry.vector.length !== dimensions)) { + throw new Error("openai_compatible returned incompatible embedding dimensions for chunked input"); + } + const totalWeight = entries.reduce((sum, entry) => sum + Math.max(1, entry.chunk.tokens.length), 0); + const mean = Array.from({ length: dimensions }, (_value, dimension) => + entries.reduce((sum, entry) => + sum + entry.vector[dimension]! * Math.max(1, entry.chunk.tokens.length), 0) / totalWeight + ); + const norm = Math.hypot(...mean); + return norm > 0 ? mean.map((value) => value / norm) : mean; + }); +} + +function isKnownOpenAiEmbeddingModel(model?: string): boolean { + return /(?:^|[/.:])text-embedding-(?:3-(?:small|large)|ada-002)(?:$|[/.:])/i.test(model?.trim() ?? ""); +} + +function batchChunks(chunks: OpenAiEmbeddingChunk[]): OpenAiEmbeddingChunk[][] { + const batches: OpenAiEmbeddingChunk[][] = []; + let current: OpenAiEmbeddingChunk[] = []; + let currentTokens = 0; + for (const chunk of chunks) { + if (current.length > 0 && currentTokens + chunk.tokens.length > OPENAI_EMBEDDING_BATCH_TOKEN_BUDGET) { + batches.push(current); + current = []; + currentTokens = 0; + } + current.push(chunk); + currentTokens += chunk.tokens.length; + } + if (current.length > 0) batches.push(current); + return batches; +} diff --git a/Memory/src/service/worker/job-handlers.ts b/Memory/src/service/worker/job-handlers.ts index e7ead8736..3b736e708 100644 --- a/Memory/src/service/worker/job-handlers.ts +++ b/Memory/src/service/worker/job-handlers.ts @@ -536,6 +536,14 @@ export function classifyProcessingError(error: unknown): { if (/trace payload is missing|memory content is missing|corrupt|malformed memory/.test(normalized)) { return { code: "memory_corrupt", retryAction: "none" }; } + if (error instanceof ModelHttpError && error.httpStatus === 400 && + /maximum.{0,40}(?:input|context).{0,40}(?:length|tokens?)|input.{0,40}(?:too long|token limit)|too many tokens/i.test(message)) { + return { code: "model_input_too_long", retryAction: "none" }; + } + if (error instanceof ModelHttpError && error.httpStatus >= 400 && error.httpStatus < 500 && + error.httpStatus !== 408 && error.httpStatus !== 429) { + return { code: "invalid_model_request", retryAction: "none" }; + } if (/timeout|timed out|network|connect|temporar|rate.?limit|\b429\b|\b5\d\d\b/.test(normalized)) { return { code: "transient_provider_error", retryAction: "retry" }; } diff --git a/Memory/src/service/worker/worker-runner.ts b/Memory/src/service/worker/worker-runner.ts index 1378e3d16..b45cf8d8c 100644 --- a/Memory/src/service/worker/worker-runner.ts +++ b/Memory/src/service/worker/worker-runner.ts @@ -439,12 +439,13 @@ export class WorkerRunner { } } } catch (error) { + const classification = classifyProcessingError(error); + if (classification.code === "model_input_too_long" || classification.code === "invalid_model_request") { + for (const item of batch) results.push(await this.runLeasedEmbeddingItem(item)); + continue; + } for (const item of batch) { - if (!this.deps.embeddingJobs.enqueueEmbeddingRetryAfterFailure(item, error)) { - results.push(this.failLeasedWorkerJob(item.job, error)); - } else { - results.push(this.completeLeasedWorkerJob(item.job)); - } + results.push(this.finishFailedEmbeddingItem(item, error)); } } } @@ -452,6 +453,26 @@ export class WorkerRunner { return results; } + async runLeasedEmbeddingItem(item: PreparedEmbeddingJob): Promise { + try { + const vector = await this.deps.embedder.embedOne(item.text || "(empty)", item.role); + this.deps.embeddingJobs.applyEmbeddingVector(item, vector); + return this.completeLeasedWorkerJob(item.job); + } catch (error) { + return this.finishFailedEmbeddingItem(item, error); + } + } + + finishFailedEmbeddingItem(item: PreparedEmbeddingJob, error: unknown): WorkerJobRunResult { + if (classifyProcessingError(error).retryAction !== "retry") { + return this.failLeasedWorkerJob(item.job, error); + } + if (!this.deps.embeddingJobs.enqueueEmbeddingRetryAfterFailure(item, error)) { + return this.failLeasedWorkerJob(item.job, error); + } + return this.completeLeasedWorkerJob(item.job); + } + completeLeasedWorkerJob(job: EvolutionJobRecord): WorkerJobRunResult { const completed = this.deps.repos.runtime.completeJob(job.id) ?? { ...job, @@ -472,7 +493,14 @@ export class WorkerRunner { const errorMessage = processingStageForJob(job.jobType) ? sanitizeProcessingError(error) : error instanceof Error ? error.message : String(error); - const failedJob = this.deps.repos.runtime.failJob(job.id, errorMessage) ?? { + const stage = processingStageForJob(job.jobType); + const forceDeadLetter = Boolean(stage && classifyProcessingError(error).retryAction !== "retry"); + const failedJob = this.deps.repos.runtime.failJob( + job.id, + errorMessage, + this.deps.nowIso(), + forceDeadLetter + ) ?? { ...job, status: "failed" as const, leasedUntil: null, @@ -524,6 +552,20 @@ export class WorkerRunner { const message = sanitizeProcessingError(error); const terminal = job.status === "dead_letter"; const classification = classifyProcessingError(error); + if (terminal && stage === "embedding" && classification.code === "model_input_too_long") { + this.deps.repos.processing.update(job.targetMemoryId, { + state: "ready_text_only", + stage: null, + activeJobId: null, + attemptCount: job.attempts, + retryAction: "none", + errorCode: classification.code, + errorMessage: message, + failedAt: this.deps.nowIso(), + updatedAt: this.deps.nowIso() + }, ["embedding_pending", "embedding", "failed"]); + return; + } this.deps.repos.processing.update(job.targetMemoryId, { state: terminal ? "failed" : stage === "summary" ? "summary_pending" : "embedding_pending", stage, @@ -583,6 +625,13 @@ export class WorkerRunner { } } } catch (error) { + const classification = classifyProcessingError(error); + if (classification.code === "model_input_too_long" || classification.code === "invalid_model_request") { + for (const item of batch) { + results.push(await this.runClaimedEmbeddingRetryItem(item.retry, item.claim, item.attemptNo)); + } + continue; + } for (const item of batch) { results.push(this.failClaimedEmbeddingRetry(item.retry, item.claim, item.attemptNo, error)); } @@ -597,6 +646,19 @@ export class WorkerRunner { }; } + async runClaimedEmbeddingRetryItem( + retry: EmbeddingRetryRecord, + claim: EmbeddingRetryClaim, + attemptNo: number + ): Promise { + try { + const vector = await this.deps.embedder.embedOne(retry.sourceText || "(empty)", retry.embedRole); + return this.applyEmbeddingRetryVector(retry, claim, vector); + } catch (error) { + return this.failClaimedEmbeddingRetry(retry, claim, attemptNo, error); + } + } + applyEmbeddingRetryVector( retry: EmbeddingRetryRecord, claim: EmbeddingRetryClaim, @@ -653,8 +715,8 @@ export class WorkerRunner { attemptNo: number, error: unknown ): EmbeddingRetryResult { - const message = error instanceof Error ? error.message : String(error); - const terminal = attemptNo >= retry.maxAttempts; + const message = sanitizeProcessingError(error); + const terminal = classifyProcessingError(error).retryAction !== "retry" || attemptNo >= retry.maxAttempts; const updated = terminal ? this.deps.repos.runtime.markEmbeddingRetryFailedClaimed(retry.id, { ...claim, diff --git a/Memory/tests/config.test.ts b/Memory/tests/config.test.ts index 79a82cd5a..414979548 100644 --- a/Memory/tests/config.test.ts +++ b/Memory/tests/config.test.ts @@ -83,11 +83,21 @@ describe("memmy memory config", () => { const { config } = loadMemmyConfig(configPath); expect(config.summary.enableThinking).toBe(false); + expect(config.summary.timeoutMs).toBe(180_000); expect(config.evolution.enableThinking).toBe(true); expect(config.evolution.thinkingBudget).toBeUndefined(); expect(config.evolution.timeoutMs).toBe(180_000); }); + it("allows the summary timeout default to be overridden by environment", () => { + const root = tempRoot(); + const configPath = join(root, "config.yaml"); + writeFileSync(configPath, YAML.stringify({ memmyMemory: {} })); + setEnv("MEMMY_SUMMARY_TIMEOUT_MS", "240000"); + + expect(loadMemmyConfig(configPath).config.summary.timeoutMs).toBe(240_000); + }); + it("expands home-relative sqlite paths from config files", () => { const root = tempRoot(); const configPath = join(root, "config.yaml"); diff --git a/Memory/tests/embedder.test.ts b/Memory/tests/embedder.test.ts index 68c1279fd..1daed46b2 100644 --- a/Memory/tests/embedder.test.ts +++ b/Memory/tests/embedder.test.ts @@ -68,10 +68,104 @@ describe("embedder", () => { expect(new Headers(init.headers).get("x-endpoint-tenant")).toBe("tenant-1"); expect(JSON.parse(String(init.body))).toMatchObject({ model: "embedding-model", + input: ["remember this"], endpoint_option: "exact-endpoint" }); }); + it("chunks oversized OpenAI embedding inputs and preserves original vector order", async () => { + const requestBodies: Array<{ input: number[][] }> = []; + let vectorIndex = 0; + vi.stubGlobal("fetch", vi.fn(async (_url, init) => { + const body = JSON.parse(String(init?.body)) as { input: number[][] }; + requestBodies.push(body); + return new Response(JSON.stringify({ + data: body.input.map(() => { + const embedding = vectorIndex === 0 ? [1, 0] : vectorIndex === 1 ? [0, 1] : [0, 2]; + vectorIndex += 1; + return { embedding }; + }) + }), { status: 200, headers: { "content-type": "application/json" } }); + })); + const embedder = createEmbedder({ + ...DEFAULT_MEMMY_CONFIG.embedding, + provider: "openai_compatible", + endpoint: "https://api.example.test/v1", + model: "text-embedding-3-small", + apiKey: "sk-test", + cache: false, + maxRetries: 0 + }); + + const vectors = await embedder.embed([`<|endoftext|>${" memory".repeat(8_001)}`, "short"]); + const sentInputs = requestBodies.flatMap((body) => body.input); + + expect(sentInputs).toHaveLength(3); + expect(sentInputs.every((input) => Array.isArray(input) && input.length <= 7_500)).toBe(true); + const firstWeight = sentInputs[0]!.length; + const secondWeight = sentInputs[1]!.length; + const totalWeight = firstWeight + secondWeight; + const mean = [firstWeight / totalWeight, secondWeight / totalWeight]; + const norm = Math.hypot(...mean); + expect(vectors[0]?.[0]).toBeCloseTo(mean[0]! / norm, 8); + expect(vectors[0]?.[1]).toBeCloseTo(mean[1]! / norm, 8); + expect(vectors[1]).toEqual([0, 2]); + }); + + it("keeps chunked OpenAI embedding request batches below the aggregate token budget", async () => { + const requestTokenCounts: number[] = []; + vi.stubGlobal("fetch", vi.fn(async (_url, init) => { + const body = JSON.parse(String(init?.body)) as { input: number[][] }; + requestTokenCounts.push(body.input.reduce((sum, input) => sum + input.length, 0)); + return new Response(JSON.stringify({ + data: body.input.map(() => ({ embedding: [1, 0] })) + }), { status: 200, headers: { "content-type": "application/json" } }); + })); + const embedder = createEmbedder({ + ...DEFAULT_MEMMY_CONFIG.embedding, + provider: "openai_compatible", + endpoint: "https://api.example.test/v1", + model: "text-embedding-3-large", + apiKey: "sk-test", + cache: false, + maxRetries: 0 + }); + + await expect(embedder.embedOne(" memory".repeat(300_001))).resolves.toEqual([1, 0]); + + expect(requestTokenCounts.length).toBeGreaterThan(1); + expect(requestTokenCounts.every((count) => count <= 290_000)).toBe(true); + }); + + it("splits OpenAI embedding batches when individually valid inputs exceed the aggregate token budget", async () => { + const requestTokenCounts: number[] = []; + const requestInputCounts: number[] = []; + vi.stubGlobal("fetch", vi.fn(async (_url, init) => { + const body = JSON.parse(String(init?.body)) as { input: number[][] }; + requestTokenCounts.push(body.input.reduce((sum, input) => sum + input.length, 0)); + requestInputCounts.push(body.input.length); + return new Response(JSON.stringify({ + data: body.input.map(() => ({ embedding: [1, 0] })) + }), { status: 200, headers: { "content-type": "application/json" } }); + })); + const embedder = createEmbedder({ + ...DEFAULT_MEMMY_CONFIG.embedding, + provider: "openai_compatible", + endpoint: "https://api.example.test/v1", + model: "text-embedding-3-large", + apiKey: "sk-test", + cache: false, + maxRetries: 0 + }); + + const vectors = await embedder.embed(Array.from({ length: 40 }, () => " memory".repeat(7_400))); + + expect(vectors).toHaveLength(40); + expect(requestTokenCounts.length).toBeGreaterThan(1); + expect(requestTokenCounts.every((count) => count <= 290_000)).toBe(true); + expect(requestInputCounts.reduce((sum, count) => sum + count, 0)).toBe(40); + }); + it("does not ask the local extractor to normalize by default", async () => { transformerMocks.extractor.mockResolvedValue({ data: [3, 4] }); transformerMocks.pipeline.mockResolvedValue(transformerMocks.extractor); diff --git a/Memory/tests/model-http.test.ts b/Memory/tests/model-http.test.ts index 94b5170a9..3a48fffab 100644 --- a/Memory/tests/model-http.test.ts +++ b/Memory/tests/model-http.test.ts @@ -3,6 +3,7 @@ import { ModelHttpError, postJsonWithRetry } from "../src/model/http.js"; import { classifyProcessingError } from "../src/service/worker/job-handlers.js"; afterEach(() => { + vi.useRealTimers(); vi.unstubAllGlobals(); }); @@ -147,6 +148,64 @@ describe("model HTTP responses", () => { 400, "invalid_request", "provider metadata mentions old code 40309" - ))).toEqual({ code: "processing_failed", retryAction: "retry" }); + ))).toEqual({ code: "invalid_model_request", retryAction: "none" }); + }); + + it("does not retry deterministic HTTP 400 failures", async () => { + vi.useFakeTimers(); + const fetchMock = vi.fn(async () => new Response( + JSON.stringify({ error: { code: "invalid_request", message: "invalid embedding input" } }), + { status: 400, headers: { "content-type": "application/json" } } + )); + vi.stubGlobal("fetch", fetchMock); + + const request = postJsonWithRetry({ + provider: "openai_compatible", + url: "https://api.example/v1/embeddings", + body: {}, + timeoutMs: 1_000, + maxRetries: 2 + }); + const rejected = expect(request).rejects.toThrow("invalid embedding input"); + await vi.runAllTimersAsync(); + await rejected; + + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("retries HTTP 429 failures and returns the recovered response", async () => { + vi.useFakeTimers(); + const fetchMock = vi.fn() + .mockResolvedValueOnce(new Response( + JSON.stringify({ error: { message: "rate limited" } }), + { status: 429, headers: { "content-type": "application/json" } } + )) + .mockResolvedValueOnce(new Response( + JSON.stringify({ data: "ok" }), + { status: 200, headers: { "content-type": "application/json" } } + )); + vi.stubGlobal("fetch", fetchMock); + + const request = postJsonWithRetry<{ data: string }>({ + provider: "openai_compatible", + url: "https://api.example/v1/embeddings", + body: {}, + timeoutMs: 1_000, + maxRetries: 2 + }); + await vi.runAllTimersAsync(); + + await expect(request).resolves.toEqual({ data: "ok" }); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it("classifies embedding token-limit failures as terminal", () => { + expect(classifyProcessingError(new ModelHttpError( + "openai_compatible HTTP 400: maximum context length exceeded", + "openai_compatible", + 400, + "context_length_exceeded", + "This model's maximum context length is 8192 tokens, however 12000 tokens were requested" + ))).toEqual({ code: "model_input_too_long", retryAction: "none" }); }); }); diff --git a/Memory/tests/service/embedding/embedding-processing.test.ts b/Memory/tests/service/embedding/embedding-processing.test.ts index 27566d0c4..bd9ce7ee9 100644 --- a/Memory/tests/service/embedding/embedding-processing.test.ts +++ b/Memory/tests/service/embedding/embedding-processing.test.ts @@ -14,6 +14,7 @@ import { embeddingTextForMemory, updateMemoryVectorField } from "../../../src/service/embedding/embedding-pipeline.js"; +import { ModelHttpError } from "../../../src/model/http.js"; import { Repositories } from "../../../src/storage/repositories.js"; import { createBatchReflectionLlm, @@ -172,6 +173,153 @@ describe("MemoryService / embedding / processing", () => { db.close(); }); + it("isolates a deterministic embedding failure and keeps the valid sibling", async () => { + const llmCalls: Array<{ + messages: Array<{ role: string; content: string }>; + options: { operation: string }; + }> = []; + const embedder = createSelectiveFailureEmbedder(); + const { db, service } = createTestService({ + llm: createBatchReflectionLlm(llmCalls, "Imported memory summary."), + embedder + }); + const session = service.openSession({ + namespace: { source: "codex", profileId: "batch-isolation", userId: "user-batch-isolation" } + }); + const good = service.completeTurn("turn-embedding-good", { + sessionId: session.sessionId, + query: "Remember the valid embedding item.", + answer: "VALID_EMBEDDING_ITEM" + }); + const bad = service.completeTurn("turn-embedding-bad", { + sessionId: session.sessionId, + query: "Remember the oversized embedding item.", + answer: "BAD_EMBEDDING_ITEM" + }); + + await service.runWorkerOnce(20, { priorityCohortOnly: true }); + const embeddingRun = await service.runWorkerOnce(20, { priorityCohortOnly: true }); + const repositories = new Repositories(db.db); + + expect(embeddingRun.jobs).toEqual(expect.arrayContaining([ + expect.objectContaining({ targetMemoryId: good.l1MemoryId, status: "succeeded" }), + expect.objectContaining({ targetMemoryId: bad.l1MemoryId, status: "dead_letter" }) + ])); + expect(repositories.processing.get(good.l1MemoryId)?.state).toBe("ready"); + expect(repositories.processing.get(bad.l1MemoryId)).toMatchObject({ + state: "ready_text_only", + stage: null, + retryAction: "none", + errorCode: "model_input_too_long" + }); + db.close(); + }); + + it("terminates a legacy embedding retry after a deterministic provider failure", async () => { + const { db, service } = createTestService({ embedder: createForbiddenEmbedder() }); + const repositories = new Repositories(db.db); + const memory = skillMemory(); + repositories.memories.insert(memory); + const retry = repositories.runtime.enqueueEmbeddingRetry({ + targetKind: "skill", + targetId: memory.id, + vectorField: "vec", + sourceText: embeddingTextForMemory(memory), + embedRole: "query", + now: Date.now() - 1 + }); + + const run = await service.runWorkerOnce(10); + + expect(run.embeddingRetries.items).toEqual([ + expect.objectContaining({ id: retry.id, status: "failed", attempts: 1 }) + ]); + expect(repositories.runtime.getEmbeddingRetry(retry.id)).toMatchObject({ + status: "failed", + attempts: 1, + lastError: "Access to the configured model is forbidden." + }); + db.close(); + }); + + it("does not enqueue a legacy retry for a processing-less deterministic worker failure", async () => { + const { db, service } = createTestService({ embedder: createSelectiveFailureEmbedder() }); + const repositories = new Repositories(db.db); + const memory = skillMemory(undefined, { + id: "skill_deterministic_worker_failure", + content: "BAD_EMBEDDING_ITEM" + }); + repositories.memories.insert(memory); + repositories.runtime.enqueueJob({ + id: "job_skill_deterministic_worker_failure", + jobType: "embedding", + status: "queued", + dedupeKey: `embedding:${memory.id}`, + userId: memory.userId, + targetMemoryId: memory.id, + payload: {}, + attempts: 0, + maxAttempts: 3, + createdAt: memory.createdAt, + updatedAt: memory.updatedAt + }); + + const run = await service.runWorkerOnce(10); + + expect(run.jobs).toEqual([ + expect.objectContaining({ + jobId: "job_skill_deterministic_worker_failure", + status: "dead_letter" + }) + ]); + expect(db.db.prepare( + `SELECT id FROM embedding_retry_queue WHERE target_id = ?` + ).all(memory.id)).toEqual([]); + db.close(); + }); + + it("isolates a deterministic legacy retry failure and keeps the valid sibling", async () => { + const { db, service } = createTestService({ embedder: createSelectiveFailureEmbedder() }); + const repositories = new Repositories(db.db); + const good = skillMemory(undefined, { + id: "skill_legacy_retry_good", + content: "VALID_EMBEDDING_ITEM" + }); + const bad = skillMemory(undefined, { + id: "skill_legacy_retry_bad", + content: "BAD_EMBEDDING_ITEM" + }); + repositories.memories.insert(good); + repositories.memories.insert(bad); + const goodRetry = repositories.runtime.enqueueEmbeddingRetry({ + targetKind: "skill", + targetId: good.id, + vectorField: "vec", + sourceText: embeddingTextForMemory(good), + embedRole: "query", + now: Date.now() - 1 + }); + const badRetry = repositories.runtime.enqueueEmbeddingRetry({ + targetKind: "skill", + targetId: bad.id, + vectorField: "vec", + sourceText: embeddingTextForMemory(bad), + embedRole: "query", + now: Date.now() - 1 + }); + + const run = await service.runWorkerOnce(10); + + expect(run.embeddingRetries.items).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: goodRetry.id, status: "succeeded" }), + expect.objectContaining({ id: badRetry.id, status: "failed", attempts: 1 }) + ])); + expect(repositories.runtime.getEmbeddingRetry(goodRetry.id)?.status).toBe("succeeded"); + expect(repositories.runtime.getEmbeddingRetry(badRetry.id)?.status).toBe("failed"); + expect(retrievalDocumentIsCurrent(repositories.memories.get(good.id)!)).toBe(true); + db.close(); + }); + it("embeds L1 summary together with bounded user and assistant text", async () => { const root = createTestRoot("mindock-memory-dual-embedding-"); const db = new MemoryDb({ @@ -343,17 +491,18 @@ function negativePolicyMemory(title: string, trigger: string): MemoryRow { function skillMemory(short?: { retrievalBlurb: string; triggerContext: string; -}): MemoryRow { +}, override: { id?: string; content?: string } = {}): MemoryRow { const now = "2026-07-24T00:00:00.000Z"; + const content = override.content ?? "# SQLite migration\n\nPROCEDURE_ONLY_SENTINEL"; return { - id: "skill_retrieval_document", + id: override.id ?? "skill_retrieval_document", timeline: now, userId: "skill-retrieval-user", memoryType: "SkillMemory", status: "activated", visibility: "private", memoryKey: "skill:sqlite-migration", - memoryValue: "# SQLite migration\n\nPROCEDURE_ONLY_SENTINEL", + memoryValue: content, tags: ["sqlite", "migration"], info: {}, properties: { @@ -363,7 +512,7 @@ function skillMemory(short?: { skill: { name: "SQLite migration", status: "active", - invocation_guide: "# SQLite migration\n\nPROCEDURE_ONLY_SENTINEL", + invocation_guide: content, ...(short ? { procedure_json: short } : {}) } } @@ -444,3 +593,74 @@ function createFlakyEmbedder(): Embedder { } }; } + +function createSelectiveFailureEmbedder(): Embedder { + const inputTooLong = () => new ModelHttpError( + "openai_compatible HTTP 400: maximum context length exceeded", + "openai_compatible", + 400, + "context_length_exceeded", + "This model's maximum context length is 8192 tokens" + ); + return { + config: { + ...DEFAULT_MEMMY_CONFIG.embedding, + provider: "openai_compatible", + model: "selective-test-embedding" + }, + isRemote() { + return true; + }, + async embed(texts: string[]) { + if (texts.length > 1) throw inputTooLong(); + if (texts[0]?.includes("BAD_EMBEDDING_ITEM")) throw inputTooLong(); + return texts.map((text) => stableTestVector(text)); + }, + async embedOne(text: string) { + if (text.includes("BAD_EMBEDDING_ITEM")) throw inputTooLong(); + return stableTestVector(text); + }, + status() { + return { + provider: "openai_compatible", + model: "selective-test-embedding", + configured: true, + remote: true + }; + } + }; +} + +function createForbiddenEmbedder(): Embedder { + const forbidden = () => new ModelHttpError( + "openai_compatible HTTP 403: forbidden", + "openai_compatible", + 403, + "model_access_denied", + "Access to the configured model is forbidden." + ); + return { + config: { + ...DEFAULT_MEMMY_CONFIG.embedding, + provider: "openai_compatible", + model: "forbidden-test-embedding" + }, + isRemote() { + return true; + }, + async embed() { + throw forbidden(); + }, + async embedOne() { + throw forbidden(); + }, + status() { + return { + provider: "openai_compatible", + model: "forbidden-test-embedding", + configured: true, + remote: true + }; + } + }; +} diff --git a/Memory/tests/service/import/import-processing.test.ts b/Memory/tests/service/import/import-processing.test.ts index def50d42d..8b0f064ad 100644 --- a/Memory/tests/service/import/import-processing.test.ts +++ b/Memory/tests/service/import/import-processing.test.ts @@ -430,7 +430,7 @@ describe("MemoryService / import / processing", () => { db.close(); }); - it("sanitizes provider failures and retries only the failed summary stage", async () => { + it("sanitizes deterministic provider failures and allows manual summary retry", async () => { const root = createTestRoot("mindock-memory-processing-retry-summary-"); const db = new MemoryDb({ path: join(root, "memory.sqlite") }); const llmCalls: Array<{ @@ -459,15 +459,13 @@ describe("MemoryService / import / processing", () => { const namespace = { source: "hermes", profileId: "default", userId: "summary-retry-user" }; const added = addAgentSourceImport(service, namespace, "retry this protected summary", "protected-summary"); - await service.runWorkerOnce(1); - await service.runWorkerOnce(1); await service.runWorkerOnce(1); const failed = service.memoryProcessingStatus([added.id], { namespace }).items[0]; expect(failed).toMatchObject({ state: "failed", stage: "summary", - attemptCount: 3, + attemptCount: 1, retryAction: "open_settings", errorCode: "model_configuration" }); @@ -556,7 +554,7 @@ describe("MemoryService / import / processing", () => { db.close(); }); - it("persists quota details and derives automatic retry state after reopening", async () => { + it("persists terminal quota details and derives queue state after reopening", async () => { const root = createTestRoot("mindock-memory-processing-quota-"); const dbPath = join(root, "memory.sqlite"); const db = new MemoryDb({ path: dbPath }); @@ -586,13 +584,13 @@ describe("MemoryService / import / processing", () => { await service.runWorkerOnce(1); expect(service.memoryProcessingStatus([added.id], { namespace }).items[0]).toMatchObject({ - state: "summary_pending", + state: "failed", stage: "summary", attemptCount: 1, - retryAction: "retry", + retryAction: "open_settings", errorCode: "40309", errorMessage: detail, - autoRetryScheduled: true + autoRetryScheduled: false }); db.close(); @@ -601,12 +599,12 @@ describe("MemoryService / import / processing", () => { expect(repos.processing.get(added.id)).toMatchObject({ errorCode: "40309", errorMessage: detail, - autoRetryScheduled: true + autoRetryScheduled: false }); reopened.db.prepare( `UPDATE evolution_jobs SET status = 'queued' WHERE target_memory_id = ?` ).run(added.id); - expect(repos.processing.get(added.id)?.autoRetryScheduled).toBe(true); + expect(repos.processing.get(added.id)?.autoRetryScheduled).toBe(false); reopened.db.prepare( `UPDATE evolution_jobs SET status = 'leased' WHERE target_memory_id = ?` ).run(added.id); @@ -643,12 +641,12 @@ describe("MemoryService / import / processing", () => { const namespace = { source: "codex", profileId: "default", userId: "html-response-user" }; const added = addAgentSourceImport(service, namespace, "bad model endpoint", "html-response"); - await runWorkerRounds(service, 3, 1); + await service.runWorkerOnce(1); expect(service.memoryProcessingStatus([added.id], { namespace }).items[0]).toMatchObject({ state: "failed", stage: "summary", - attemptCount: 3, + attemptCount: 1, retryAction: "open_settings", errorCode: "model_configuration", errorMessage: expect.stringContaining("HTTP 200") diff --git a/package-lock.json b/package-lock.json index b5f31bd47..4597451a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -724,6 +724,7 @@ "jsonc-parser": "^3.3.1", "smol-toml": "1.7.0", "sqlite-vec": "0.1.9", + "tiktoken": "^1.0.22", "typescript": "^6.0.3", "yaml": "^2.9.0", "zod": "^4.3.6" @@ -14517,6 +14518,12 @@ "integrity": "sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==", "license": "MIT" }, + "node_modules/tiktoken": { + "version": "1.0.22", + "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.22.tgz", + "integrity": "sha512-PKvy1rVF1RibfF3JlXBSP0Jrcw2uq3yXdgcEXtKTYn3QJ/cBRBHDnrJ5jHky+MENZ6DIPwNUGWpkVx+7joCpNA==", + "license": "MIT" + }, "node_modules/tiny-async-pool": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/tiny-async-pool/-/tiny-async-pool-1.3.0.tgz",