Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions apps/memos-local-plugin/core/config/defaults.ts
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,14 @@ export const DEFAULT_CONFIG: ResolvedConfig = {
archiveEta: 0.1,
minEtaForRetrieval: 0.1,
idleArchiveMs: 30 * 24 * 60 * 60 * 1000,
// Persistent per-policy back-off for skill crystallization retries
// (issue #2319). Defaults picked to keep transient failures snappy
// (5 min → 10 min → 20 min ...) while permanent-failure policies get
// quarantined after ~10 h of cumulative exponential wait — a >300×
// reduction on the observed 2,640-failures-in-25-days workload.
crystallizationBackoffBaseMs: 5 * 60 * 1000,
crystallizationBackoffMaxMs: 24 * 60 * 60 * 1000,
crystallizationMaxAttempts: 8,
},
feedback: {
failureThreshold: 3,
Expand Down
17 changes: 17 additions & 0 deletions apps/memos-local-plugin/core/config/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,23 @@ export function resolveConfig(raw: unknown, warnings?: string[], agent?: string)
throw new MemosError("config_invalid", `invalid logging.timezone: ${completed.logging.timezone}`);
}

// Cross-field: baseMs must not exceed maxMs, otherwise the exponential
// back-off's very first `wait = min(base * 2^0, max)` clamps to `max`
// and the retry curve collapses to a flat delay. The individual field
// ranges overlap (base up to 24h, max down to 1min) so the schema step
// above cannot catch this on its own — issue #2319 PR #2325 review.
{
const base = completed.algorithm.skill.crystallizationBackoffBaseMs;
const max = completed.algorithm.skill.crystallizationBackoffMaxMs;
if (base > max) {
throw new MemosError(
"config_invalid",
`algorithm.skill.crystallizationBackoffBaseMs (${base}) must be <= crystallizationBackoffMaxMs (${max})`,
{ base, max },
);
}
}

return Object.freeze(completed) as ResolvedConfig;
}

Expand Down
20 changes: 20 additions & 0 deletions apps/memos-local-plugin/core/config/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,26 @@ const AlgorithmSchema = Type.Object({
60 * 60 * 1000,
365 * 24 * 60 * 60 * 1000,
),
/**
* Initial back-off before retrying a failed crystallization for the
* same policy. Doubles per attempt, capped at `crystallizationBackoffMaxMs`.
* Must not exceed `crystallizationBackoffMaxMs` — cross-checked at
* config-load time (`resolveConfig`) because the individual field ranges
* overlap. See issue #2319 for the retry-storm this replaces.
*/
crystallizationBackoffBaseMs: NumberInRange(5 * 60 * 1000, 1000, 24 * 60 * 60 * 1000),
/**
* Upper bound of the exponential back-off between crystallization retries.
* Must be >= `crystallizationBackoffBaseMs` (see cross-field check in
* `resolveConfig`).
*/
crystallizationBackoffMaxMs: NumberInRange(
24 * 60 * 60 * 1000,
60 * 1000,
30 * 24 * 60 * 60 * 1000,
),
/** Consecutive failures before a policy is quarantined from crystallization. */
crystallizationMaxAttempts: NumberInRange(8, 1, 100),
}, { default: {} }),
feedback: Type.Object({
/** Raise a burst after this many failures of the same tool in-window. */
Expand Down
30 changes: 30 additions & 0 deletions apps/memos-local-plugin/core/skill/ALGORITHMS.md
Original file line number Diff line number Diff line change
Expand Up @@ -299,3 +299,33 @@ A skill run never throws because one policy in a batch misbehaves.
Run-level cancellation is never used — the orchestrator instead
returns `rejected` / `warnings` counts in `RunSkillResult` so the
caller can assert on them.

### Persistent back-off (issue #2319)

`runSkill` also persists per-policy failure state on the `policies`
row (migration 019: `crystallization_attempts`,
`crystallization_backoff_until`, `crystallization_last_attempt_at`,
`crystallization_last_failure_reason`). Every failure branch —
`evidence` (no-evidence), `crystallize` (LLM refusal / parse / disabled),
`verify` (coverage or resonance rejection) — calls
`policies.recordCrystallizationFailure` which:

* increments `attempts`
* schedules `backoffUntil = now + min(baseMs * 2^(attempts-1), maxMs)`
* stamps `lastAttemptAt = now` and the failure reason

At `attempts >= crystallizationMaxAttempts` the policy is quarantined:
`backoffUntil` is cleared, and eligibility keeps skipping until either
the policy is updated (bumping `updatedAt` past `lastAttemptAt`) or a
successful crystallization calls
`policies.resetCrystallizationFailure`.

Eligibility auto-invalidates the back-off when a fresh policy update
arrives: if `policy.updatedAt > lastAttemptAt` the guard is ignored and
the retry proceeds. This is the natural rehab path — new evidence
means the previous failure may no longer apply.

Defaults: `baseMs=5min`, `maxMs=24h`, `maxAttempts=8`. Under the
observed 25-day workload from #2319 this turns 2,640 retries into 8
retries then quarantine — a >300× budget reduction.

5 changes: 4 additions & 1 deletion apps/memos-local-plugin/core/skill/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,10 @@ See `algorithm.skill` in
| `minSupport` | `2` | Min distinct-episode support to crystallize. |
| `minGain` | `0.02` | Min policy gain required (paired with the new shrinkage-anchored gain in `core/memory/l2/gain.ts`). |
| `candidateTrials` | `3` | Trials required to transition out of `candidate`. NOTE: legacy docs called this `probationaryTrials`; the schema field is `candidateTrials`. |
| `cooldownMs` | `60000` | Debounce between runs triggered by the same policy. |
| `cooldownMs` | `60000` | Debounce between runs triggered by the same policy. **Documented for legacy YAML files — currently a no-op; see `crystallizationBackoffBaseMs` for the wired-up mechanism.** |
| `crystallizationBackoffBaseMs` | `300000` | Initial wait before retrying a failed crystallization (doubles per attempt, issue #2319). |
| `crystallizationBackoffMaxMs` | `86400000` | Upper bound of the exponential back-off between crystallization retries. |
| `crystallizationMaxAttempts` | `8` | Consecutive failures before a policy is quarantined from crystallization. |
| `traceCharCap` | `600` | Char cap per evidence trace in the crystallize prompt.|
| `evidenceLimit` | `4` | Max evidence traces per crystallize call. |
| `useLlm` | `true` | Toggle the LLM off (tests / degraded mode). |
Expand Down
70 changes: 69 additions & 1 deletion apps/memos-local-plugin/core/skill/eligibility.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,16 @@
* anchor before they can crystallize into a Skill.
* 5. It is not already represented by a non-archived skill, OR the existing
* skill was built before the policy's latest `updatedAt` (→ rebuild).
* 6. The per-policy crystallization back-off (issue #2319) has not been
* tripped. The check ignores stale state, so a fresh policy update
* (`policy.updatedAt > lastAttemptAt`) always lets a retry through.
*
* The check returns a structured verdict per policy so the orchestrator can
* emit a single rollup event. We never mutate anything here — this module is
* read-only on purpose to make it trivially unit-testable.
*/

import { now as nowMs } from "../time.js";
import type { PolicyRow, SkillRow } from "../types.js";
import type { SkillConfig } from "./types.js";

Expand All @@ -34,6 +38,12 @@ export interface EligibilityInput {
* Callers collect this via `skillsRepo.list()` once per run.
*/
skillsByPolicy: Map<string, SkillRow>;
/**
* Wall-clock time used for back-off comparisons. Optional so existing
* unit tests keep working — production callers thread the same
* `nowMs()` they use elsewhere in `runSkill` for consistent timings.
*/
now?: number;
}

export interface EligibilityResult {
Expand All @@ -49,10 +59,11 @@ export function evaluateEligibility(
const decisions: EligibilityDecision[] = [];
let eligibleCount = 0;
let skippedCount = 0;
const now = input.now ?? nowMs();

for (const policy of input.policies) {
const existing = input.skillsByPolicy.get(policy.id) ?? null;
const decision = decide(policy, existing, config);
const decision = decide(policy, existing, config, now);
decisions.push(decision);
if (decision.action === "skip") skippedCount += 1;
else eligibleCount += 1;
Expand All @@ -65,7 +76,33 @@ function decide(
policy: PolicyRow,
existing: SkillRow | null,
cfg: SkillConfig,
now: number,
): EligibilityDecision {
// Back-off / quarantine gate — check first so a permanently-failing
// policy is skipped before we spend cycles on the other gates. The
// guard is skipped entirely when the state is "stale" (i.e., the
// policy itself has been updated since we last tried it), because a
// fresh update likely reflects new evidence and warrants a retry.
//
// Ordering caveat: because this runs before the `policy.status` gate,
// a policy that is *both* quarantined and inactive/archived reports
// the back-off reason, hiding the underlying status. Operators who
// clear a quarantine (via a policy edit that bumps `updatedAt` past
// `lastAttemptAt`) may then see a status-based skip they did not
// expect. That is intentional — quarantine is a "do not retry" signal
// that must dominate any downstream reasoning — so operators should
// consult `policy.status` independently rather than rely on eligibility
// skip reasons alone.
const backoffSkip = evaluateBackoff(policy, cfg, now);
if (backoffSkip !== null) {
return {
policy,
existingSkill: existing,
action: "skip",
reason: backoffSkip,
};
}

if (policy.status !== "active") {
return {
policy,
Expand Down Expand Up @@ -124,6 +161,37 @@ function decide(
};
}

/**
* Returns a skip reason string when the policy is currently blocked by
* the crystallization back-off / quarantine gate, or `null` when there
* is no active back-off (either it never failed, or the state is stale).
*/
function evaluateBackoff(
policy: PolicyRow,
cfg: SkillConfig,
now: number,
): string | null {
const bo = policy.crystallizationBackoff;
if (!bo || bo.attempts <= 0) return null;

// A policy update after the last attempt invalidates the back-off —
// new evidence has arrived, retry immediately.
if (bo.lastAttemptAt !== null && policy.updatedAt > bo.lastAttemptAt) {
return null;
}

if (bo.attempts >= cfg.crystallizationMaxAttempts) {
const reason = bo.lastFailureReason ?? "unknown";
return `crystallization-quarantined attempts=${bo.attempts} reason=${reason}`;
}

if (bo.backoffUntil !== null && now < bo.backoffUntil) {
return `crystallization-backoff attempts=${bo.attempts} until=${bo.backoffUntil}`;
}

return null;
}

function hasSuccessAnchor(policy: PolicyRow): boolean {
if (policy.skillEligible === false) return false;
const type = policy.experienceType ?? "success_pattern";
Expand Down
87 changes: 85 additions & 2 deletions apps/memos-local-plugin/core/skill/skill.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,10 @@ export async function runSkill(
const warnings: RunSkillResult["warnings"] = [];

const tEligibility = nowMs();
const eligibility = evaluateEligibility({ policies, skillsByPolicy }, config);
const eligibility = evaluateEligibility(
{ policies, skillsByPolicy, now: tEligibility },
config,
);
timings.eligibility = nowMs() - tEligibility;

bus.emit({
Expand Down Expand Up @@ -105,6 +108,7 @@ export async function runSkill(
});
if (evidence.traces.length === 0) {
warnings.push({ policyId: decision.policy.id, reason: "no-evidence" });
recordCrystallizationFailure(decision.policy.id, "no-evidence", deps);
bus.emit({
kind: "skill.failed",
at: nowMs(),
Expand Down Expand Up @@ -146,6 +150,11 @@ export async function runSkill(
policyId: decision.policy.id,
reason: crystResult.skippedReason,
});
recordCrystallizationFailure(
decision.policy.id,
crystResult.skippedReason,
deps,
);
bus.emit({
kind: "skill.failed",
at: nowMs(),
Expand All @@ -166,10 +175,12 @@ export async function runSkill(

if (!verdict.ok) {
rejected += 1;
const verifyReason = `verify:${verdict.reason ?? "verify-failed"}`;
warnings.push({
policyId: decision.policy.id,
reason: verdict.reason ?? "verify-failed",
reason: verifyReason,
});
recordCrystallizationFailure(decision.policy.id, verifyReason, deps);
bus.emit({
kind: "skill.verification.failed",
at: nowMs(),
Expand Down Expand Up @@ -209,6 +220,36 @@ export async function runSkill(
}

repos.skills.upsert(row);
// Success clears any prior back-off — the next natural policy update
// or reward tick can retry without artificial delay. A failure here
// leaves the policy stuck in whatever backoff / quarantine state it
// was in, so we surface it as both a warning and a `skill.failed`
// bus event (stage=persist) — otherwise operators would only see
// `resetCrystallizationFailure` errors in the log stream, and the
// quarantine would look permanent even after a successful upsert.
try {
repos.policies.resetCrystallizationFailure(decision.policy.id);
} catch (err) {
const errMsg = err instanceof Error ? err.message : String(err);
log.warn("skill.backoff.reset_failed", {
policyId: decision.policy.id,
skillId: row.id,
err: errMsg,
});
warnings.push({
policyId: decision.policy.id,
skillId: row.id,
reason: `backoff-reset-failed: ${errMsg}`,
});
bus.emit({
kind: "skill.failed",
at: nowMs(),
policyId: decision.policy.id,
skillId: row.id,
stage: "persist",
reason: `backoff-reset-failed: ${errMsg}`,
});
}
if (!row.vec && deps.embedder) {
repos.embeddingRetryQueue.enqueue({
id: `er_${ids.span()}`,
Expand Down Expand Up @@ -338,6 +379,48 @@ export function applySkillFeedback(

// ─── Helpers ──────────────────────────────────────────────────────────────

/**
* Persist a per-policy failure into the `policies` row so eligibility can
* skip the policy on the next tick (issue #2319). Isolated from `runSkill`
* so a repo-side error can never leak past the orchestrator and abort
* remaining policies in the batch.
*/
function recordCrystallizationFailure(
policyId: PolicyRow["id"],
reason: string,
deps: RunSkillDeps,
): void {
try {
const result = deps.repos.policies.recordCrystallizationFailure(policyId, {
reason,
baseMs: deps.config.crystallizationBackoffBaseMs,
maxMs: deps.config.crystallizationBackoffMaxMs,
maxAttempts: deps.config.crystallizationMaxAttempts,
now: nowMs(),
});
if (result.quarantined) {
deps.log.warn("skill.backoff.quarantined", {
policyId,
attempts: result.attempts,
reason,
});
} else {
deps.log.debug("skill.backoff.recorded", {
policyId,
attempts: result.attempts,
backoffUntil: result.backoffUntil,
reason,
});
}
} catch (err) {
deps.log.warn("skill.backoff.record_failed", {
policyId,
reason,
err: err instanceof Error ? err.message : String(err),
});
}
}

function gatherPolicies(input: RunSkillInput, repos: Repos): PolicyRow[] {
if (input.policyId) {
const single = repos.policies.getById(input.policyId);
Expand Down
20 changes: 20 additions & 0 deletions apps/memos-local-plugin/core/skill/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,26 @@ export interface SkillConfig {
minEtaForRetrieval: number;
/** Archive a low-η active skill after it has not been retrieved for this long. */
idleArchiveMs: number;
/**
* V7 §2.5 addendum — first back-off between crystallization retries after
* a per-policy failure (`crystallize`, `verify`, or `no-evidence`).
* Doubles per attempt via `min(baseMs * 2^(attempts-1), maxMs)`. Failure
* state is stored on the `policies` row so it survives daemon restarts;
* see issue #2319 for the retry-storm the field was designed to prevent.
*/
crystallizationBackoffBaseMs: number;
/**
* Cap for exponential back-off between crystallization retries.
* Must be >= `crystallizationBackoffBaseMs`. Enforced at config-load
* time in `resolveConfig`.
*/
crystallizationBackoffMaxMs: number;
/**
* After this many consecutive failures the policy is quarantined and
* never retried until its `updatedAt` moves past `lastAttemptAt` (i.e.
* new evidence arrived) or the failure state is explicitly cleared.
*/
crystallizationMaxAttempts: number;
}

/**
Expand Down
Loading
Loading