diff --git a/scripts/cross_validate_longmemeval.py b/scripts/cross_validate_longmemeval.py index 40e00f0..c7c08e2 100644 --- a/scripts/cross_validate_longmemeval.py +++ b/scripts/cross_validate_longmemeval.py @@ -96,6 +96,7 @@ def _stratified_cap(questions: list, n: int, field: str) -> list: head-to-head apples-to-apples. """ from collections import defaultdict + groups: dict = defaultdict(list) for q in questions: key = getattr(q, field, None) @@ -122,6 +123,7 @@ def _stratified_cap(questions: list, n: int, field: str) -> list: def _make_full_context_adapter(per_q_vault: Path) -> SMEAdapter: from sme.conditions.full_context import FullContextAdapter + return FullContextAdapter(per_q_vault) @@ -260,7 +262,8 @@ def _make_mempalace_adapter(per_q_vault: Path) -> SMEAdapter: # pragma: no cove def _make_mempalace_daemon_adapter_factory( - *, api_url: Optional[str] = None, + *, + api_url: Optional[str] = None, api_key: Optional[str] = None, kind: Optional[str] = None, ) -> AdapterFactory: # pragma: no cover — network-dependent @@ -368,6 +371,43 @@ def _make_karpathy_compiled_adapter(per_q_vault: Path) -> SMEAdapter: return KarpathyCompiledAdapter(compiled_dir) +def _make_mempalace_server_adapter( + _per_q_vault: Path, +) -> SMEAdapter: # pragma: no cover — network-dependent + """Go MemPalace server (sefodo26) — per-question isolation via full wipe. + + The adapter's ``reset_before_ingest`` wipes the target server before each + question's haystack loads, mirroring the reference benchmark's + reset-store-per-question methodology. Point it at a DISPOSABLE eval + instance (docker-compose default), never a store you care about. + Config: MEMPALACE_SERVER_URL / MEMPALACE_SERVER_API_KEY env vars. + """ + import os as _os + + from sme.adapters.mempalace_server_adapter import MemPalaceServerAdapter + + adapter = MemPalaceServerAdapter( + api_url=_os.environ.get("MEMPALACE_SERVER_URL", "http://localhost:8000"), + api_key=_os.environ.get("MEMPALACE_SERVER_API_KEY", "local-dev-key-change-me"), + reset_before_ingest=True, + read_only=False, + ) + # Factory contract: the run loop only builds + queries, so the factory + # ingests the per-question haystack itself (hindsight pattern). + # source_file = session id (file stem) → the adapter surfaces it as + # Entity.id, which is what the session-level R@K scorer matches. + corpus: list[dict] = [] + for md_file in sorted(_per_q_vault.glob("*.md")): + text = md_file.read_text(encoding="utf-8", errors="replace") + if text.strip(): + corpus.append({"content": text, "source_file": md_file.stem}) + if corpus: + ing = adapter.ingest_corpus(corpus) + if ing.get("errors"): + raise RuntimeError(f"mempalace-server ingest errors: {ing['errors'][:3]}") + return adapter + + _ADAPTER_FACTORIES: dict[str, AdapterFactory] = { "full-context": _make_full_context_adapter, "flat": _make_flat_adapter, @@ -376,11 +416,13 @@ def _make_karpathy_compiled_adapter(per_q_vault: Path) -> SMEAdapter: "omega": _make_omega_adapter, "hindsight": _make_hindsight_adapter, "postgres": _make_postgres_adapter, + "mempalace-server": _make_mempalace_server_adapter, } # --- Scoring helpers -------------------------------------------------------- + def sme_substring_recall(retrieved: str, expected: list[str]) -> tuple[float, list[str]]: """Substring-match the SME way: count how many expected sources appear as substrings of the retrieved context_string. @@ -418,16 +460,17 @@ def judge_label_to_correct(label: str) -> Optional[bool]: # are plain factual correctness, which IS the base template, so they map to a # base-template type. adversarial is handled separately via is_abstention. _LOCOMO_TO_JUDGE_TYPE = { - "single-hop": "single-session-user", # base correctness template - "multi-hop": "multi-session", # base correctness template - "open-domain": "multi-session", # base correctness template - "temporal": "temporal-reasoning", # base + off-by-one-days tolerance + "single-hop": "single-session-user", # base correctness template + "multi-hop": "multi-session", # base correctness template + "open-domain": "multi-session", # base correctness template + "temporal": "temporal-reasoning", # base + off-by-one-days tolerance # adversarial -> abstention is driven by is_abstention, not this map. } # --- Reader pass ------------------------------------------------------------ + def generate_hypothesis( question: str, context_string: str, @@ -450,6 +493,7 @@ def generate_hypothesis( # --- Per-question loop ------------------------------------------------------ + def run_one_question( q: LMEQuestion, *, @@ -474,9 +518,7 @@ def run_one_question( """ # 1. Materialize ONLY this question to a per-question dir out_dir = work_dir / q.question_id - materialize_sme_corpus( - [q], out_dir, max_questions=1, content_rules=content_rules - ) + materialize_sme_corpus([q], out_dir, max_questions=1, content_rules=content_rules) per_q_vault = out_dir / "vault" / q.question_id # 2. Build adapter @@ -574,7 +616,7 @@ def _score_and_judge( # comparison so per-session retrieval recall is real; the raw ids are # still stored verbatim on the record for provenance. def _norm_id(rid: str) -> str: - return rid[len("chunk:"):] if rid.startswith("chunk:") else rid + return rid[len("chunk:") :] if rid.startswith("chunk:") else rid norm_ids = [_norm_id(rid) for rid in retrieved_entity_ids] rank_1 = norm_ids[0] if norm_ids else None @@ -614,10 +656,7 @@ def _norm_id(rid: str) -> str: record["judge"] = None return record - qtype_for_judge = ( - "abstention" if is_abstention - else (judge_question_type or question_type) - ) + qtype_for_judge = "abstention" if is_abstention else (judge_question_type or question_type) if skip_reader: # Hand the judge the raw retrieval (option (a) in the planning @@ -625,7 +664,8 @@ def _norm_id(rid: str) -> str: hypothesis = ctx[:8000] # cap to keep judge prompt manageable else: hypothesis = generate_hypothesis( - question, ctx, + question, + ctx, reader_model=reader_model, client=reader_client, ) @@ -645,6 +685,7 @@ def _norm_id(rid: str) -> str: # --- LoCoMo per-sample loop ------------------------------------------------- + def run_locomo_questions( questions: list[Any], # list[LoCoMoQuestion] *, @@ -731,9 +772,7 @@ def run_locomo_questions( reader_client=reader_client, judge_client=judge_client, capture_context=capture_context, - judge_question_type=_LOCOMO_TO_JUDGE_TYPE.get( - q.question_type - ), + judge_question_type=_LOCOMO_TO_JUDGE_TYPE.get(q.question_type), extra_fields={ "sample_id": q.sample_id, "locomo_category": q.category, @@ -743,7 +782,10 @@ def run_locomo_questions( records.append(rec) log.info( "[%s] %s (%s / %s)", - sample_id, q.question_id, q.question_type, q.sme_category, + sample_id, + q.question_id, + q.question_type, + q.sme_category, ) finally: try: @@ -858,7 +900,10 @@ def run_beam_questions( records.append(rec) log.info( "[%s] %s (%s / %s)", - conv_id, q.question_id, q.ability_type, q.sme_category, + conv_id, + q.question_id, + q.ability_type, + q.sme_category, ) finally: try: @@ -870,6 +915,7 @@ def run_beam_questions( # --- Aggregation ------------------------------------------------------------ + def _empty_category_slot() -> dict[str, Any]: return { "n": 0, @@ -940,32 +986,34 @@ def aggregate(records: list[dict]) -> dict: _accumulate_usage(total_usage, judge.get("usage") or {}) if _is_disagreement(r["sme_recall"], judge_label_to_correct(label)): - disagreements.append({ - "question_id": r["question_id"], - "sme_category": cat, - "sme_recall": r["sme_recall"], - "judge_label": label, - "judge_rationale": judge.get("rationale", ""), - }) + disagreements.append( + { + "question_id": r["question_id"], + "sme_category": cat, + "sme_recall": r["sme_recall"], + "judge_label": label, + "judge_rationale": judge.get("rationale", ""), + } + ) per_cat = {} for cat, slot in sorted(by_cat.items()): n = slot["n"] judged = ( - slot["judge_correct"] + slot["judge_incorrect"] - + slot["judge_partial"] + slot["judge_abstain"] + slot["judge_correct"] + + slot["judge_incorrect"] + + slot["judge_partial"] + + slot["judge_abstain"] ) sme_recall_mean = slot["sme_recall_sum"] / n if n else 0.0 judge_correct_rate = ( - (slot["judge_correct"] + slot["judge_abstain"]) / judged - if judged else None + (slot["judge_correct"] + slot["judge_abstain"]) / judged if judged else None ) per_cat[cat] = { "n": n, "sme_recall_mean": round(sme_recall_mean, 4), "judge_correct_rate": ( - round(judge_correct_rate, 4) - if judge_correct_rate is not None else None + round(judge_correct_rate, 4) if judge_correct_rate is not None else None ), "judge_label_counts": { "CORRECT": slot["judge_correct"], @@ -989,14 +1037,16 @@ def aggregate(records: list[dict]) -> dict: for r in records: if r.get("hit_at_1") is False: qtype = r.get("question_type", "unknown") - r1_misses.append({ - "question_id": r["question_id"], - "question_type": qtype, - "retrieved_rank_1": r.get("retrieved_rank_1"), - "expected_sources": r.get("expected_sources", []), - "hit_at_5": r.get("hit_at_5"), - "hit_at_10": r.get("hit_at_10"), - }) + r1_misses.append( + { + "question_id": r["question_id"], + "question_type": qtype, + "retrieved_rank_1": r.get("retrieved_rank_1"), + "expected_sources": r.get("expected_sources", []), + "hit_at_5": r.get("hit_at_5"), + "hit_at_10": r.get("hit_at_10"), + } + ) r1_miss_by_type[qtype] = r1_miss_by_type.get(qtype, 0) + 1 return { @@ -1019,6 +1069,7 @@ def aggregate(records: list[dict]) -> dict: # --- Main ------------------------------------------------------------------- + def build_arg_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser( description=( @@ -1027,88 +1078,128 @@ def build_arg_parser() -> argparse.ArgumentParser: "report per-category disagreement." ), ) - p.add_argument("--dataset", required=True, type=Path, - help="Path to the dataset JSON. For --corpus longmemeval: " - "longmemeval_oracle.json (or _s / _m). For --corpus " - "locomo: locomo10.json. For --corpus beam: a cached " - "per-bucket BEAM split (e.g. beam_100K.json) or the " - "committed sample (sme/corpora/beam/sample/" - "beam_100K_sample.json).") - p.add_argument("--corpus", default="longmemeval", - choices=["longmemeval", "locomo", "beam"], - help="Dataset shape. 'longmemeval' (default) = one " - "haystack per question. 'locomo' = one conversation " - "per sample shared by that sample's questions; " - "questions are grouped by sample_id and ingested " - "per sample, and adversarial (cat-5) items are judged " - "abstention-aware. 'beam' = one (very long) " - "conversation per conversation_id shared by its 20 " - "probing questions; grouped + ingested per " - "conversation, graded at a token --bucket, and " - "abstention items judged abstention-aware.") - p.add_argument("--bucket", default="100K", - choices=["100K", "500K", "1M", "10M"], - help="BEAM token bucket (only used with --corpus beam). " - "The same conversation at a different bucket is a " - "different retrieval problem, so the bucket is " - "recorded on every record.") - p.add_argument("--beam-n-results", type=int, default=5, - help="BEAM per-query retrieval depth (top-K sessions; " - "only used with --corpus beam). Default 5 returns " - "the whole conversation at the 100K bucket (3-5 " - "sessions). At 500K/1M (~10 sessions, >500K/>1M " - "tokens) set this low (2-3) so the retrieved top-K " - "fits the reader window.") - p.add_argument("--adapter", required=True, - choices=sorted(_ADAPTER_FACTORIES), - help="SME adapter to run.") - p.add_argument("--max-questions", type=int, default=None, - help="Smoke-test cap on number of questions. NOTE: the " - "oracle/S corpora are question_type-sorted, so a bare " - "cap is a single-category slice — pair with " - "--stratify-by or --shuffle (#122). longmemeval only.") - p.add_argument("--shuffle", type=int, default=None, metavar="SEED", - help="Deterministically shuffle questions (with SEED) before " - "the --max-questions cap, so the cap is not a single-" - "category slice. longmemeval corpus only.") - p.add_argument("--stratify-by", default=None, metavar="FIELD", - help="Stratify the --max-questions cap across this question " - "field (e.g. question_type) — even round-robin per " - "category for representative coverage (#122). Matches the " - "mempalace-daemon strat150 baseline. longmemeval only.") - p.add_argument("--reader-model", default="gpt-4o-mini", - help="Model used to turn retrieved context into an " - "answer the judge can score. Default kept as " - "gpt-4o-mini for back-compat with prior runs; " - "the `sme-eval longmemeval` CLI defaults to " - "gpt-4.1-mini per issue #17.") - p.add_argument("--judge-model", default="gpt-4o-2024-08-06", - help="LongMemEval judge model.") - p.add_argument("--skip-judge", action="store_true", - help="Run SME-only — no reader, no judge, no API key " - "required.") - p.add_argument("--skip-reader", action="store_true", - help="Skip the reader pass; feed raw retrieval to the " - "judge (diagnostic mode).") - p.add_argument("--out", type=Path, default=None, - help="Where to write the report JSON.") - p.add_argument("--work-dir", type=Path, default=None, - help="Where per-question vaults are materialized " - "(default: tmpdir).") - p.add_argument("--content-rules", default="sme-rich", - choices=["sme-rich", "upstream-exact"], - help="Session rendering rules. 'sme-rich' (default) = " - "frontmatter + role headers + user + assistant turns. " - "'upstream-exact' = user turns only, no metadata — " - "matches upstream protocol per #54 / #51.") + p.add_argument( + "--dataset", + required=True, + type=Path, + help="Path to the dataset JSON. For --corpus longmemeval: " + "longmemeval_oracle.json (or _s / _m). For --corpus " + "locomo: locomo10.json. For --corpus beam: a cached " + "per-bucket BEAM split (e.g. beam_100K.json) or the " + "committed sample (sme/corpora/beam/sample/" + "beam_100K_sample.json).", + ) + p.add_argument( + "--corpus", + default="longmemeval", + choices=["longmemeval", "locomo", "beam"], + help="Dataset shape. 'longmemeval' (default) = one " + "haystack per question. 'locomo' = one conversation " + "per sample shared by that sample's questions; " + "questions are grouped by sample_id and ingested " + "per sample, and adversarial (cat-5) items are judged " + "abstention-aware. 'beam' = one (very long) " + "conversation per conversation_id shared by its 20 " + "probing questions; grouped + ingested per " + "conversation, graded at a token --bucket, and " + "abstention items judged abstention-aware.", + ) + p.add_argument( + "--bucket", + default="100K", + choices=["100K", "500K", "1M", "10M"], + help="BEAM token bucket (only used with --corpus beam). " + "The same conversation at a different bucket is a " + "different retrieval problem, so the bucket is " + "recorded on every record.", + ) + p.add_argument( + "--beam-n-results", + type=int, + default=5, + help="BEAM per-query retrieval depth (top-K sessions; " + "only used with --corpus beam). Default 5 returns " + "the whole conversation at the 100K bucket (3-5 " + "sessions). At 500K/1M (~10 sessions, >500K/>1M " + "tokens) set this low (2-3) so the retrieved top-K " + "fits the reader window.", + ) + p.add_argument( + "--adapter", required=True, choices=sorted(_ADAPTER_FACTORIES), help="SME adapter to run." + ) + p.add_argument( + "--max-questions", + type=int, + default=None, + help="Smoke-test cap on number of questions. NOTE: the " + "oracle/S corpora are question_type-sorted, so a bare " + "cap is a single-category slice — pair with " + "--stratify-by or --shuffle (#122). longmemeval only.", + ) + p.add_argument( + "--shuffle", + type=int, + default=None, + metavar="SEED", + help="Deterministically shuffle questions (with SEED) before " + "the --max-questions cap, so the cap is not a single-" + "category slice. longmemeval corpus only.", + ) + p.add_argument( + "--stratify-by", + default=None, + metavar="FIELD", + help="Stratify the --max-questions cap across this question " + "field (e.g. question_type) — even round-robin per " + "category for representative coverage (#122). Matches the " + "mempalace-daemon strat150 baseline. longmemeval only.", + ) + p.add_argument( + "--reader-model", + default="gpt-4o-mini", + help="Model used to turn retrieved context into an " + "answer the judge can score. Default kept as " + "gpt-4o-mini for back-compat with prior runs; " + "the `sme-eval longmemeval` CLI defaults to " + "gpt-4.1-mini per issue #17.", + ) + p.add_argument("--judge-model", default="gpt-4o-2024-08-06", help="LongMemEval judge model.") + p.add_argument( + "--skip-judge", + action="store_true", + help="Run SME-only — no reader, no judge, no API key required.", + ) + p.add_argument( + "--skip-reader", + action="store_true", + help="Skip the reader pass; feed raw retrieval to the judge (diagnostic mode).", + ) + p.add_argument("--out", type=Path, default=None, help="Where to write the report JSON.") + p.add_argument( + "--work-dir", + type=Path, + default=None, + help="Where per-question vaults are materialized (default: tmpdir).", + ) + p.add_argument( + "--content-rules", + default="sme-rich", + choices=["sme-rich", "upstream-exact"], + help="Session rendering rules. 'sme-rich' (default) = " + "frontmatter + role headers + user + assistant turns. " + "'upstream-exact' = user turns only, no metadata — " + "matches upstream protocol per #54 / #51.", + ) p.add_argument("-v", "--verbose", action="store_true") return p -def run(args: argparse.Namespace, - *, - reader_client: Optional[Any] = None, - judge_client: Optional[Any] = None) -> dict: +def run( + args: argparse.Namespace, + *, + reader_client: Optional[Any] = None, + judge_client: Optional[Any] = None, +) -> dict: """Programmatic entry point — used by tests.""" factory = _ADAPTER_FACTORIES[args.adapter] @@ -1125,6 +1216,7 @@ def run(args: argparse.Namespace, try: if corpus == "locomo": from sme.corpora.locomo import load_questions as load_locomo + records = run_locomo_questions( list(load_locomo(args.dataset)), adapter_factory=factory, @@ -1139,6 +1231,7 @@ def run(args: argparse.Namespace, ) elif corpus == "beam": from sme.corpora.beam import load_questions as load_beam + bucket = getattr(args, "bucket", "100K") records = run_beam_questions( list(load_beam(args.dataset, bucket=bucket)), @@ -1165,18 +1258,20 @@ def run(args: argparse.Namespace, stratify_by = getattr(args, "stratify_by", None) if shuffle_seed is not None: import random + random.Random(shuffle_seed).shuffle(questions) if args.max_questions is not None: if stratify_by: - questions = _stratified_cap( - questions, args.max_questions, stratify_by - ) + questions = _stratified_cap(questions, args.max_questions, stratify_by) else: questions = questions[: args.max_questions] for i, q in enumerate(questions): log.info( "[%d] %s (%s / %s)", - i, q.question_id, q.question_type, q.sme_category, + i, + q.question_id, + q.question_type, + q.sme_category, ) rec = run_one_question( q, @@ -1201,11 +1296,9 @@ def run(args: argparse.Namespace, "dataset": str(args.dataset), "corpus": corpus, "bucket": (getattr(args, "bucket", None) if corpus == "beam" else None), - "beam_n_results": (getattr(args, "beam_n_results", 5) - if corpus == "beam" else None), + "beam_n_results": (getattr(args, "beam_n_results", 5) if corpus == "beam" else None), "adapter": args.adapter, - "reader_model": (None if args.skip_reader or args.skip_judge - else args.reader_model), + "reader_model": (None if args.skip_reader or args.skip_judge else args.reader_model), "judge_model": (None if args.skip_judge else args.judge_model), "skip_judge": bool(args.skip_judge), "skip_reader": bool(args.skip_reader), diff --git a/sme/adapters/mempalace_server_adapter.py b/sme/adapters/mempalace_server_adapter.py index ba9fd59..90ac4aa 100644 --- a/sme/adapters/mempalace_server_adapter.py +++ b/sme/adapters/mempalace_server_adapter.py @@ -121,21 +121,9 @@ def __init__( kg_entity_limit: int = DEFAULT_KG_ENTITY_LIMIT, read_only: bool = False, ) -> None: - resolved_url = ( - api_url - or os.environ.get("MEMPALACE_SERVER_URL") - or DEFAULT_API_URL - ) - resolved_key = ( - api_key - or os.environ.get("MEMPALACE_SERVER_API_KEY") - or DEFAULT_API_KEY - ) - resolved_tenant = ( - tenant - or os.environ.get("MEMPALACE_SERVER_TENANT") - or DEFAULT_TENANT - ) + resolved_url = api_url or os.environ.get("MEMPALACE_SERVER_URL") or DEFAULT_API_URL + resolved_key = api_key or os.environ.get("MEMPALACE_SERVER_API_KEY") or DEFAULT_API_KEY + resolved_tenant = tenant or os.environ.get("MEMPALACE_SERVER_TENANT") or DEFAULT_TENANT self.api_url = resolved_url.rstrip("/") self.api_key = resolved_key self.tenant = resolved_tenant @@ -178,12 +166,7 @@ def ingest_corpus(self, corpus: list[dict]) -> dict: created = 0 existed = 0 for row in corpus: - content = ( - row.get("document") - or row.get("content") - or row.get("text") - or "" - ) + content = row.get("document") or row.get("content") or row.get("text") or "" if not str(content).strip(): continue meta = row.get("metadata") or {} @@ -293,12 +276,17 @@ def query( raw_id = hit.get("drawer_id") or hit.get("id") drawer_id = str(raw_id) if raw_id is not None else f"drawer_hit:{i}" label = source_label or drawer_id - context_parts.append( - f"[{i + 1}] [{wing_name}/{room_name}] {label}\n{content}" - ) + # Retrieval scorers (LongMemEval cross-validation) match + # ``Entity.id`` against the corpus/session ids supplied at + # ingest, which this adapter stores in ``source_file``. The + # server's own drawer id is a content hash and can never match, + # so surface the source stem as the entity id and keep the + # drawer hash in properties for provenance. + entity_id = Path(source_file).stem if source_file else drawer_id + context_parts.append(f"[{i + 1}] [{wing_name}/{room_name}] {label}\n{content}") retrieved.append( Entity( - id=drawer_id, + id=entity_id, name=label, entity_type=f"drawer:{room_name}", properties={ @@ -308,6 +296,7 @@ def query( "similarity": hit.get("similarity"), "distance": hit.get("distance"), "source_file": source_file, + "drawer_id": drawer_id, "rank": i + 1, }, ) @@ -540,8 +529,11 @@ def get_ontology_source(self) -> dict: { "kind": "hall_vocabulary", "values": [ - "facts", "events", "discoveries", - "preferences", "advice", + "facts", + "events", + "discoveries", + "preferences", + "advice", ], }, ], @@ -594,9 +586,7 @@ def _probe_mcp_health(self) -> ProbeResult: latency_ms = (time.perf_counter() - t0) * 1000.0 if isinstance(body, QueryResult): return ProbeResult(success=False, latency_ms=latency_ms, error=body.error) - return ProbeResult( - success=True, latency_ms=latency_ms, output=json.dumps(body)[:200] - ) + return ProbeResult(success=True, latency_ms=latency_ms, output=json.dumps(body)[:200]) def _probe_rest_search(self) -> ProbeResult: """A minimal authenticated search — read-only, mutates nothing.""" @@ -609,8 +599,9 @@ def _probe_rest_search(self) -> ProbeResult: # --- HTTP plumbing ------------------------------------------------ - def _http(self, method: str, path: str, payload: Optional[dict] = None, - *, auth: bool = True) -> Any: + def _http( + self, method: str, path: str, payload: Optional[dict] = None, *, auth: bool = True + ) -> Any: """Issue an HTTP request, returning parsed JSON on 2xx or a ``QueryResult`` carrying an ``error`` string on any failure. @@ -640,13 +631,9 @@ def _http(self, method: str, path: str, payload: Optional[dict] = None, err = f"HTTP {e.code}: {detail}" return QueryResult(answer="", context_string="", error=err) except (urllib.error.URLError, TimeoutError, OSError) as e: - return QueryResult( - answer="", context_string="", error=f"CONNECTION: {e}" - ) + return QueryResult(answer="", context_string="", error=f"CONNECTION: {e}") except Exception as e: # pragma: no cover - defensive - return QueryResult( - answer="", context_string="", error=f"INTERNAL: {e}" - ) + return QueryResult(answer="", context_string="", error=f"INTERNAL: {e}") def _mcp_call(self, tool: str, arguments: dict) -> Optional[dict]: """Call an MCP tool via ``POST /mp/mcp`` (stateless JSON-RPC — no diff --git a/sme/cli.py b/sme/cli.py index 181c03d..b7a8488 100644 --- a/sme/cli.py +++ b/sme/cli.py @@ -168,87 +168,160 @@ def _agentmemory_loader() -> type[SMEAdapter]: _AdapterSpec( aliases=("ladybugdb", "ladybug"), loader=_ladybugdb_loader, - accepts=frozenset({ - "db_path", "read_only", "buffer_pool_size", - "include_node_tables", "include_edge_tables", "auto_discover", - "skip_infrastructure", "api_url", "default_query_mode", - "api_timeout", - }), + accepts=frozenset( + { + "db_path", + "read_only", + "buffer_pool_size", + "include_node_tables", + "include_edge_tables", + "auto_discover", + "skip_infrastructure", + "api_url", + "default_query_mode", + "api_timeout", + } + ), ), _AdapterSpec( aliases=("mempalace-daemon", "mempalace_daemon"), loader=_mempalace_daemon_loader, - accepts=frozenset({ - "api_url", "api_key", "env_file", "kind", "api_timeout", - "prefer_graph_endpoint", "read_only", - # #140 — age-fused endpoint selection + candidate-pool strategy - "search_endpoint", "candidate_strategy", - # #147 — real-KG structural measurement (Cat 4/5/8) - "graph_kg_only", "graph_limit", - }), + accepts=frozenset( + { + "api_url", + "api_key", + "env_file", + "kind", + "api_timeout", + "prefer_graph_endpoint", + "read_only", + # #140 — age-fused endpoint selection + candidate-pool strategy + "search_endpoint", + "candidate_strategy", + # #147 — real-KG structural measurement (Cat 4/5/8) + "graph_kg_only", + "graph_limit", + } + ), ), _AdapterSpec( aliases=("mempalace-server", "mempalace_server", "mp-server"), loader=_mempalace_server_loader, - accepts=frozenset({ - "api_url", "api_key", "tenant", "wing", "room", "n_results", - "max_distance", "api_timeout", "reset_before_ingest", - "kg_entity_limit", "read_only", - }), + accepts=frozenset( + { + "api_url", + "api_key", + "tenant", + "wing", + "room", + "n_results", + "max_distance", + "api_timeout", + "reset_before_ingest", + "kg_entity_limit", + "read_only", + } + ), ), _AdapterSpec( aliases=("engram", "engram-ts"), loader=_engram_loader, - accepts=frozenset({ - "engram_path", "node_bin", "db_path", "n_results", "include_graph", - "importance", "reset_before_ingest", "startup_timeout", - "call_timeout", "read_only", - }), + accepts=frozenset( + { + "engram_path", + "node_bin", + "db_path", + "n_results", + "include_graph", + "importance", + "reset_before_ingest", + "startup_timeout", + "call_timeout", + "read_only", + } + ), ), _AdapterSpec( aliases=("rlm",), loader=_rlm_loader, - accepts=frozenset({ - "api_url", "api_key", "backend", "backend_kwargs", - "environment", "verbose", "kind", "timeout_s", - }), + accepts=frozenset( + { + "api_url", + "api_key", + "backend", + "backend_kwargs", + "environment", + "verbose", + "kind", + "timeout_s", + } + ), ), _AdapterSpec( aliases=("longhand",), loader=_longhand_loader, - accepts=frozenset({ - "bin_path", "home_dir", "n_results", "timeout_s", "project", - }), + accepts=frozenset( + { + "bin_path", + "home_dir", + "n_results", + "timeout_s", + "project", + } + ), ), _AdapterSpec( aliases=("postgres-ingest", "postgres_ingest", "postgres"), loader=_postgres_ingest_loader, - accepts=frozenset({ - "dsn", "table_name", "n_results", "mempalace_path", "read_only", - }), + accepts=frozenset( + { + "dsn", + "table_name", + "n_results", + "mempalace_path", + "read_only", + } + ), ), _AdapterSpec( aliases=("familiar",), loader=_familiar_loader, - accepts=frozenset({ - "base_url", "timeout_s", "mock_inference", "opener", - }), + accepts=frozenset( + { + "base_url", + "timeout_s", + "mock_inference", + "opener", + } + ), rename={"api_url": "base_url"}, ), _AdapterSpec( aliases=("mempalace",), loader=_mempalace_loader, - accepts=frozenset({ - "db_path", "read_only", "kg_path", "collection_name", - "include_kg", "include_drawers", "max_drawer_nodes", - }), + accepts=frozenset( + { + "db_path", + "read_only", + "kg_path", + "collection_name", + "include_kg", + "include_drawers", + "max_drawer_nodes", + } + ), ), _AdapterSpec( aliases=("flat", "flat_baseline"), loader=_flat_loader, - accepts=frozenset({ - "db_path", "read_only", "collection_name", "n_results", - }), + accepts=frozenset( + { + "db_path", + "read_only", + "collection_name", + "n_results", + } + ), ), _AdapterSpec( aliases=("full-context", "full_context"), @@ -259,26 +332,44 @@ def _agentmemory_loader() -> type[SMEAdapter]: _AdapterSpec( aliases=("omega",), loader=_omega_loader, - accepts=frozenset({ - "omega_home", "db_path", "default_memory_type", "n_results", - "read_only", - }), + accepts=frozenset( + { + "omega_home", + "db_path", + "default_memory_type", + "n_results", + "read_only", + } + ), ), _AdapterSpec( aliases=("hindsight",), loader=_hindsight_loader, - accepts=frozenset({ - "base_url", "bank_id", "api_key", "n_results", "use_reflect", - "api_timeout", "read_only", - }), + accepts=frozenset( + { + "base_url", + "bank_id", + "api_key", + "n_results", + "use_reflect", + "api_timeout", + "read_only", + } + ), rename={"api_url": "base_url"}, ), _AdapterSpec( aliases=("mem0", "mem0_oss"), loader=_mem0_loader, - accepts=frozenset({ - "config", "user_id", "n_results", "memory", "read_only", - }), + accepts=frozenset( + { + "config", + "user_id", + "n_results", + "memory", + "read_only", + } + ), ), _AdapterSpec( aliases=("karpathy-compiled", "karpathy_compiled"), @@ -305,18 +396,30 @@ def _agentmemory_loader() -> type[SMEAdapter]: _AdapterSpec( aliases=("ai-memory", "ai_memory", "aimemory"), loader=_ai_memory_loader, - accepts=frozenset({ - "api_url", "namespace", "tier", "n_results", "api_timeout", - "read_only", - }), + accepts=frozenset( + { + "api_url", + "namespace", + "tier", + "n_results", + "api_timeout", + "read_only", + } + ), ), _AdapterSpec( aliases=("agentmemory", "agent-memory", "agent_memory"), loader=_agentmemory_loader, - accepts=frozenset({ - "api_url", "project", "n_results", "api_timeout", - "include_lessons", "read_only", - }), + accepts=frozenset( + { + "api_url", + "project", + "n_results", + "api_timeout", + "include_lessons", + "read_only", + } + ), ), ) @@ -375,7 +478,7 @@ def _print_report( print(f" components: {_fmt_int(health['components'])}") print( f" largest component: {_fmt_int(health['largest_component_size'])}" - f" ({health['largest_component_ratio']*100:.1f}% of nodes)" + f" ({health['largest_component_ratio'] * 100:.1f}% of nodes)" ) print(f" isolated nodes: {_fmt_int(health['isolated_nodes'])}") print(f" avg degree: {health['avg_degree']:.2f}") @@ -392,8 +495,7 @@ def _print_report( print(f" {et:35s} {c:>8,} ({pct:5.1f}%)") print(f"\n edge type entropy: {health['edge_type_entropy_bits']:.2f} bits") print( - " (higher = more diverse vocabulary; " - "low bits indicate monoculture)" + " (higher = more diverse vocabulary; low bits indicate monoculture)" ) print("\nCommunity structure (Louvain)") @@ -401,14 +503,12 @@ def _print_report( print(f" modularity: {community.modularity:.3f}") print( f" inter-community: {_fmt_int(community.inter_community_edges)} edges " - f"({community.inter_community_ratio*100:.1f}%)" + f"({community.inter_community_ratio * 100:.1f}%)" ) print(f" top sizes: {community.sizes[:10]}") print("\nPer-edge-type component count (Cat 4c monoculture signal)") - for et, n_comp in sorted( - edge_type_components.items(), key=lambda kv: -kv[1] - )[:15]: + for et, n_comp in sorted(edge_type_components.items(), key=lambda kv: -kv[1])[:15]: print(f" {et:35s} {n_comp:>8,} components") if betti is not None: @@ -420,21 +520,13 @@ def _print_report( if betti.skipped: print(f" SKIPPED: {betti.skip_reason}") else: - print( - f" Betti-0: {betti.betti_0}" - " (should be 1 for a single component)" - ) - print( - f" Betti-1: {betti.betti_1}" - " (structural loops / holes)" - ) + print(f" Betti-0: {betti.betti_0} (should be 1 for a single component)") + print(f" Betti-1: {betti.betti_1} (structural loops / holes)") if betti.h1_bars: print(f" max H1 persistence: {betti.max_h1_persistence:.2f} hops") print(" top H1 bars (birth, death, persistence):") for b, d, p in betti.h1_bars[:10]: - print( - f" birth={b:5.2f} death={d:5.2f} persistence={p:5.2f}" - ) + print(f" birth={b:5.2f} death={d:5.2f} persistence={p:5.2f}") else: print(" no H1 features found — graph is acyclic / tree-like") @@ -448,10 +540,7 @@ def _print_report( elif kind == "rel": print(f" rel tables: {', '.join(entry['tables'])}") elif kind == "entity_edge_types": - print( - " entity_type vocab: " - + (", ".join(entry["values"]) or "") - ) + print(" entity_type vocab: " + (", ".join(entry["values"]) or "")) else: # Generic shape: print whatever list-valued keys are there for key, val in entry.items(): @@ -466,6 +555,7 @@ def _print_report( if doc: # Wrap to 66 cols under a "documentation:" label import textwrap + print(" documentation:") for line in textwrap.wrap(doc, width=66): print(f" {line}") @@ -664,8 +754,7 @@ def cmd_cat8(args: argparse.Namespace) -> int: if getattr(kg_adapter, "graph_kg_only", False): kg_entities, kg_edges = kg_adapter.get_graph_snapshot() log.info( - "Cat 8 KG snapshot for graph=kg claims: %d entities, " - "%d edges", + "Cat 8 KG snapshot for graph=kg claims: %d entities, %d edges", len(kg_entities), len(kg_edges), ) @@ -717,7 +806,9 @@ def cmd_cat8(args: argparse.Namespace) -> int: print(f" found: {len(report.edges_found)} ({', '.join(report.edges_found) or '—'})") print(f" missing: {len(report.edges_missing)} ({', '.join(report.edges_missing) or '—'})") if report.edges_undeclared: - print(f" undeclared: {len(report.edges_undeclared)} ({', '.join(report.edges_undeclared[:8])}{'...' if len(report.edges_undeclared) > 8 else ''})") + print( + f" undeclared: {len(report.edges_undeclared)} ({', '.join(report.edges_undeclared[:8])}{'...' if len(report.edges_undeclared) > 8 else ''})" + ) print(f" coverage: {report.edge_vocabulary_coverage:.1%}") print("\n8c Schema-data alignment") @@ -740,8 +831,7 @@ def cmd_cat8(args: argparse.Namespace) -> int: print("\n Hall usage (MemPalace-specific):") print(f" total drawers: {hu['total_drawers']}") print( - f" drawers with hall set: {hu['populated_count']} " - f"({hu['fraction_populated']:.1%})" + f" drawers with hall set: {hu['populated_count']} ({hu['fraction_populated']:.1%})" ) print(f" declared vocabulary: {', '.join(hu['declared_vocabulary'])}") if hu["distribution"]: @@ -765,7 +855,7 @@ def cmd_cat8(args: argparse.Namespace) -> int: "skipped": "-", }.get(c.status, "?") print(f" {marker} [{c.status:10s}] {c.claim_id}") - print(f" \"{c.claim_text}\"") + print(f' "{c.claim_text}"') if c.operational_definition: op_short = " ".join(c.operational_definition.split())[:100] print(f" op: {op_short}") @@ -773,8 +863,7 @@ def cmd_cat8(args: argparse.Namespace) -> int: print(f" note: {c.notes}") if c.metrics: short_metrics = { - k: (round(v, 3) if isinstance(v, float) else v) - for k, v in c.metrics.items() + k: (round(v, 3) if isinstance(v, float) else v) for k, v in c.metrics.items() } print(f" data: {short_metrics}") print() @@ -791,10 +880,7 @@ def cmd_cat8(args: argparse.Namespace) -> int: print(f" self-reported drift_score: {drift['drift_score']:.1%}") absent = drift.get("declared_edge_types_absent") if absent: - print( - " declared edge types absent at KG layer: " - + ", ".join(absent) - ) + print(" declared edge types absent at KG layer: " + ", ".join(absent)) else: print( " (most systems have no self-report API for type drift, " @@ -814,10 +900,7 @@ def cmd_cat8(args: argparse.Namespace) -> int: mod = full_stats["modularity"] print("\nEXACT full-graph modularity (Cat 8 hierarchical — publishable)") print(f" modularity: {mod:.3f} (over the whole RELATION graph)") - print( - f" verdict: {'PASS' if mod > 0.5 else 'FAIL'} " - "(threshold 0.5)" - ) + print(f" verdict: {'PASS' if mod > 0.5 else 'FAIL'} (threshold 0.5)") print( " ↑ exact daemon-side Louvain over all RELATION edges; " "supersedes the\n sampled 8e modularity above for publication." @@ -887,8 +970,7 @@ def _add_db_or_api_args(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--db", default=None, - help="path to the adapter's db file (file mode). Optional when " - "--api-url is supplied.", + help="path to the adapter's db file (file mode). Optional when --api-url is supplied.", ) parser.add_argument( "--api-url", @@ -928,16 +1010,14 @@ def _add_db_or_api_args(parser: argparse.ArgumentParser) -> None: "--no-mock", dest="mock_inference", action="store_false", - help="(familiar) run inference; for future Cat 9 work where the " - "model writes the answer.", + help="(familiar) run inference; for future Cat 9 work where the model writes the answer.", ) parser.add_argument( "--familiar-timeout", type=float, default=None, metavar="SECONDS", - help="(familiar) HTTP timeout for /api/familiar/eval and " - "/api/familiar/graph. Default 30s.", + help="(familiar) HTTP timeout for /api/familiar/eval and /api/familiar/graph. Default 30s.", ) _add_real_kg_args(parser) @@ -1024,15 +1104,12 @@ def cmd_cat4(args: argparse.Namespace) -> int: # without the capability return None and Cat 4 counts the sampled edges. # Skipped under --structural-projection (the scaffold opt-out). edge_type_override = None - if _use_real_kg(args) and hasattr( - adapter, "get_edge_type_distribution" - ): + if _use_real_kg(args) and hasattr(adapter, "get_edge_type_distribution"): try: edge_type_override = adapter.get_edge_type_distribution() if edge_type_override: log.info( - "Cat 4 using exact RELATION distribution: %d types, " - "%d edges", + "Cat 4 using exact RELATION distribution: %d types, %d edges", len(edge_type_override), sum(edge_type_override.values()), ) @@ -1051,9 +1128,7 @@ def cmd_cat4(args: argparse.Namespace) -> int: bcubed = None if args.gold_aliases: - bcubed = score_alias_resolution_against_gold( - report, entities, args.gold_aliases - ) + bcubed = score_alias_resolution_against_gold(report, entities, args.gold_aliases) print() print("Cat 4a — Alias resolution vs gold registry (B-Cubed)") print("─" * 60) @@ -1300,7 +1375,7 @@ def cmd_cat5(args: argparse.Namespace) -> int: print(f" components: {_fmt_int(full_stats.get('component_count', 0))}") print( f" largest component: {_fmt_int(lc or 0)}" - + (f" ({lcf*100:.1f}% of entities)" if isinstance(lcf, (int, float)) else "") + + (f" ({lcf * 100:.1f}% of entities)" if isinstance(lcf, (int, float)) else "") ) print(f" isolates: {_fmt_int(full_stats.get('isolate_count', 0))}") print( @@ -1366,7 +1441,9 @@ def cmd_cat3(args: argparse.Namespace) -> int: surfaced = adapter.get_contradiction_pairs() log.info( "snapshot: %d entities, %d edges; %d contradiction pairs surfaced", - len(entities), len(edges), len(surfaced), + len(entities), + len(edges), + len(surfaced), ) report = score_cat3( @@ -1440,9 +1517,7 @@ def cmd_cat2c(args: argparse.Namespace) -> int: print(format_report(report)) if args.json: - Path(args.json).write_text( - json.dumps(report.to_dict(), indent=2, default=str) - ) + Path(args.json).write_text(json.dumps(report.to_dict(), indent=2, default=str)) print(f"JSON report written to {args.json}") return 0 @@ -1534,10 +1609,7 @@ def cmd_compile_wiki(args: argparse.Namespace) -> int: elif args.llm_provider == "openai": client = _OpenAILLMClient(model=args.llm_model) else: - raise SystemExit( - f"unknown --llm-provider {args.llm_provider!r}; " - "supported: stub, openai" - ) + raise SystemExit(f"unknown --llm-provider {args.llm_provider!r}; supported: stub, openai") report = compile_vault( args.vault, @@ -1589,10 +1661,7 @@ def complete(self, prompt: str, **kwargs) -> str: if end_marker in body: body = body.split(end_marker, 1)[0] head = body.strip().split("\n", 1)[0][:160] - return ( - f"# Stub summary of {rel}\n\n" - f"First line of source: {head}\n" - ) + return f"# Stub summary of {rel}\n\nFirst line of source: {head}\n" # Index prompt return "# Index\n\n(stub-generated)\n" @@ -1699,14 +1768,12 @@ def count_tokens(text: str) -> int: per_question: list[dict] = [] print() print("=" * 80) - print(f" Retrieval test — adapter={args.adapter} corpus={qdoc.get('version','?')}") + print(f" Retrieval test — adapter={args.adapter} corpus={qdoc.get('version', '?')}") print(f" n_results={args.n_results} questions={len(questions)}") print("=" * 80) query_params = inspect.signature(adapter.query).parameters - has_var_keyword = any( - p.kind == inspect.Parameter.VAR_KEYWORD for p in query_params.values() - ) + has_var_keyword = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in query_params.values()) for q in questions: qid = q.get("id", "?") @@ -1723,7 +1790,15 @@ def count_tokens(text: str) -> int: result = adapter.query(text, **query_kwargs) except Exception as e: # pragma: no cover result = type( - "QR", (), {"answer": "", "context_string": "", "error": str(e), "retrieved_entities": [], "retrieval_path": []} + "QR", + (), + { + "answer": "", + "context_string": "", + "error": str(e), + "retrieved_entities": [], + "retrieval_path": [], + }, )() elapsed = time.time() - t0 @@ -1743,7 +1818,7 @@ def count_tokens(text: str) -> int: status = "✓" if recall >= 1.0 else ("~" if hit else "✗") print( f"\n{qid} (hops={min_hops}) {status} recall={recall:.2f} " - f"tokens={tokens} {elapsed*1000:.0f}ms{path_note}" + f"tokens={tokens} {elapsed * 1000:.0f}ms{path_note}" ) print(f" Q: {text}") print(f" expected: {expected}") @@ -1786,9 +1861,7 @@ def count_tokens(text: str) -> int: avg_recall = sum(pq["recall"] for pq in group) / n if n else 0.0 hit_rate = sum(1 for pq in group if pq["hit"]) / n if n else 0.0 avg_tokens = sum(pq["tokens"] for pq in group) / n if n else 0.0 - print( - f"{hops:>6} {n:>4} {avg_recall:>7.2%} {hit_rate:>9.2%} {avg_tokens:>8.0f}" - ) + print(f"{hops:>6} {n:>4} {avg_recall:>7.2%} {hit_rate:>9.2%} {avg_tokens:>8.0f}") total_n = len(per_question) total_recall = sum(pq["recall"] for pq in per_question) / total_n if total_n else 0.0 @@ -1822,9 +1895,7 @@ def count_tokens(text: str) -> int: "partial_hit": sum(1 for pq in per_question if pq["hit"]), "mean_recall": total_recall, "mean_tokens": total_tokens / total_n if total_n else 0.0, - "tokens_per_correct_answer": ( - tokens_per_correct if correct_count else None - ), + "tokens_per_correct_answer": (tokens_per_correct if correct_count else None), "by_hop": { str(h): { "n": len(g), @@ -1857,14 +1928,13 @@ def cmd_candidate_strategy(args: argparse.Namespace) -> int: import os from datetime import datetime, timezone from sme.eval.candidate_strategy import ( - DEFAULT_STRATEGIES, probe_rerank, run_eval, run_eval_multi_limit, + DEFAULT_STRATEGIES, + probe_rerank, + run_eval, + run_eval_multi_limit, ) - api_url = ( - args.api_url - or os.environ.get("PALACE_DAEMON_URL") - or "http://localhost:8085" - ) + api_url = args.api_url or os.environ.get("PALACE_DAEMON_URL") or "http://localhost:8085" api_key = args.api_key or os.environ.get("PALACE_API_KEY") if not api_key: log.error("--api-key required (or set PALACE_API_KEY)") @@ -1895,8 +1965,10 @@ def _progress(i, n, qid): # Multi-limit sweep path if args.limits: report_core = run_eval_multi_limit( - api_url=api_url, api_key=api_key, - queries=queries, strategies=strategies, + api_url=api_url, + api_key=api_key, + queries=queries, + strategies=strategies, limits=list(args.limits), progress=_progress, ) @@ -1917,14 +1989,15 @@ def _progress(i, n, qid): print() print("=" * 60) - print(f" diagnostic: candidate_strategy n: {len(queries)} " - f"limits: {list(args.limits)}") + print(f" diagnostic: candidate_strategy n: {len(queries)} limits: {list(args.limits)}") for limit, summary in report["summary_by_limit"].items(): print(f"\n limit={limit}:") for s, blk in summary["per_strategy"].items(): if blk.get("n", 0): - print(f" {s:10} R@5={blk['R@5']:.3f} R@10={blk['R@10']:.3f} " - f"MRR={blk['MRR']:.3f} p50={blk['p50_ms']:.0f}ms") + print( + f" {s:10} R@5={blk['R@5']:.3f} R@10={blk['R@10']:.3f} " + f"MRR={blk['MRR']:.3f} p50={blk['p50_ms']:.0f}ms" + ) if args.json: args.json.parent.mkdir(parents=True, exist_ok=True) @@ -1934,7 +2007,8 @@ def _progress(i, n, qid): # Single-limit path (unchanged) report_core = run_eval( - api_url=api_url, api_key=api_key, + api_url=api_url, + api_key=api_key, queries=queries, strategies=strategies, limit=args.limit, @@ -1960,9 +2034,11 @@ def _progress(i, n, qid): print(f" diagnostic: candidate_strategy n: {len(queries)} limit: {args.limit}") for s, blk in s_map.items(): if blk.get("n", 0): - print(f" {s:10} R@5={blk['R@5']:.3f} R@10={blk['R@10']:.3f} " - f"MRR={blk['MRR']:.3f} p50={blk['p50_ms']:.0f}ms " - f"p95={blk['p95_ms']:.0f}ms") + print( + f" {s:10} R@5={blk['R@5']:.3f} R@10={blk['R@10']:.3f} " + f"MRR={blk['MRR']:.3f} p50={blk['p50_ms']:.0f}ms " + f"p95={blk['p95_ms']:.0f}ms" + ) # Strategy-flip table — the #57 actionable diagnostic flips = report["summary"]["strategy_flips"] @@ -1975,8 +2051,10 @@ def _progress(i, n, qid): new_n = len(fl["new_hits"]) lost_n = len(fl["lost_hits"]) same_n = fl["unchanged_count"] - print(f" {pair:20} moved_up={up} moved_down={dn} " - f"new_hits={new_n} lost_hits={lost_n} same={same_n}") + print( + f" {pair:20} moved_up={up} moved_down={dn} " + f"new_hits={new_n} lost_hits={lost_n} same={same_n}" + ) if args.json: args.json.parent.mkdir(parents=True, exist_ok=True) @@ -2002,6 +2080,7 @@ def cmd_longmem(args: argparse.Namespace) -> int: a partial reading is possible without API access. """ import sys as _sys + _scripts_dir = str(Path(__file__).resolve().parent.parent / "scripts") if _scripts_dir not in _sys.path: _sys.path.insert(0, _scripts_dir) @@ -2039,6 +2118,7 @@ def cmd_longmem(args: argparse.Namespace) -> int: out_path = args.json if out_path is None: import datetime as _dt + ts = _dt.datetime.now().strftime("%Y%m%dT%H%M%S") out_path = Path(f"longmem_{args.adapter}_{ts}.json") Path(out_path).write_text(json.dumps(report, indent=2, default=str)) @@ -2049,10 +2129,7 @@ def cmd_longmem(args: argparse.Namespace) -> int: print() print("=" * 78) - print( - f" LongMemEval E2E — adapter={args.adapter} " - f"n={summary['total_questions']}" - ) + print(f" LongMemEval E2E — adapter={args.adapter} n={summary['total_questions']}") print("=" * 78) print(f"\n{'category':22s} {'n':>4} {'R@5':>8} {'QA-acc':>8} {'gap':>8}") for cat, slot in dual.get("per_category", {}).items(): @@ -2060,10 +2137,7 @@ def cmd_longmem(args: argparse.Namespace) -> int: gap = slot.get("retrieval_qa_gap") qa_str = f"{qa:>7.2%}" if qa is not None else " n/a" gap_str = f"{gap:+.3f}" if gap is not None else " n/a" - print( - f"{cat:22s} {slot['n']:>4} " - f"{slot['sme_recall_mean']:>7.2%} {qa_str} {gap_str:>8}" - ) + print(f"{cat:22s} {slot['n']:>4} {slot['sme_recall_mean']:>7.2%} {qa_str} {gap_str:>8}") overall_qa = overall.get("qa_accuracy") overall_gap = overall.get("retrieval_qa_gap") overall_qa_str = f"{overall_qa:>7.2%}" if overall_qa is not None else " n/a" @@ -2082,9 +2156,7 @@ def main(argv: list[str] | None = None) -> int: prog="sme-eval", description="Structural Memory Evaluation — analyze a memory system's graph.", ) - parser.add_argument( - "-v", "--verbose", action="store_true", help="enable info logging" - ) + parser.add_argument("-v", "--verbose", action="store_true", help="enable info logging") sub = parser.add_subparsers(dest="command", required=True) ana = sub.add_parser( @@ -2131,8 +2203,7 @@ def main(argv: list[str] | None = None) -> int: ana.add_argument( "--collection-name", metavar="NAME", - help="(mempalace adapter) ChromaDB collection name. " - "Defaults to mempalace_drawers.", + help="(mempalace adapter) ChromaDB collection name. Defaults to mempalace_drawers.", ) ana.add_argument( "--betti", @@ -2225,8 +2296,7 @@ def main(argv: list[str] | None = None) -> int: "--age-fused", dest="age_fused", action="store_true", - help="(mempalace-daemon) shorthand for " - "--search-endpoint /search/age-fused.", + help="(mempalace-daemon) shorthand for --search-endpoint /search/age-fused.", ) ret.add_argument( "--candidate-strategy", @@ -2279,16 +2349,14 @@ def main(argv: list[str] | None = None) -> int: "--no-mock", dest="mock_inference", action="store_false", - help="(familiar) run inference; for future Cat 9 work where the " - "model writes the answer.", + help="(familiar) run inference; for future Cat 9 work where the model writes the answer.", ) ret.add_argument( "--familiar-timeout", type=float, default=None, metavar="SECONDS", - help="(familiar) HTTP timeout for /api/familiar/eval and " - "/api/familiar/graph. Default 30s.", + help="(familiar) HTTP timeout for /api/familiar/eval and /api/familiar/graph. Default 30s.", ) ret.set_defaults(func=cmd_retrieve) @@ -2497,8 +2565,7 @@ def main(argv: list[str] | None = None) -> int: "--betti-max-nodes", type=int, default=2000, - help="skip homology when the largest component exceeds this size. " - "Default: 2000.", + help="skip homology when the largest component exceeds this size. Default: 2000.", ) c5.add_argument( "--min-component-size", @@ -2537,15 +2604,19 @@ def main(argv: list[str] | None = None) -> int: c3.add_argument("--adapter", required=True) _add_db_or_api_args(c3) c3.add_argument( - "--collection-name", metavar="NAME", + "--collection-name", + metavar="NAME", help="(mempalace/flat) ChromaDB collection name", ) c3.add_argument( - "--kg-path", metavar="PATH", + "--kg-path", + metavar="PATH", help="(mempalace) SQLite knowledge graph path override", ) c3.add_argument( - "--flat-detection-rate", type=float, default=0.0, + "--flat-detection-rate", + type=float, + default=0.0, help="flat-baseline contradiction detection rate for the " "(structural − flat) delta. Default 0.0 — a flat retriever " "surfaces no structured pairs by construction.", @@ -2565,15 +2636,19 @@ def main(argv: list[str] | None = None) -> int: c6.add_argument("--adapter", required=True) _add_db_or_api_args(c6) c6.add_argument( - "--collection-name", metavar="NAME", + "--collection-name", + metavar="NAME", help="(mempalace/flat) ChromaDB collection name", ) c6.add_argument( - "--kg-path", metavar="PATH", + "--kg-path", + metavar="PATH", help="(mempalace) SQLite knowledge graph path override", ) c6.add_argument( - "--flat-completeness", type=float, default=0.0, + "--flat-completeness", + type=float, + default=0.0, help="flat-baseline supersession completeness for the " "(structural − flat) delta. Default 0.0 — a flat retriever has " "no edges and no _superseded_by field.", @@ -2589,9 +2664,7 @@ def main(argv: list[str] | None = None) -> int: "Compares Condition A (flat) / B (full pipeline) / C (structure " "disabled) by hop depth.", ) - c2c.add_argument( - "--flat", metavar="JSON", help="retrieve-results JSON for Condition A" - ) + c2c.add_argument("--flat", metavar="JSON", help="retrieve-results JSON for Condition A") c2c.add_argument( "--graph", required=True, @@ -2605,9 +2678,7 @@ def main(argv: list[str] | None = None) -> int: ) c2c.add_argument("--flat-label", help="custom label for Condition A") c2c.add_argument("--graph-label", help="custom label for Condition B") - c2c.add_argument( - "--no-structure-label", help="custom label for Condition C" - ) + c2c.add_argument("--no-structure-label", help="custom label for Condition C") c2c.add_argument("--json", metavar="PATH", help="write full report as JSON") c2c.set_defaults(func=cmd_cat2c) @@ -2684,39 +2755,50 @@ def main(argv: list[str] | None = None) -> int: cs = sub.add_parser( "candidate-strategy", help="A/B/C ablation of palace-daemon's candidate retrieval " - "strategies (vector / union / hybrid). Reports per-strategy " - "R@5/10, MRR, latency, and a strategy-flip diagnostic.", + "strategies (vector / union / hybrid). Reports per-strategy " + "R@5/10, MRR, latency, and a strategy-flip diagnostic.", ) cs.add_argument( - "--queries", required=True, type=Path, + "--queries", + required=True, + type=Path, help="Labeled query JSON (palace-daemon rerank_eval_queries shape).", ) cs.add_argument( - "--api-url", default=None, - help="Daemon base URL. Defaults to PALACE_DAEMON_URL env or " - "http://localhost:8085.", + "--api-url", + default=None, + help="Daemon base URL. Defaults to PALACE_DAEMON_URL env or http://localhost:8085.", ) cs.add_argument( - "--api-key", default=None, + "--api-key", + default=None, help="X-API-Key for the daemon. Defaults to PALACE_API_KEY env.", ) cs.add_argument( - "--strategies", nargs="+", default=None, + "--strategies", + nargs="+", + default=None, help="Strategies to ablate (default: vector union hybrid).", ) cs.add_argument( - "--limit", type=int, default=None, - help="Per-query candidate pool size (default: 20). Mutually " - "exclusive with --limits.", + "--limit", + type=int, + default=None, + help="Per-query candidate pool size (default: 20). Mutually exclusive with --limits.", ) cs.add_argument( - "--limits", type=int, nargs="+", default=None, + "--limits", + type=int, + nargs="+", + default=None, help="Multi-limit sweep — runs each strategy at every K in the " - "list (e.g. --limits 5 10 20 50). Shows how the top-K cliff " - "shifts across strategies. Mutually exclusive with --limit.", + "list (e.g. --limits 5 10 20 50). Shows how the top-K cliff " + "shifts across strategies. Mutually exclusive with --limit.", ) cs.add_argument( - "--json", type=Path, default=None, + "--json", + type=Path, + default=None, help="Output JSON path.", ) cs.set_defaults(func=cmd_candidate_strategy) @@ -2726,38 +2808,38 @@ def main(argv: list[str] | None = None) -> int: lm = sub.add_parser( "longmemeval", help="Run LongMemEval E2E QA through an SME adapter — reports " - "both R@5 retrieval recall and judge-scored QA accuracy " - "plus the retrieval/QA gap per category.", + "both R@5 retrieval recall and judge-scored QA accuracy " + "plus the retrieval/QA gap per category.", ) lm.add_argument( "--adapter", required=True, help="Adapter to run the LongMemEval haystack through. The harness " - "currently wires {full-context, flat, karpathy-compiled, " - "mempalace, mempalace-daemon}.", + "currently wires {full-context, flat, karpathy-compiled, " + "mempalace, mempalace-daemon}.", ) lm.add_argument( "--questions", required=True, metavar="JSON", help="Path to the dataset JSON. --corpus longmemeval (default): " - "longmemeval_oracle.json / _s / _m. --corpus locomo: " - "locomo10.json.", + "longmemeval_oracle.json / _s / _m. --corpus locomo: " + "locomo10.json.", ) lm.add_argument( "--corpus", default="longmemeval", choices=["longmemeval", "locomo"], help="Dataset shape. 'longmemeval' (default) = one haystack per " - "question. 'locomo' = one conversation per sample (questions " - "grouped by sample_id, ingested per sample; cat-5 adversarial " - "judged abstention-aware). Composes with --age-fused / --kind.", + "question. 'locomo' = one conversation per sample (questions " + "grouped by sample_id, ingested per sample; cat-5 adversarial " + "judged abstention-aware). Composes with --age-fused / --kind.", ) lm.add_argument( "--answer-model", default="gpt-4.1-mini", help="Reader model (turns retrieved context into an answer the " - "judge can score). Default: gpt-4.1-mini.", + "judge can score). Default: gpt-4.1-mini.", ) lm.add_argument( "--judge", @@ -2774,14 +2856,14 @@ def main(argv: list[str] | None = None) -> int: "--skip-judge", action="store_true", help="Skip reader + judge entirely. Reports R@5 only — useful " - "for an API-key-free retrieval-only run.", + "for an API-key-free retrieval-only run.", ) lm.add_argument( "--skip-reader", action="store_true", help="Feed the raw retrieved context to the judge instead of " - "running a reader pass. Diagnostic mode — not apples-to-apples " - "with published LongMemEval numbers.", + "running a reader pass. Diagnostic mode — not apples-to-apples " + "with published LongMemEval numbers.", ) lm.add_argument( "--json", diff --git a/tests/test_mempalace_server_adapter.py b/tests/test_mempalace_server_adapter.py index b057db7..ee62640 100644 --- a/tests/test_mempalace_server_adapter.py +++ b/tests/test_mempalace_server_adapter.py @@ -45,8 +45,7 @@ def _adapter(**over): def test_default_construction_uses_docker_compose_defaults(monkeypatch): - for var in ("MEMPALACE_SERVER_URL", "MEMPALACE_SERVER_API_KEY", - "MEMPALACE_SERVER_TENANT"): + for var in ("MEMPALACE_SERVER_URL", "MEMPALACE_SERVER_API_KEY", "MEMPALACE_SERVER_TENANT"): monkeypatch.delenv(var, raising=False) a = MemPalaceServerAdapter() assert a.api_url == "http://localhost:8000" @@ -92,15 +91,24 @@ def add_route(req): fake_urlopen_factory({f"POST {BASE}/mp/api/v1/drawers": add_route}) a = _adapter(wing="fallback-wing") result = a.ingest_corpus( - [{"id": "doc-1", "text": "hello world", "wing": "projects", - "room": "decisions", "source_file": "d1.md"}] + [ + { + "id": "doc-1", + "text": "hello world", + "wing": "projects", + "room": "decisions", + "source_file": "d1.md", + } + ] ) assert result["entities_created"] == 1 assert result["edges_created"] == 0 assert result["errors"] == [] assert captured["body"] == { - "wing": "projects", "room": "decisions", - "content": "hello world", "source_file": "d1.md", + "wing": "projects", + "room": "decisions", + "content": "hello world", + "source_file": "d1.md", "added_by": "sme", } assert captured["auth"] == "Bearer test-key" @@ -115,13 +123,17 @@ def add_route(req): fake_urlopen_factory({f"POST {BASE}/mp/api/v1/drawers": add_route}) a = _adapter() - a.ingest_corpus([ - {"id": "1", "document": "from-document", "content": "c", "text": "t"}, - {"id": "2", "content": "from-content", "text": "t"}, - {"id": "3", "text": "from-text"}, - ]) + a.ingest_corpus( + [ + {"id": "1", "document": "from-document", "content": "c", "text": "t"}, + {"id": "2", "content": "from-content", "text": "t"}, + {"id": "3", "text": "from-text"}, + ] + ) assert [b["content"] for b in bodies] == [ - "from-document", "from-content", "from-text", + "from-document", + "from-content", + "from-text", ] @@ -134,29 +146,42 @@ def add_route(req): fake_urlopen_factory({f"POST {BASE}/mp/api/v1/drawers": add_route}) a = _adapter(wing="w", room=None) - a.ingest_corpus([ - {"id": "id-1", "text": "a", "session_id": "sess-9"}, # session wins - {"id": "id-2", "text": "b"}, # id used - ]) + a.ingest_corpus( + [ + {"id": "id-1", "text": "a", "session_id": "sess-9"}, # session wins + {"id": "id-2", "text": "b"}, # id used + ] + ) assert [b["room"] for b in bodies] == ["sess-9", "id-2"] def test_ingest_counts_bullets(fake_urlopen_factory): - fake_urlopen_factory({ - f"POST {BASE}/mp/api/v1/drawers": - {"success": True, "bullets_stored": 3, "bullets_total": 3, - "wing": "w", "room": "r"}, - }) + fake_urlopen_factory( + { + f"POST {BASE}/mp/api/v1/drawers": { + "success": True, + "bullets_stored": 3, + "bullets_total": 3, + "wing": "w", + "room": "r", + }, + } + ) a = _adapter() result = a.ingest_corpus([{"id": "1", "text": "- a\n- b\n- c"}]) assert result["entities_created"] == 3 def test_ingest_already_exists_counts_zero(fake_urlopen_factory): - fake_urlopen_factory({ - f"POST {BASE}/mp/api/v1/drawers": - {"success": True, "reason": "already_exists", "drawer_id": "abc"}, - }) + fake_urlopen_factory( + { + f"POST {BASE}/mp/api/v1/drawers": { + "success": True, + "reason": "already_exists", + "drawer_id": "abc", + }, + } + ) a = _adapter() result = a.ingest_corpus([{"id": "1", "text": "dup"}]) assert result["entities_created"] == 0 @@ -171,9 +196,7 @@ def test_ingest_skips_blank_content(fake_urlopen_factory): def test_ingest_http_error_captured_not_raised(fake_urlopen_factory): - err = urllib.error.HTTPError( - f"{BASE}/mp/api/v1/drawers", 500, "Server Error", {}, None - ) + err = urllib.error.HTTPError(f"{BASE}/mp/api/v1/drawers", 500, "Server Error", {}, None) fake_urlopen_factory({f"POST {BASE}/mp/api/v1/drawers": err}) a = _adapter() result = a.ingest_corpus([{"id": "1", "text": "boom"}]) @@ -198,11 +221,13 @@ def add_route(req): seq.append(("add", req.full_url)) return {"success": True, "drawer_id": "new"} - fake_urlopen_factory({ - f"GET {BASE}/mp/api/v1/drawers": list_route, - f"DELETE {BASE}/mp/api/v1/drawers/old1": del_route, - f"POST {BASE}/mp/api/v1/drawers": add_route, - }) + fake_urlopen_factory( + { + f"GET {BASE}/mp/api/v1/drawers": list_route, + f"DELETE {BASE}/mp/api/v1/drawers/old1": del_route, + f"POST {BASE}/mp/api/v1/drawers": add_route, + } + ) a = _adapter(wing="scoped", reset_before_ingest=True) result = a.ingest_corpus([{"id": "1", "text": "fresh"}]) kinds = [k for k, _ in seq] @@ -221,23 +246,36 @@ def _search_body(results): def test_query_happy_path(fake_urlopen_factory): - fake_urlopen_factory({ - f"POST {BASE}/mp/api/v1/search": _search_body([ - { - "drawer_id": "d-1", "wing": "projects", "room": "decisions", - "similarity": 0.87, "distance": 0.13, "content": "We use JWT.", - "filed_at": "2026-07-03T00:00:00Z", "source_file": "auth.md", - "metadata": {"wing": "projects", "room": "decisions"}, - }, - ]), - }) + fake_urlopen_factory( + { + f"POST {BASE}/mp/api/v1/search": _search_body( + [ + { + "drawer_id": "d-1", + "wing": "projects", + "room": "decisions", + "similarity": 0.87, + "distance": 0.13, + "content": "We use JWT.", + "filed_at": "2026-07-03T00:00:00Z", + "source_file": "auth.md", + "metadata": {"wing": "projects", "room": "decisions"}, + }, + ] + ), + } + ) a = _adapter() r = a.query("auth?", n_results=3) assert r.error is None assert "We use JWT." in r.context_string assert len(r.retrieved_entities) == 1 e = r.retrieved_entities[0] - assert e.id == "d-1" + # Retrieval scorers match Entity.id against corpus/session ids, which + # ingest stores in source_file — so the id is the source stem, with the + # server's content-hash drawer id kept in properties for provenance. + assert e.id == "auth" + assert e.properties["drawer_id"] == "d-1" assert e.properties["wing"] == "projects" assert e.properties["room"] == "decisions" assert e.properties["similarity"] == 0.87 @@ -248,18 +286,22 @@ def test_query_happy_path(fake_urlopen_factory): def test_query_without_n_results_kwarg(fake_urlopen_factory): - fake_urlopen_factory({ - f"POST {BASE}/mp/api/v1/search": _search_body([]), - }) + fake_urlopen_factory( + { + f"POST {BASE}/mp/api/v1/search": _search_body([]), + } + ) a = _adapter() r = a.query("anything") assert isinstance(r, QueryResult) def test_query_no_results_sets_error(fake_urlopen_factory): - fake_urlopen_factory({ - f"POST {BASE}/mp/api/v1/search": _search_body([]), - }) + fake_urlopen_factory( + { + f"POST {BASE}/mp/api/v1/search": _search_body([]), + } + ) a = _adapter() r = a.query("nothing here") assert r.context_string == "" @@ -311,9 +353,7 @@ def search_route(req): def test_query_auth_error_no_raise(fake_urlopen_factory): - err = urllib.error.HTTPError( - f"{BASE}/mp/api/v1/search", 401, "Unauthorized", {}, None - ) + err = urllib.error.HTTPError(f"{BASE}/mp/api/v1/search", 401, "Unauthorized", {}, None) fake_urlopen_factory({f"POST {BASE}/mp/api/v1/search": err}) a = _adapter() r = a.query("q") @@ -322,10 +362,11 @@ def test_query_auth_error_no_raise(fake_urlopen_factory): def test_query_connection_error_no_raise(fake_urlopen_factory): - fake_urlopen_factory({ - f"POST {BASE}/mp/api/v1/search": - urllib.error.URLError("Connection refused"), - }) + fake_urlopen_factory( + { + f"POST {BASE}/mp/api/v1/search": urllib.error.URLError("Connection refused"), + } + ) a = _adapter() r = a.query("q") assert r.error and r.error.startswith("CONNECTION:") @@ -345,36 +386,39 @@ def route(req): env = json.loads(req.data.decode()) rid = env.get("id") if error is not None: - return {"jsonrpc": "2.0", "id": rid, - "error": {"code": -32603, "message": error}} + return {"jsonrpc": "2.0", "id": rid, "error": {"code": -32603, "message": error}} name = env["params"]["name"] args = env["params"].get("arguments", {}) if name == "mempalace_kg_search_entities": payload = {"entities": entities, "count": len(entities)} elif name == "mempalace_kg_get_entity": ename = args.get("name") - payload = {"entity": {"name": ename}, - "relations": relations_by_entity.get(ename, [])} + payload = {"entity": {"name": ename}, "relations": relations_by_entity.get(ename, [])} else: payload = {"ok": True} - return {"jsonrpc": "2.0", "id": rid, - "result": {"content": [{"type": "text", "text": json.dumps(payload)}]}} + return { + "jsonrpc": "2.0", + "id": rid, + "result": {"content": [{"type": "text", "text": json.dumps(payload)}]}, + } return route def test_graph_snapshot_from_taxonomy(fake_urlopen_factory): # KG empty → deterministic fallback to the wing/room taxonomy snapshot. - fake_urlopen_factory({ - f"POST {BASE}/mp/mcp": _mcp_route(entities=[]), - f"GET {BASE}/mp/api/v1/taxonomy": { - "total": 6, - "taxonomy": { - "projects": {"decisions": 3, "notes": 2}, - "people": {"jp": 1}, + fake_urlopen_factory( + { + f"POST {BASE}/mp/mcp": _mcp_route(entities=[]), + f"GET {BASE}/mp/api/v1/taxonomy": { + "total": 6, + "taxonomy": { + "projects": {"decisions": 3, "notes": 2}, + "people": {"jp": 1}, + }, }, - }, - }) + } + ) a = _adapter() entities, edges = a.get_graph_snapshot() ids = {e.id for e in entities} @@ -391,13 +435,13 @@ def test_graph_snapshot_from_taxonomy(fake_urlopen_factory): def test_graph_snapshot_error_returns_empty(fake_urlopen_factory): - err = urllib.error.HTTPError( - f"{BASE}/mp/api/v1/taxonomy", 500, "Server Error", {}, None + err = urllib.error.HTTPError(f"{BASE}/mp/api/v1/taxonomy", 500, "Server Error", {}, None) + fake_urlopen_factory( + { + f"POST {BASE}/mp/mcp": _mcp_route(entities=[]), # KG empty + f"GET {BASE}/mp/api/v1/taxonomy": err, # taxonomy errors + } ) - fake_urlopen_factory({ - f"POST {BASE}/mp/mcp": _mcp_route(entities=[]), # KG empty - f"GET {BASE}/mp/api/v1/taxonomy": err, # taxonomy errors - }) a = _adapter() entities, edges = a.get_graph_snapshot() assert entities == [] @@ -408,20 +452,22 @@ def test_graph_snapshot_error_returns_empty(fake_urlopen_factory): def test_graph_snapshot_from_kg(fake_urlopen_factory): - fake_urlopen_factory({ - f"POST {BASE}/mp/mcp": _mcp_route( - entities=[ - {"name": "Sarah", "entity_type": "person"}, - {"name": "Acme", "entity_type": "organization"}, - ], - relations_by_entity={ - "Sarah": [{"type": "works_at", "from": "Sarah", "to": "Acme"}], - "Acme": [{"type": "works_at", "from": "Sarah", "to": "Acme"}], - }, - ), - # taxonomy present too — but KG is non-empty so it must NOT be used. - f"GET {BASE}/mp/api/v1/taxonomy": {"total": 9, "taxonomy": {"w": {"r": 9}}}, - }) + fake_urlopen_factory( + { + f"POST {BASE}/mp/mcp": _mcp_route( + entities=[ + {"name": "Sarah", "entity_type": "person"}, + {"name": "Acme", "entity_type": "organization"}, + ], + relations_by_entity={ + "Sarah": [{"type": "works_at", "from": "Sarah", "to": "Acme"}], + "Acme": [{"type": "works_at", "from": "Sarah", "to": "Acme"}], + }, + ), + # taxonomy present too — but KG is non-empty so it must NOT be used. + f"GET {BASE}/mp/api/v1/taxonomy": {"total": 9, "taxonomy": {"w": {"r": 9}}}, + } + ) a = _adapter() entities, edges = a.get_graph_snapshot() ids = {e.id for e in entities} @@ -436,10 +482,12 @@ def test_graph_snapshot_from_kg(fake_urlopen_factory): def test_graph_snapshot_kg_unavailable_falls_back_to_taxonomy(fake_urlopen_factory): # AGE not installed → kg_search_entities returns a JSON-RPC error. - fake_urlopen_factory({ - f"POST {BASE}/mp/mcp": _mcp_route(error="knowledge graph not available"), - f"GET {BASE}/mp/api/v1/taxonomy": {"total": 1, "taxonomy": {"w": {"r": 1}}}, - }) + fake_urlopen_factory( + { + f"POST {BASE}/mp/mcp": _mcp_route(error="knowledge graph not available"), + f"GET {BASE}/mp/api/v1/taxonomy": {"total": 1, "taxonomy": {"w": {"r": 1}}}, + } + ) a = _adapter() entities, edges = a.get_graph_snapshot() assert a._graph_basis == "taxonomy" @@ -448,14 +496,16 @@ def test_graph_snapshot_kg_unavailable_falls_back_to_taxonomy(fake_urlopen_facto def test_graph_snapshot_kg_synthesizes_missing_endpoint(fake_urlopen_factory): # A relation points to an entity beyond the enumerated set → synthesize it. - fake_urlopen_factory({ - f"POST {BASE}/mp/mcp": _mcp_route( - entities=[{"name": "Sarah", "entity_type": "person"}], - relations_by_entity={ - "Sarah": [{"type": "knows", "from": "Sarah", "to": "Ghost"}], - }, - ), - }) + fake_urlopen_factory( + { + f"POST {BASE}/mp/mcp": _mcp_route( + entities=[{"name": "Sarah", "entity_type": "person"}], + relations_by_entity={ + "Sarah": [{"type": "knows", "from": "Sarah", "to": "Ghost"}], + }, + ), + } + ) a = _adapter() entities, edges = a.get_graph_snapshot() ids = {e.id for e in entities} @@ -467,22 +517,24 @@ def test_graph_snapshot_kg_synthesizes_missing_endpoint(fake_urlopen_factory): def test_kg_contradiction_pairs_via_base_default(fake_urlopen_factory): # A CONTRADICTS-typed KG relation → base-class get_contradiction_pairs # derives one pair from the KG snapshot (no override needed). - fake_urlopen_factory({ - f"POST {BASE}/mp/mcp": _mcp_route( - entities=[ - {"name": "salary_50k", "entity_type": "claim"}, - {"name": "salary_80k", "entity_type": "claim"}, - ], - relations_by_entity={ - "salary_50k": [ - {"type": "CONTRADICTS", "from": "salary_50k", "to": "salary_80k"} - ], - "salary_80k": [ - {"type": "CONTRADICTS", "from": "salary_50k", "to": "salary_80k"} + fake_urlopen_factory( + { + f"POST {BASE}/mp/mcp": _mcp_route( + entities=[ + {"name": "salary_50k", "entity_type": "claim"}, + {"name": "salary_80k", "entity_type": "claim"}, ], - }, - ), - }) + relations_by_entity={ + "salary_50k": [ + {"type": "CONTRADICTS", "from": "salary_50k", "to": "salary_80k"} + ], + "salary_80k": [ + {"type": "CONTRADICTS", "from": "salary_50k", "to": "salary_80k"} + ], + }, + ), + } + ) a = _adapter() pairs = a.get_contradiction_pairs() assert len(pairs) == 1 @@ -490,12 +542,14 @@ def test_kg_contradiction_pairs_via_base_default(fake_urlopen_factory): def test_graph_basis_recorded_in_harness_manifest(fake_urlopen_factory): - fake_urlopen_factory({ - f"POST {BASE}/mp/mcp": _mcp_route( - entities=[{"name": "A", "entity_type": "concept"}], - relations_by_entity={"A": []}, - ), - }) + fake_urlopen_factory( + { + f"POST {BASE}/mp/mcp": _mcp_route( + entities=[{"name": "A", "entity_type": "concept"}], + relations_by_entity={"A": []}, + ), + } + ) a = _adapter() a.get_graph_snapshot() # sets basis = kg mcp = next(d for d in a.get_harness_manifest() if d.kind == "mcp_resource") @@ -519,11 +573,13 @@ def del_route(req): deleted.append(req.full_url.rsplit("/", 1)[-1]) return {"success": True, "drawer_id": deleted[-1]} - fake_urlopen_factory({ - f"GET {BASE}/mp/api/v1/drawers": list_route, - f"DELETE {BASE}/mp/api/v1/drawers/d1": del_route, - f"DELETE {BASE}/mp/api/v1/drawers/d2": del_route, - }) + fake_urlopen_factory( + { + f"GET {BASE}/mp/api/v1/drawers": list_route, + f"DELETE {BASE}/mp/api/v1/drawers/d1": del_route, + f"DELETE {BASE}/mp/api/v1/drawers/d2": del_route, + } + ) a = _adapter() n = a.reset() assert n == 2 @@ -531,10 +587,11 @@ def del_route(req): def test_reset_unreachable_returns_zero_no_raise(fake_urlopen_factory): - fake_urlopen_factory({ - f"GET {BASE}/mp/api/v1/drawers": - urllib.error.URLError("Connection refused"), - }) + fake_urlopen_factory( + { + f"GET {BASE}/mp/api/v1/drawers": urllib.error.URLError("Connection refused"), + } + ) a = _adapter() assert a.reset() == 0 @@ -551,11 +608,15 @@ def test_get_ontology_source_typed(): def test_get_flat_retrieval_delegates_to_query(fake_urlopen_factory): - fake_urlopen_factory({ - f"POST {BASE}/mp/api/v1/search": _search_body([ - {"drawer_id": "d1", "content": "hi", "wing": "w", "room": "r"}, - ]), - }) + fake_urlopen_factory( + { + f"POST {BASE}/mp/api/v1/search": _search_body( + [ + {"drawer_id": "d1", "content": "hi", "wing": "w", "room": "r"}, + ] + ), + } + ) a = _adapter() r = a.get_flat_retrieval("hi") assert isinstance(r, QueryResult) @@ -581,9 +642,11 @@ def test_harness_mcp_probe_success(fake_urlopen_factory): def test_harness_probe_failure_no_raise(fake_urlopen_factory): - fake_urlopen_factory({ - f"GET {BASE}/mp/mcp/health": urllib.error.URLError("down"), - }) + fake_urlopen_factory( + { + f"GET {BASE}/mp/mcp/health": urllib.error.URLError("down"), + } + ) a = _adapter() mcp = next(d for d in a.get_harness_manifest() if d.kind == "mcp_resource") res = mcp.probe_fn() @@ -604,9 +667,7 @@ def test_close_is_idempotent(): # --- Live integration (opt-in) ---------------------------------------- -@pytest.mark.skipif( - not _LIVE, reason="set MEMPALACE_SERVER_LIVE=1 to run against a live Go server" -) +@pytest.mark.skipif(not _LIVE, reason="set MEMPALACE_SERVER_LIVE=1 to run against a live Go server") def test_live_ingest_search_reset_roundtrip(): """End-to-end against a real server: ingest → search → snapshot → reset. @@ -618,12 +679,12 @@ def test_live_ingest_search_reset_roundtrip(): if not mcp.probe_fn().success: pytest.skip("Go MemPalace server not reachable") try: - res = a.ingest_corpus([ - {"id": "live-1", "text": "The capital of France is Paris.", - "room": "geo-1"}, - {"id": "live-2", "text": "The Eiffel Tower stands in Paris.", - "room": "geo-2"}, - ]) + res = a.ingest_corpus( + [ + {"id": "live-1", "text": "The capital of France is Paris.", "room": "geo-1"}, + {"id": "live-2", "text": "The Eiffel Tower stands in Paris.", "room": "geo-2"}, + ] + ) assert res["entities_created"] >= 1, res q = a.query("Where is the Eiffel Tower?", n_results=5) # Either we retrieved something, or the server honestly reports none; @@ -635,9 +696,7 @@ def test_live_ingest_search_reset_roundtrip(): a.reset(wing="sme_selftest") -@pytest.mark.skipif( - not _LIVE, reason="set MEMPALACE_SERVER_LIVE=1 to run against a live Go server" -) +@pytest.mark.skipif(not _LIVE, reason="set MEMPALACE_SERVER_LIVE=1 to run against a live Go server") def test_live_kg_snapshot_reads_real_entity_graph(): """Seed the AGE entity graph explicitly (kg_add_entity/relation), then confirm get_graph_snapshot reads it back with basis='kg'. Proves the @@ -647,24 +706,31 @@ def test_live_kg_snapshot_reads_real_entity_graph(): mcp = next(d for d in a.get_harness_manifest() if d.kind == "mcp_resource") if not mcp.probe_fn().success: pytest.skip("Go MemPalace server not reachable") - seeded = a._mcp_call("mempalace_kg_add_entity", - {"name": "SME_KGProbe_Sarah", "entity_type": "person"}) + seeded = a._mcp_call( + "mempalace_kg_add_entity", {"name": "SME_KGProbe_Sarah", "entity_type": "person"} + ) if seeded is None: pytest.skip("AGE entity graph not available on this server") try: - a._mcp_call("mempalace_kg_add_entity", - {"name": "SME_KGProbe_Acme", "entity_type": "organization"}) - a._mcp_call("mempalace_kg_add_relation", { - "from_entity": "SME_KGProbe_Sarah", - "relation_type": "works_at", - "to_entity": "SME_KGProbe_Acme", - }) + a._mcp_call( + "mempalace_kg_add_entity", {"name": "SME_KGProbe_Acme", "entity_type": "organization"} + ) + a._mcp_call( + "mempalace_kg_add_relation", + { + "from_entity": "SME_KGProbe_Sarah", + "relation_type": "works_at", + "to_entity": "SME_KGProbe_Acme", + }, + ) entities, edges = a.get_graph_snapshot() assert a._graph_basis == "kg", a._graph_basis names = {e.name for e in entities} assert "SME_KGProbe_Sarah" in names and "SME_KGProbe_Acme" in names - assert any(e.source_id == "kg:SME_KGProbe_Sarah" - and e.target_id == "kg:SME_KGProbe_Acme" for e in edges) + assert any( + e.source_id == "kg:SME_KGProbe_Sarah" and e.target_id == "kg:SME_KGProbe_Acme" + for e in edges + ) finally: a._mcp_call("mempalace_kg_delete_entity", {"name": "SME_KGProbe_Sarah"}) a._mcp_call("mempalace_kg_delete_entity", {"name": "SME_KGProbe_Acme"})