From 8fa428435ecb13311d3f1015595cfbdc3acdc43c Mon Sep 17 00:00:00 2001 From: Jack McCarthy Date: Sun, 30 Aug 2026 00:23:07 -0400 Subject: [PATCH 1/2] [autoform] Scale graph and runtime traversal --- autoform_cli/graph.py | 285 +++++++++++++++++++---- autoform_cli/graph_views.py | 14 +- autoform_cli/render.py | 16 +- autoform_cli/runtime.py | 88 +++++-- autoform_cli/status.py | 39 ++-- tests/test_graph_scale.py | 452 ++++++++++++++++++++++++++++++++++++ tests/test_runtime.py | 1 + 7 files changed, 803 insertions(+), 92 deletions(-) create mode 100644 tests/test_graph_scale.py diff --git a/autoform_cli/graph.py b/autoform_cli/graph.py index ec98cdd0..134bd1d8 100644 --- a/autoform_cli/graph.py +++ b/autoform_cli/graph.py @@ -11,8 +11,10 @@ import hashlib import re +from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path +from types import MappingProxyType from urllib.parse import unquote, urlsplit @@ -96,20 +98,107 @@ def formalizable(self) -> bool: return self.declaration is not None +class _TrackedNodeDict(dict[str, Node]): + """A normal mutable node dictionary with a cheap structural revision.""" + + __slots__ = ("_revision",) + + def __init__(self, *args, **kwargs) -> None: + self._revision = getattr(self, "_revision", -1) + 1 + super().__init__(*args, **kwargs) + + @property + def revision(self) -> int: + return getattr(self, "_revision", 0) + + def _touch(self) -> None: + self._revision = getattr(self, "_revision", 0) + 1 + + def __setitem__(self, key: str, value: Node) -> None: + self._touch() + super().__setitem__(key, value) + + def __delitem__(self, key: str) -> None: + self._touch() + super().__delitem__(key) + + def clear(self) -> None: + self._touch() + super().clear() + + def pop(self, key, *args): + self._touch() + return super().pop(key, *args) + + def popitem(self): + self._touch() + return super().popitem() + + def setdefault(self, key, default=None): + self._touch() + return super().setdefault(key, default) + + def update(self, *args, **kwargs) -> None: + self._touch() + super().update(*args, **kwargs) + + def __ior__(self, other): + self._touch() + return super().__ior__(other) + + def __getstate__(self) -> int: + return self._revision + + def __setstate__(self, state: int) -> None: + self._revision = max(self.revision, state) + + +class _GraphCache: + __slots__ = ("_children_by_parent", "_children_revision") + + _children_by_parent: Mapping[str | None, tuple[str, ...]] + _children_revision: int + + @dataclass(frozen=True, slots=True) -class Graph: +class Graph(_GraphCache): """A validated blueprint graph, keyed by stable node id.""" blueprint_dir: Path nodes: dict[str, Node] + def __post_init__(self) -> None: + if not isinstance(self.nodes, _TrackedNodeDict): + object.__setattr__(self, "nodes", _TrackedNodeDict(self.nodes)) + self._refresh_children() + + def __setstate__(self, state: list[object]) -> None: + """Restore legacy slot pickles through the current cache initializer.""" + blueprint_dir, nodes = state + object.__setattr__(self, "blueprint_dir", blueprint_dir) + object.__setattr__(self, "nodes", nodes) + self.__post_init__() + + def _refresh_children(self) -> None: + children: dict[str | None, list[str]] = {} + for node in self.nodes.values(): + children.setdefault(node.parent, []).append(node.id) + object.__setattr__( + self, + "_children_by_parent", + MappingProxyType({parent: tuple(node_ids) for parent, node_ids in children.items()}), + ) + object.__setattr__(self, "_children_revision", self.nodes.revision) + @property def edge_count(self) -> int: return sum(len(node.dependencies) for node in self.nodes.values()) def children(self, node_id: str) -> tuple[str, ...]: """Return the direct contained articles of *node_id*.""" - return tuple(node.id for node in self.nodes.values() if node.parent == node_id) + if getattr(self, "_children_revision", -1) != self.nodes.revision: + self._refresh_children() + return self._children_by_parent.get(node_id, ()) @dataclass(frozen=True, slots=True) @@ -515,79 +604,173 @@ def _resolve_target( def _find_cycles(nodes: dict[str, Node]) -> list[str]: state: dict[str, int] = {} stack: list[str] = [] + stack_indexes: dict[str, int] = {} issues: list[str] = [] + seen_issues: set[str] = set() - def visit(node_id: str) -> None: - state[node_id] = 1 - stack.append(node_id) - for dependency in nodes[node_id].dependencies: - if state.get(dependency, 0) == 0: - visit(dependency) - elif state.get(dependency) == 1: - start = stack.index(dependency) - cycle = stack[start:] + [dependency] + for root_id in sorted(nodes): + if state.get(root_id, 0) != 0: + continue + state[root_id] = 1 + stack_indexes[root_id] = len(stack) + stack.append(root_id) + frames = [(root_id, 0)] + while frames: + node_id, dependency_index = frames[-1] + dependencies = nodes[node_id].dependencies + if dependency_index == len(dependencies): + frames.pop() + stack.pop() + stack_indexes.pop(node_id) + state[node_id] = 2 + continue + + dependency = dependencies[dependency_index] + frames[-1] = (node_id, dependency_index + 1) + dependency_state = state.get(dependency, 0) + if dependency_state == 0: + state[dependency] = 1 + stack_indexes[dependency] = len(stack) + stack.append(dependency) + frames.append((dependency, 0)) + elif dependency_state == 1: + cycle = stack[stack_indexes[dependency] :] + [dependency] message = f"dependency cycle: {' -> '.join(cycle)}" - if message not in issues: + if message not in seen_issues: + seen_issues.add(message) issues.append(message) - stack.pop() - state[node_id] = 2 - - for node_id in sorted(nodes): - if state.get(node_id, 0) == 0: - visit(node_id) return issues def _find_rollup_cycles(nodes: dict[str, Node]) -> list[str]: """Reject cycles introduced by contracting articles at any hierarchy level.""" children: dict[str | None, list[str]] = {} + parents: dict[str, str | None] = {} for node in nodes.values(): children.setdefault(node.parent, []).append(node.id) + parents[node.id] = node.parent + + depths: dict[str, int] = {} + roots: dict[str, str] = {} + for node_id in nodes: + if node_id in depths: + continue + trail: list[str] = [] + seen: set[str] = set() + current: str | None = node_id + while current is not None and current not in depths: + if current in seen or current not in parents: + raise ValueError("article containment is not a forest") + seen.add(current) + trail.append(current) + current = parents[current] + depth = depths[current] if current is not None else -1 + root = roots[current] if current is not None else trail[-1] + for descendant in reversed(trail): + depth += 1 + depths[descendant] = depth + roots[descendant] = root + + ancestors: list[dict[str, str | None]] = [parents] + maximum_depth = max(depths.values(), default=0) + while 1 << len(ancestors) <= maximum_depth: + previous = ancestors[-1] + ancestors.append( + {node_id: previous[parent] if parent is not None else None for node_id, parent in previous.items()} + ) - def direct_child(scope: str | None, node_id: str) -> str | None: - current = node_id - while nodes[current].parent != scope: - parent = nodes[current].parent - if parent is None: - return None - current = parent - return current + def lift(node_id: str, distance: int) -> str: + level = 0 + while distance: + if distance & 1: + parent = ancestors[level][node_id] + if parent is None: + raise ValueError("article containment depth is inconsistent") + node_id = parent + distance >>= 1 + level += 1 + return node_id + + def lowest_common_ancestor(first: str, second: str) -> str | None: + if roots[first] != roots[second]: + return None + if depths[first] < depths[second]: + first, second = second, first + first = lift(first, depths[first] - depths[second]) + if first == second: + return first + for level in range(len(ancestors) - 1, -1, -1): + first_parent = ancestors[level][first] + second_parent = ancestors[level][second] + if first_parent != second_parent: + if first_parent is None or second_parent is None: + continue + first = first_parent + second = second_parent + return parents[first] + + def direct_child(scope: str | None, node_id: str) -> str: + scope_depth = depths[scope] if scope is not None else -1 + return lift(node_id, depths[node_id] - scope_depth - 1) + + projections: dict[str | None, dict[str, set[str]]] = {} + for target in nodes.values(): + for dependency in target.dependencies: + scope = lowest_common_ancestor(target.id, dependency) + if scope == target.id or scope == dependency: + continue + target_child = direct_child(scope, target.id) + source_child = direct_child(scope, dependency) + projections.setdefault(scope, {}).setdefault(target_child, set()).add(source_child) issues: list[str] = [] + seen_issues: set[str] = set() for scope, siblings in children.items(): if len(siblings) < 2: continue - dependencies = {sibling: set() for sibling in siblings} - for target in nodes.values(): - target_child = direct_child(scope, target.id) - if target_child not in dependencies: - continue - for dependency in target.dependencies: - source_child = direct_child(scope, dependency) - if source_child in dependencies and source_child != target_child: - dependencies[target_child].add(source_child) + projected = projections.get(scope) + if not projected: + continue + dependencies = {sibling: projected.get(sibling, set()) for sibling in siblings} state: dict[str, int] = {} stack: list[str] = [] + stack_indexes: dict[str, int] = {} + ordered_dependencies = { + article_id: tuple(sorted(prerequisites)) for article_id, prerequisites in dependencies.items() + } - def visit(article_id: str) -> None: - state[article_id] = 1 - stack.append(article_id) - for prerequisite in sorted(dependencies[article_id]): - if state.get(prerequisite, 0) == 0: - visit(prerequisite) - elif state.get(prerequisite) == 1: - start = stack.index(prerequisite) - cycle = stack[start:] + [prerequisite] + for root_id in sorted(dependencies): + if state.get(root_id, 0) != 0: + continue + state[root_id] = 1 + stack_indexes[root_id] = len(stack) + stack.append(root_id) + frames = [(root_id, 0)] + while frames: + article_id, dependency_index = frames[-1] + prerequisites = ordered_dependencies[article_id] + if dependency_index == len(prerequisites): + frames.pop() + stack.pop() + stack_indexes.pop(article_id) + state[article_id] = 2 + continue + + prerequisite = prerequisites[dependency_index] + frames[-1] = (article_id, dependency_index + 1) + prerequisite_state = state.get(prerequisite, 0) + if prerequisite_state == 0: + state[prerequisite] = 1 + stack_indexes[prerequisite] = len(stack) + stack.append(prerequisite) + frames.append((prerequisite, 0)) + elif prerequisite_state == 1: + cycle = stack[stack_indexes[prerequisite] :] + [prerequisite] label = scope or "root" message = f"rolled-up dependency cycle in {label}: {' -> '.join(cycle)}" - if message not in issues: + if message not in seen_issues: + seen_issues.add(message) issues.append(message) - stack.pop() - state[article_id] = 2 - - for article_id in sorted(dependencies): - if state.get(article_id, 0) == 0: - visit(article_id) return issues diff --git a/autoform_cli/graph_views.py b/autoform_cli/graph_views.py index 6b690784..642b01d5 100644 --- a/autoform_cli/graph_views.py +++ b/autoform_cli/graph_views.py @@ -480,10 +480,16 @@ def _direct_child(graph: Graph, scope: str, node_id: str) -> str | None: def _leaf_descendants(graph: Graph, node_id: str) -> tuple[str, ...]: - children = graph.children(node_id) - if not children: - return (node_id,) - return tuple(leaf for child in children for leaf in _leaf_descendants(graph, child)) + leaves: list[str] = [] + pending = [node_id] + while pending: + current = pending.pop() + children = graph.children(current) + if children: + pending.extend(reversed(children)) + else: + leaves.append(current) + return tuple(leaves) __all__ = [ diff --git a/autoform_cli/render.py b/autoform_cli/render.py index 7bc5894c..0b0c2daf 100644 --- a/autoform_cli/render.py +++ b/autoform_cli/render.py @@ -645,20 +645,21 @@ def _book_page_order(blueprint: Path, destination: Path, graph: Graph) -> list[P for node in graph.nodes.values() if graph.children(node.id) or not node.formalizable } - - def visit(source: Path) -> None: - source = source.resolve() + pending = [blueprint / "README.md"] + while pending: + source = pending.pop().resolve() try: relative = source.relative_to(blueprint) except ValueError: - return + continue output = (destination / relative).resolve() if output.is_file() and output not in seen_outputs: seen_outputs.add(output) ordered.append(output) if source in visited_sources or not source.is_file(): - return + continue visited_sources.add(source) + linked_sources: list[Path] = [] def collect(line: str) -> str: for match in _MARKDOWN_LINK.finditer(line): @@ -680,12 +681,11 @@ def collect(line: str) -> str: continue if candidate not in book_sources: continue - visit(candidate) + linked_sources.append(candidate) return line _outside_fences(source.read_text(encoding="utf-8"), collect) - - visit(blueprint / "README.md") + pending.extend(reversed(linked_sources)) return ordered diff --git a/autoform_cli/runtime.py b/autoform_cli/runtime.py index ec22dd8b..2435d883 100644 --- a/autoform_cli/runtime.py +++ b/autoform_cli/runtime.py @@ -9,8 +9,10 @@ import hashlib import json +from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path, PureWindowsPath +from types import MappingProxyType from urllib.parse import unquote, urlsplit from .graph import Graph, load_graph @@ -139,8 +141,14 @@ def as_dict(self) -> dict[str, object]: } +class _RuntimeGraphCache: + __slots__ = ("_nodes_by_id",) + + _nodes_by_id: Mapping[str, RuntimeNode] + + @dataclass(frozen=True, slots=True) -class RuntimeGraph: +class RuntimeGraph(_RuntimeGraphCache): """The complete versioned runtime view of an authored roadmap.""" schema: str @@ -154,10 +162,21 @@ class RuntimeGraph: dependency_count: int maximum_depth: int + def __post_init__(self) -> None: + index: dict[str, RuntimeNode] = {} + for node in self.nodes: + index.setdefault(node.id, node) + object.__setattr__(self, "_nodes_by_id", MappingProxyType(index)) + def get(self, node_id: str) -> RuntimeNode | None: """Return a node without exposing mutable lookup state.""" - return next((node for node in self.nodes if node.id == node_id), None) + try: + index = self._nodes_by_id + except AttributeError: + self.__post_init__() + index = self._nodes_by_id + return index.get(node_id) def as_dict(self) -> dict[str, object]: """Return a canonical JSON-compatible compatibility snapshot.""" @@ -247,6 +266,7 @@ def build_runtime_graph( _reject_roadmap_symlinks(blueprint) node_ids = set(graph.nodes) article_paths: dict[str, str] = {} + seen_article_paths: set[str] = set() revision_paths: dict[str, str] = {} article_bytes: dict[str, bytes] = {} @@ -281,9 +301,14 @@ def build_runtime_graph( except OSError: issues.append(f"{node.id}: article cannot be read") continue + if node.source_sha256 is None: + issues.append(f"{node.id}: article source digest is unavailable") + elif hashlib.sha256(content).hexdigest() != node.source_sha256: + issues.append(f"{node.id}: article changed after graph load") article_path = relative_project.as_posix() - if article_path in article_paths.values(): + if article_path in seen_article_paths: issues.append(f"{node.id}: article path is duplicated") + seen_article_paths.add(article_path) article_paths[node.id] = article_path revision_paths[node.id] = relative_blueprint.as_posix() article_bytes[node.id] = content @@ -393,7 +418,11 @@ def _reject_roadmap_symlinks(blueprint: Path) -> None: def _ordered_union(first: tuple[str, ...], second: tuple[str, ...]) -> tuple[str, ...]: values = list(first) - values.extend(value for value in second if value not in values) + seen = set(values) + for value in second: + if value not in seen: + values.append(value) + seen.add(value) return tuple(values) @@ -437,17 +466,45 @@ def _is_portable_relative_path(value: str) -> bool: def _validate_depths(graph: Graph, issues: list[str]) -> None: + resolved: dict[str, int] = {} for node_id in sorted(graph.nodes): node = graph.nodes[node_id] - seen: set[str] = set() - parent = node.parent - depth = 0 - while parent is not None: - if parent in seen or parent not in graph.nodes: - break - seen.add(parent) - depth += 1 - parent = graph.nodes[parent].parent + if node_id not in resolved: + trail: list[str] = [] + seen: set[str] = set() + current = node_id + valid = True + while current not in resolved: + if current in seen or current not in graph.nodes: + valid = False + break + seen.add(current) + trail.append(current) + parent = graph.nodes[current].parent + if parent is None: + depth = -1 + break + current = parent + else: + depth = resolved[current] + + if valid: + for candidate in reversed(trail): + depth += 1 + resolved[candidate] = depth + + if node_id in resolved: + depth = resolved[node_id] + else: + seen = set() + parent = node.parent + depth = 0 + while parent is not None: + if parent in seen or parent not in graph.nodes: + break + seen.add(parent) + depth += 1 + parent = graph.nodes[parent].parent if node.depth != depth: issues.append(f"{node.id}: depth does not match the parent chain") @@ -467,6 +524,9 @@ def _source_revision(article_paths: dict[str, str], article_bytes: dict[str, byt def _validate_runtime(runtime: RuntimeGraph) -> None: issues: list[str] = [] nodes = {node.id: node for node in runtime.nodes} + parents_with_children = { + node.parent for node in runtime.nodes if node.parent is not None + } if len(nodes) != len(runtime.nodes): issues.append("runtime node ids are not unique") for node in runtime.nodes: @@ -476,7 +536,7 @@ def _validate_runtime(runtime: RuntimeGraph) -> None: issues.append(f"{node.id}: runtime dependency union is inconsistent") if any(dependency not in nodes for dependency in node.dependencies): issues.append(f"{node.id}: runtime dependency does not resolve") - has_children = any(other.parent == node.id for other in runtime.nodes) + has_children = node.id in parents_with_children if node.dispatchable and (not node.formalizable or has_children): issues.append(f"{node.id}: dispatchable node is not a formalizable leaf") if Path(node.article_path).is_absolute() or PureWindowsPath(node.article_path).is_absolute(): diff --git a/autoform_cli/status.py b/autoform_cli/status.py index ce1da6c7..3eb9d489 100644 --- a/autoform_cli/status.py +++ b/autoform_cli/status.py @@ -165,26 +165,35 @@ def _classify( def topological_order(graph: Graph) -> list[str]: """Order nodes so every prerequisite precedes its dependents. - ``load_graph`` rejects cycles, so a plain depth-first walk suffices; the - ``visiting`` guard only protects callers who build a ``Graph`` by hand. + ``load_graph`` rejects cycles. The explicit stack keeps the same depth-first + order without depending on Python's recursion limit; the ``visiting`` guard + only protects callers who build a ``Graph`` by hand. """ order: list[str] = [] seen: set[str] = set() visiting: set[str] = set() - def visit(node_id: str) -> None: - if node_id in seen or node_id in visiting: - return - visiting.add(node_id) - for dependency in graph.nodes[node_id].dependencies: - if dependency in graph.nodes: - visit(dependency) - visiting.discard(node_id) - seen.add(node_id) - order.append(node_id) - - for node_id in sorted(graph.nodes): - visit(node_id) + for root_id in sorted(graph.nodes): + if root_id in seen: + continue + visiting.add(root_id) + frames = [(root_id, 0)] + while frames: + node_id, dependency_index = frames[-1] + dependencies = graph.nodes[node_id].dependencies + if dependency_index == len(dependencies): + frames.pop() + visiting.discard(node_id) + seen.add(node_id) + order.append(node_id) + continue + + dependency = dependencies[dependency_index] + frames[-1] = (node_id, dependency_index + 1) + if dependency not in graph.nodes or dependency in seen or dependency in visiting: + continue + visiting.add(dependency) + frames.append((dependency, 0)) return order diff --git a/tests/test_graph_scale.py b/tests/test_graph_scale.py new file mode 100644 index 00000000..15e387fe --- /dev/null +++ b/tests/test_graph_scale.py @@ -0,0 +1,452 @@ +from __future__ import annotations + +import base64 +import hashlib +import pickle +import random +from dataclasses import asdict, fields, replace +from pathlib import Path + +import pytest + +from autoform_cli.graph import ( + Graph, + Node, + _find_cycles, + _find_rollup_cycles, + _TrackedNodeDict, + load_graph, +) +from autoform_cli.graph_views import scope_view +from autoform_cli.render import _book_page_order +from autoform_cli.runtime import ( + RuntimeGraph, + RuntimeProjectionError, + _validate_depths, + _validate_runtime, + build_runtime_graph, + load_runtime_graph, +) +from autoform_cli.status import derive, topological_order + + +# Protocol-5 pickle produced by Graph/Node at parent commit d9e29c210b385b52889ff243422f2d0342778b60. +_PARENT_GRAPH_PICKLE = base64.b64decode( + "gAWVkgEAAAAAAACMEmF1dG9mb3JtX2NsaS5ncmFwaJSMBUdyYXBolJOUKYGUXZQojAdwYXRobGlilIwJUG9zaXhQYXRolJOUjA5s" + "ZWdhY3ktcHJvamVjdJSMCWJsdWVwcmludJSGlFKUfZQojAdyb2FkbWFwlGgAjAROb2RllJOUKYGUXZQoaA2MB1JvYWRtYXCUaAco" + "aAhoCWgNjAlSRUFETUUubWSUdJRSlCkpKYwHYXJ0aWNsZZROTomJiU5OiU5OKU5LAE6MQDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw" + "MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDCUZWKMBWNoaWxklGgPKYGUXZQoaBiMBUNoaWxklGgHKGgI" + "aAloDYwIY2hpbGQubWSUdJRSlCkpKWgWTk6JiYlOTolOTiloDUsBToxAMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTEx" + "MTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMZRlYnVlYi4=" +) + + +class _CountingDict(_TrackedNodeDict): + def __init__(self, values: dict[str, Node]) -> None: + super().__init__(values) + self.values_calls = 0 + self.lookups = 0 + + def __contains__(self, key: object) -> bool: + self.lookups += 1 + return super().__contains__(key) + + def __getitem__(self, key: str) -> Node: + self.lookups += 1 + return super().__getitem__(key) + + def values(self): + self.values_calls += 1 + return super().values() + + +class _CountingTuple(tuple): + def __new__(cls, values): + instance = super().__new__(cls, values) + instance.iterations = 0 + return instance + + def __iter__(self): + self.iterations += 1 + return super().__iter__() + + +def _chain_graph(tmp_path: Path, count: int, *, containment: bool = False) -> Graph: + roadmap = tmp_path / "blueprint" / "roadmap" + nodes: dict[str, Node] = {} + for index in range(count): + node_id = f"n{index:04d}" + next_id = f"n{index + 1:04d}" + dependencies = (next_id,) if not containment and index + 1 < count else () + parent = f"n{index - 1:04d}" if containment and index else None + nodes[node_id] = Node( + id=node_id, + title=node_id, + path=roadmap / f"{node_id}.md", + dependencies=dependencies, + statement_dependencies=dependencies, + parent=parent, + depth=index if containment else 0, + declaration="theorem" if containment and index + 1 == count else None, + ) + return Graph(tmp_path / "blueprint", nodes) + + +def test_graph_children_cache_tracks_public_mutations_without_changing_order(tmp_path: Path) -> None: + roadmap = tmp_path / "blueprint" / "roadmap" + nodes = _CountingDict( + { + "root": Node("root", "Root", roadmap / "README.md", ()), + "second": Node("second", "Second", roadmap / "second.md", (), parent="root"), + "first": Node("first", "First", roadmap / "first.md", (), parent="root"), + } + ) + graph = Graph(tmp_path / "blueprint", nodes) + revision = graph._children_revision + + assert tuple(field.name for field in fields(Graph)) == ("blueprint_dir", "nodes") + assert "_children_by_parent" not in repr(graph) + for _ in range(1_200): + assert graph.children("root") == ("second", "first") + assert graph.children("missing") == () + + assert graph._children_revision == revision + + graph.nodes["third"] = Node("third", "Third", roadmap / "third.md", (), parent="root") + assert graph.children("root") == ("second", "first", "third") + + graph.nodes["second"] = replace(graph.nodes["second"], parent=None) + assert graph.children("root") == ("first", "third") + + graph.nodes.pop("first") + assert graph.children("root") == ("third",) + + +def test_graph_children_cache_tracks_public_dict_reinitialization(tmp_path: Path) -> None: + roadmap = tmp_path / "blueprint" / "roadmap" + root = Node("root", "Root", roadmap / "README.md", ()) + old_child = Node("old", "Old", roadmap / "old.md", (), parent="root") + graph = Graph(tmp_path / "blueprint", {"root": root, "old": old_child}) + assert graph.children("root") == ("old",) + cached_revision = graph._children_revision + + new_child = Node("new", "New", roadmap / "new.md", (), parent="root") + graph.nodes.__init__({"old": replace(old_child, parent=None), "new": new_child}) + + assert graph.nodes == {"root": root, "old": replace(old_child, parent=None), "new": new_child} + assert graph.nodes.revision > cached_revision + assert graph.children("root") == ("new",) + + +def test_parent_format_graph_pickle_restores_cache_and_builds_runtime(tmp_path: Path) -> None: + graph = pickle.loads(_PARENT_GRAPH_PICKLE) + assert graph.children("roadmap") == ("child",) + + project = tmp_path / "project" + blueprint = project / "blueprint" + roadmap = blueprint / "roadmap" + roadmap.mkdir(parents=True) + sources = { + "roadmap": (roadmap / "README.md", b"# Roadmap\n"), + "child": (roadmap / "child.md", b"# Child\n"), + } + object.__setattr__(graph, "blueprint_dir", blueprint) + for node_id, (path, content) in sources.items(): + path.write_bytes(content) + graph.nodes[node_id] = replace( + graph.nodes[node_id], + path=path, + source_sha256=hashlib.sha256(content).hexdigest(), + ) + + runtime = build_runtime_graph(graph, project_root=project) + + assert runtime.get("child") is not None + assert runtime.get("child").parent == "roadmap" # type: ignore[union-attr] + + +@pytest.mark.parametrize("protocol", range(6)) +def test_current_graph_pickle_round_trips_at_every_supported_protocol( + tmp_path: Path, + protocol: int, +) -> None: + roadmap = tmp_path / "blueprint" / "roadmap" + root = Node("root", "Root", roadmap / "README.md", ()) + child = Node("child", "Child", roadmap / "child.md", (), parent="root") + graph = Graph(tmp_path / "blueprint", {"root": root, "child": child}) + graph.nodes["child"] = child + assert graph.children("root") == ("child",) + + restored = pickle.loads(pickle.dumps(graph, protocol=protocol)) + + assert restored == graph + assert repr(restored) == repr(graph) + assert restored.nodes.revision >= graph.nodes.revision + assert restored.children("root") == ("child",) + cached_revision = restored.nodes.revision + restored.nodes["child"] = replace(restored.nodes["child"], parent=None) + restored.nodes.__setstate__(cached_revision) + assert restored.nodes.revision > cached_revision + assert restored.children("root") == () + with pytest.raises(TypeError): + restored._children_by_parent["root"] = ("child",) # type: ignore[index] + + +def test_dependency_and_rollup_walks_handle_a_1200_node_chain(tmp_path: Path) -> None: + graph = _chain_graph(tmp_path, 1_200) + + assert _find_cycles(graph.nodes) == [] + assert _find_rollup_cycles(graph.nodes) == [] + order = topological_order(graph) + assert order[0] == "n1199" + assert order[-1] == "n0000" + assert len(derive(graph)) == 1_200 + + +def test_rollup_projection_is_subquadratic_on_a_deep_branching_hierarchy( + tmp_path: Path, +) -> None: + roadmap = tmp_path / "blueprint" / "roadmap" + raw_nodes: dict[str, Node] = {} + for index in range(400): + container = f"container{index:04d}" + leaf = f"leaf{index:04d}" + parent = f"container{index - 1:04d}" if index else None + dependencies = (f"leaf{index - 1:04d}",) if index else () + raw_nodes[container] = Node( + container, + container, + roadmap / container / "README.md", + dependencies, + parent=parent, + ) + raw_nodes[leaf] = Node(leaf, leaf, roadmap / f"{leaf}.md", (), parent=container) + nodes = _CountingDict(raw_nodes) + + assert _find_rollup_cycles(nodes) == [] + assert nodes.values_calls <= 2 + assert nodes.lookups < 20 * len(nodes) * len(nodes).bit_length() + + +def _reference_rollup_cycles(nodes: dict[str, Node]) -> list[str]: + children: dict[str | None, list[str]] = {} + for node in nodes.values(): + children.setdefault(node.parent, []).append(node.id) + + def direct_child(scope: str | None, node_id: str) -> str | None: + current = node_id + while nodes[current].parent != scope: + parent = nodes[current].parent + if parent is None: + return None + current = parent + return current + + issues: list[str] = [] + for scope, siblings in children.items(): + if len(siblings) < 2: + continue + dependencies = {sibling: set() for sibling in siblings} + for target in nodes.values(): + target_child = direct_child(scope, target.id) + if target_child not in dependencies: + continue + for dependency in target.dependencies: + source_child = direct_child(scope, dependency) + if source_child in dependencies and source_child != target_child: + dependencies[target_child].add(source_child) + state: dict[str, int] = {} + stack: list[str] = [] + + def visit(article_id: str) -> None: + state[article_id] = 1 + stack.append(article_id) + for prerequisite in sorted(dependencies[article_id]): + if state.get(prerequisite, 0) == 0: + visit(prerequisite) + elif state.get(prerequisite) == 1: + start = stack.index(prerequisite) + cycle = stack[start:] + [prerequisite] + label = scope or "root" + message = f"rolled-up dependency cycle in {label}: {' -> '.join(cycle)}" + if message not in issues: + issues.append(message) + stack.pop() + state[article_id] = 2 + + for article_id in sorted(dependencies): + if state.get(article_id, 0) == 0: + visit(article_id) + return issues + + +def test_rollup_projection_preserves_exact_randomized_cycle_diagnostics(tmp_path: Path) -> None: + generator = random.Random(87231) + roadmap = tmp_path / "blueprint" / "roadmap" + for case in range(300): + node_ids = [f"n{index:02d}" for index in range(generator.randrange(1, 28))] + nodes: dict[str, Node] = {} + for index, node_id in enumerate(node_ids): + parent = None if index == 0 or generator.random() < 0.22 else node_ids[generator.randrange(index)] + dependencies = tuple( + candidate for candidate in node_ids if candidate != node_id and generator.random() < 0.07 + ) + nodes[node_id] = Node( + node_id, + node_id, + roadmap / f"{node_id}.md", + dependencies, + parent=parent, + ) + + assert _find_rollup_cycles(nodes) == _reference_rollup_cycles(nodes), case + + +def test_runtime_depth_validation_is_linear_on_a_deep_hierarchy(tmp_path: Path) -> None: + graph = _chain_graph(tmp_path, 1_200, containment=True) + nodes = _CountingDict(graph.nodes) + graph = Graph(graph.blueprint_dir, nodes) + issues: list[str] = [] + + _validate_depths(graph, issues) + + assert issues == [] + assert nodes.lookups < 10 * len(nodes) + + +def test_scope_view_handles_a_1200_level_containment_chain(tmp_path: Path) -> None: + graph = _chain_graph(tmp_path, 1_200, containment=True) + + view = scope_view(graph, derive(graph), "n0000", include_external=False) + + assert view.member_ids == ("n1199",) + + +def test_book_order_handles_a_1200_page_link_chain(tmp_path: Path) -> None: + blueprint = tmp_path / "blueprint" + roadmap = blueprint / "roadmap" + roadmap.mkdir(parents=True) + (blueprint / "README.md").write_text( + "# Book\n\n[First](roadmap/page0000.md)\n", + encoding="utf-8", + ) + nodes: dict[str, Node] = {} + for index in range(1_200): + node_id = f"page{index:04d}" + path = roadmap / f"{node_id}.md" + next_link = f"\n[Next](page{index + 1:04d}.md)\n" if index + 1 < 1_200 else "" + path.write_text(f"# {node_id}\n{next_link}", encoding="utf-8") + nodes[node_id] = Node(node_id, node_id, path, ()) + graph = Graph(blueprint, nodes) + + ordered = _book_page_order(blueprint, blueprint, graph) + + assert len(ordered) == 1_201 + assert ordered[0] == blueprint / "README.md" + assert ordered[-1] == roadmap / "page1199.md" + + +def test_runtime_lookup_does_not_rescan_the_node_tuple(tmp_path: Path) -> None: + project = tmp_path / "project" + roadmap = project / "blueprint" / "roadmap" + roadmap.mkdir(parents=True) + (roadmap / "item.md").write_text("# Item\n", encoding="utf-8") + runtime = load_runtime_graph(project) + nodes = _CountingTuple(runtime.nodes) + runtime = replace(runtime, nodes=nodes) + scans_after_construction = nodes.iterations + + for _ in range(1_200): + assert runtime.get("item") is nodes[0] + assert runtime.get("missing") is None + + assert nodes.iterations == scans_after_construction + + +def test_runtime_lookup_cache_preserves_the_public_dataclass_contract(tmp_path: Path) -> None: + project = tmp_path / "project" + roadmap = project / "blueprint" / "roadmap" + roadmap.mkdir(parents=True) + (roadmap / "item.md").write_text("# Item\n", encoding="utf-8") + runtime = load_runtime_graph(project) + field_names = ( + "schema", + "authority", + "source_revision", + "blueprint_path", + "nodes", + "article_count", + "formalizable_count", + "dispatchable_count", + "dependency_count", + "maximum_depth", + ) + + assert tuple(field.name for field in fields(RuntimeGraph)) == field_names + assert set(asdict(runtime)) == set(field_names) + assert "_nodes_by_id" not in repr(runtime) + reconstructed = RuntimeGraph(*(getattr(runtime, name) for name in field_names)) + assert reconstructed == runtime + assert repr(reconstructed) == repr(runtime) + with pytest.raises(TypeError): + runtime._nodes_by_id["replacement"] = runtime.nodes[0] # type: ignore[index] + + +def test_runtime_validation_does_not_scan_for_children_per_node(tmp_path: Path) -> None: + project = tmp_path / "project" + roadmap = project / "blueprint" / "roadmap" + roadmap.mkdir(parents=True) + (roadmap / "item.md").write_text("# Item\n", encoding="utf-8") + runtime = load_runtime_graph(project) + base = runtime.nodes[0] + nodes = _CountingTuple( + replace( + base, + id=f"n{index:04d}", + article_path=f"blueprint/roadmap/n{index:04d}.md", + parent=f"n{index - 1:04d}" if index else None, + depth=index, + ) + for index in range(1_200) + ) + runtime = replace( + runtime, + nodes=nodes, + article_count=len(nodes), + formalizable_count=0, + dispatchable_count=0, + dependency_count=0, + maximum_depth=len(nodes) - 1, + ) + scans_after_construction = nodes.iterations + + _validate_runtime(runtime) + + assert nodes.iterations - scans_after_construction < 10 + + +def test_runtime_rejects_article_bytes_changed_after_graph_load(tmp_path: Path) -> None: + project = tmp_path / "project" + roadmap = project / "blueprint" / "roadmap" + roadmap.mkdir(parents=True) + article = roadmap / "item.md" + article.write_text("# Item\n\nOriginal.\n", encoding="utf-8") + graph = load_graph(project / "blueprint") + article.write_text("# Item\n\nChanged.\n", encoding="utf-8") + + with pytest.raises(RuntimeProjectionError, match="article changed after graph load"): + build_runtime_graph(graph, project_root=project) + + +def test_runtime_rejects_a_graph_node_without_a_source_digest(tmp_path: Path) -> None: + project = tmp_path / "project" + roadmap = project / "blueprint" / "roadmap" + roadmap.mkdir(parents=True) + (roadmap / "item.md").write_text("# Item\n", encoding="utf-8") + graph = load_graph(project / "blueprint") + graph.nodes["item"] = replace(graph.nodes["item"], source_sha256=None) + + with pytest.raises(RuntimeProjectionError) as error: + build_runtime_graph(graph, project_root=project) + + assert error.value.issues == ("item: article source digest is unavailable",) diff --git a/tests/test_runtime.py b/tests/test_runtime.py index e31c571b..d2b5d6b0 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -273,6 +273,7 @@ def test_adapter_rejects_inconsistent_hand_built_graph_without_host_paths(tmp_pa build_runtime_graph(graph, project_root=project) assert error.value.issues == ( + "chapter/section/base: article source digest is unavailable", "chapter/section/base: dependency does not name a runtime node: missing", "chapter/section/base: dependency union does not match typed dependencies", ) From ceca8e87cc69265dce3ba230191bb640fb47c208 Mon Sep 17 00:00:00 2001 From: Jack McCarthy Date: Sun, 30 Aug 2026 07:43:38 -0400 Subject: [PATCH 2/2] [autoform] Restore Graph pickles on Python 3.10 --- autoform_cli/graph.py | 21 ++++++++++++++------- tests/test_graph_scale.py | 1 + 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/autoform_cli/graph.py b/autoform_cli/graph.py index 134bd1d8..b67bec89 100644 --- a/autoform_cli/graph.py +++ b/autoform_cli/graph.py @@ -172,13 +172,6 @@ def __post_init__(self) -> None: object.__setattr__(self, "nodes", _TrackedNodeDict(self.nodes)) self._refresh_children() - def __setstate__(self, state: list[object]) -> None: - """Restore legacy slot pickles through the current cache initializer.""" - blueprint_dir, nodes = state - object.__setattr__(self, "blueprint_dir", blueprint_dir) - object.__setattr__(self, "nodes", nodes) - self.__post_init__() - def _refresh_children(self) -> None: children: dict[str | None, list[str]] = {} for node in self.nodes.values(): @@ -201,6 +194,20 @@ def children(self, node_id: str) -> tuple[str, ...]: return self._children_by_parent.get(node_id, ()) +def _restore_graph_state(graph: Graph, state: list[object]) -> None: + """Restore legacy slot pickles through the current cache initializer.""" + blueprint_dir, nodes = state + object.__setattr__(graph, "blueprint_dir", blueprint_dir) + object.__setattr__(graph, "nodes", nodes) + graph.__post_init__() + + +# Python 3.10's ``dataclass(slots=True, frozen=True)`` replaces a class-defined +# pickle hook. Installing it after decoration keeps old Graph pickles compatible +# on every supported interpreter. +setattr(Graph, "__setstate__", _restore_graph_state) + + @dataclass(frozen=True, slots=True) class _ParsedNode: id: str diff --git a/tests/test_graph_scale.py b/tests/test_graph_scale.py index 15e387fe..2e43657d 100644 --- a/tests/test_graph_scale.py +++ b/tests/test_graph_scale.py @@ -140,6 +140,7 @@ def test_graph_children_cache_tracks_public_dict_reinitialization(tmp_path: Path def test_parent_format_graph_pickle_restores_cache_and_builds_runtime(tmp_path: Path) -> None: graph = pickle.loads(_PARENT_GRAPH_PICKLE) + assert isinstance(graph.nodes, _TrackedNodeDict) assert graph.children("roadmap") == ("child",) project = tmp_path / "project"