From 258a1b91e47672c4e73078855c0543e1b4005dcc Mon Sep 17 00:00:00 2001 From: mikim <1441941+mikim@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:35:10 +0900 Subject: [PATCH] docs: reframe ARK as a concept-complete reference/idea repo Position ARK honestly as a reference implementation rather than a product in progress, and add a runnable example so the concept is verifiable from a checkout. No package behavior changes. - README: add a top "reference implementation / idea repo" status callout; replace the multi-phase Roadmap + Success Metrics with one "Status & scope" section (implemented / ideas demonstrated / explicitly out of scope); Quickstart no longer implies an npm release; "Initial Target Users" -> "Who might find it useful". - examples/pipeline: a runnable, deterministic end-to-end demo (sanitize -> classify -> policy -> report) wired to `pnpm demo`; its README output matches the actual run byte-for-byte. - Declare the time-based stale-noise gate out of scope (the burst gate already demonstrates noise suppression) and align the earmarked-but-unused recentWindow timestamp docs in @ark/core and the schema so they no longer claim stale-noise detection is implemented. - Resolve framing contradictions the reframe exposed in adjacent docs: SECURITY.md no longer asserts published @ark/* npm releases; CONTRIBUTING.md and ARCHITECTURE.md frame the release tooling as configured-but-dormant. Verified: `pnpm check` (typecheck + lint + schema + tests) green; `pnpm demo` runs and matches its documented output. Co-Authored-By: Claude Opus 4.8 --- CONTRIBUTING.md | 10 ++++- README.md | 82 ++++++++++++++++------------------ SECURITY.md | 5 ++- docs/ARCHITECTURE.md | 5 +-- examples/pipeline/README.md | 41 +++++++++++++++++ examples/pipeline/package.json | 18 ++++++++ examples/pipeline/src/index.ts | 55 +++++++++++++++++++++++ package.json | 3 +- packages/core/src/index.ts | 6 ++- pnpm-lock.yaml | 18 ++++++++ schemas/runtime-event.v2.json | 2 +- 11 files changed, 191 insertions(+), 54 deletions(-) create mode 100644 examples/pipeline/README.md create mode 100644 examples/pipeline/package.json create mode 100644 examples/pipeline/src/index.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cd566c2..4d95cbf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,7 @@ # Contributing to ARK -Thanks for helping make agent reliability a product capability. This is a +Thanks for helping explore reliability patterns for AI agent runtimes. ARK is a +reference implementation (see [Status & scope](./README.md#status--scope)), and a TypeScript monorepo managed with **pnpm workspaces**. ## Prerequisites @@ -29,7 +30,7 @@ Useful scripts (root): | `pnpm lint:fix` | Apply safe Biome fixes + format | | `pnpm schema:validate`| Validate `schemas/examples/*` against the schema | | `pnpm test` | Build, then run every package's `node --test` | -| `pnpm changeset` | Record a user-facing change for release | +| `pnpm changeset` | Record a user-facing change (Changesets) | ## Toolchain @@ -75,6 +76,11 @@ changes bump the minor and reuse the file; breaking changes publish a new ## Releasing +> ARK is a reference implementation and is **not currently published** to npm +> (see [Status & scope](./README.md#status--scope)). The release tooling below is +> configured but dormant; the notes are kept for anyone who forks ARK to publish +> their own line. + Releases use **Changesets** and publish to npm via **OIDC trusted publishing** (no `NPM_TOKEN`). Merging the “Version Packages” PR publishes the bumped packages with provenance attestations. diff --git a/README.md b/README.md index edf741f..8ddc7de 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,15 @@ ![TypeScript](https://img.shields.io/badge/TypeScript-ESM-blue) [![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](./LICENSE) -**Agent Reliability Kit (ARK)** is a reliability layer for AI agent products. +**Agent Reliability Kit (ARK)** is a reliability layer for AI agent runtimes. It helps teams prevent silent failures, reduce noisy alerts, and turn runtime incidents into actionable operational signals. +> **Status — reference implementation / idea repo.** The full pipeline is +> implemented and test-backed, meant to be read, run, and adapted. It is *not* +> a maintained product: it is not published to npm, and product-scale work +> (framework adapters, dashboards, a release cadence) is deliberately +> [out of scope](#status--scope). + ## Why ARK AI agents are moving from demos to production, but reliability practices are still fragmented. Modern agent systems fail in ways that are hard to diagnose quickly: @@ -18,7 +24,7 @@ AI agents are moving from demos to production, but reliability practices are sti - poor incident taxonomy in logs and alerts - high MTTR due to missing context at failure time -Most teams patch these ad hoc per repository. ARK centralizes the patterns into reusable primitives — aiming to be for agent reliability what structured logging and APM became for web services. +Most teams patch these ad hoc per repository. ARK centralizes the patterns into reusable primitives — a sketch of what a reliability layer for agent runtimes could look like, the way structured logging and APM did for web services. ## Core Modules @@ -32,9 +38,9 @@ All modules are TypeScript, ESM-only, and compose independently. ## Quickstart -> **Not yet published to npm.** Until the first release, use the `@ark/*` -> packages from a checkout of this repo (they are pnpm workspaces). After the -> first release: `pnpm add @ark/classify @ark/policy @ark/report`. +> **Not published to npm** — this is a reference implementation. Use the +> `@ark/*` packages from a checkout of this repo (they are pnpm workspaces). +> For a version you can run as-is, see the [runnable example](examples/pipeline). ```ts import { classifyEvent } from "@ark/classify"; @@ -57,7 +63,9 @@ console.log(formatHumanSummary(report)); // [low] rate_limit — retry with backoff (attempt 1/3, 500ms) · confidence 90% ``` -See [`examples/mcp-server`](examples/mcp-server) to expose the same pipeline as +[`examples/pipeline`](examples/pipeline) runs this exact flow end-to-end from a +checkout (`sanitize → classify → policy → report`). See +[`examples/mcp-server`](examples/mcp-server) to expose the same pipeline as [Model Context Protocol](docs/MCP.md) tools. ## Input Model @@ -99,65 +107,51 @@ and JSON incident artifacts can serve as runtime evidence. - **Composable and incremental** — adopt only what you need; integrate without an architecture rewrite. - **Automation-first outputs** — machine-readable artifacts for CI/CD, monitoring, and postmortem workflows. -### Non-goals (for now) +### Non-goals - replacing existing APM/logging stacks -- abstracting every provider-specific edge case in v1 +- abstracting every provider-specific edge case - acting as a full workflow orchestrator -## Initial Target Users +## Who might find it useful - AI product teams shipping agent features - OSS maintainers handling agent-runtime bug reports - platform/ops teams responsible for production incident hygiene -## Roadmap - -### Phase 1 — Foundation ✅ - -- [x] event schema definition (runtime-event v2) -- [x] sanitizer primitives -- [x] baseline incident taxonomy -- [x] JSON report generator - -### Phase 2 — Runtime Policies (in progress) - -- [x] retry/fallback/fail-fast decision engine -- [x] risk-tier scoring rules -- [x] burst gate — _stale-noise (time-based) gate still pending_ +## Status & scope -### Phase 3 — Integrations (planned) +A **reference implementation**: the reliability pipeline is complete and +test-backed, meant to be read, run, and adapted rather than shipped as a product. -- [x] MCP server example -- [ ] GitHub Actions incident report formatter -- [ ] dashboard-ready summary exports -- [ ] trend comparison between release windows +**Implemented (concept-complete):** -### Under exploration — Runtime Failure Conformance +- runtime-event schema v2, aligned with the OpenTelemetry GenAI conventions +- `@ark/sanitize` — payload normalization + preflight guards +- `@ark/classify` — deterministic incident taxonomy, risk tiers, confidence, and the burst gate +- `@ark/policy` — retry/fallback/fail-fast engine with decision traces +- `@ark/report` — human summary + JSON incident artifact +- a runnable end-to-end example and an MCP integration example -A complementary direction: treat ARK as a local, deterministic **conformance -kit** that verifies, from portable traces and injected failures, that an agent -runtime honors retry, idempotency, deadline, tool-result, and privacy contracts. -This would add a cross-span **rules** layer (evidence-based invariants) and an -`ark probe` fault injector (429 + `Retry-After`, timeouts, stream aborts, ACK -loss, duplicate tool calls) on top of the current primitives. Tracked as a -research track, not yet committed scope. +**The ideas it means to demonstrate:** -## Success Metrics +- suppressing false-positive alert volume and repeated provider-4xx noise +- faster incident triage (MTTR) and more reproducible bug reports across teams -ARK should create measurable outcomes: +**Out of scope — deliberately, to keep this a focused idea repo:** -- lower provider 4xx repeat rates -- lower false-positive alert volume -- faster incident triage (MTTR reduction) -- higher reproducibility of bug reports across teams +- publishing to npm or maintaining a release cadence +- adapters for specific agent frameworks, dashboards, or trend analytics +- a time-based stale-noise gate (the burst gate already demonstrates the idea) +- a runtime-failure **conformance** kit (`ark probe`, evidence-based rules) — the + assurance direction lives in the sibling [Agentic Assurance Profile](https://github.com/MosslandOpenDevs/agentic-assurance-profile) ## Repository layout ```text packages/ core, sanitize, classify, policy, report (see Core Modules) schemas/ runtime-event JSON Schema (+ examples, validated in CI) -examples/ runnable integrations (MCP server) +examples/ runnable examples (end-to-end pipeline, MCP server) docs/ architecture and integration notes ``` @@ -165,7 +159,7 @@ docs/ architecture and integration notes TypeScript, ESM-only, [pnpm](https://pnpm.io) workspaces. Built with [tsdown](https://tsdown.dev), linted/formatted with [Biome](https://biomejs.dev), -tested with the built-in `node:test` runner, released with +tested with the built-in `node:test` runner, versioned with [Changesets](https://github.com/changesets/changesets). ```bash diff --git a/SECURITY.md b/SECURITY.md index 13d83be..9d4dd7d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,8 +2,9 @@ ## Supported versions -ARK is pre-1.0. Security fixes are applied to the latest `main` and the most -recent published release of each `@ark/*` package. +ARK is a reference implementation and is not published to npm, so there are no +released package versions to patch separately. Security fixes are applied to the +latest `main`. ## Reporting a vulnerability diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0fd9c8e..69dcec8 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -58,6 +58,5 @@ Runtime Event ─▶ sanitize ─▶ classify ─▶ policy ─▶ report ## Toolchain pnpm workspaces · TypeScript (ESM-only, erasable syntax) · tsdown (Rolldown) -builds · Biome lint/format · `node:test` · Changesets + npm OIDC trusted -publishing. CI runs typecheck, lint, the schema gate, and the test matrix on -Node 22/24/26. +builds · Biome lint/format · `node:test` · Changesets for versioning. CI runs +typecheck, lint, the schema gate, and the test matrix on Node 22/24/26. diff --git a/examples/pipeline/README.md b/examples/pipeline/README.md new file mode 100644 index 0000000..1827724 --- /dev/null +++ b/examples/pipeline/README.md @@ -0,0 +1,41 @@ +# Example: end-to-end pipeline + +Runs one runtime failure through the whole Agent Reliability Kit pipeline — +`sanitize → classify → policy → report` — and prints the human summary plus the +JSON incident artifact. Everything is deterministic (fixed timestamps, no clock +reads), so the output is stable across runs. + +## Run it + +From the repository root: + +```bash +pnpm install +pnpm demo +``` + +`pnpm demo` builds the `@ark/*` packages and then runs +[`src/index.ts`](src/index.ts) with Node's built-in TypeScript support. + +## Expected output + +```text +1) sanitize → [{"role":"user","content":[{"type":"text","text":"Refund order #1234 please"}]}] +2) classify → rate_limit / low / 0.9 +3) policy → retry {"attempt":1,"maxRetries":3,"delayMs":500} +4) report → [low] rate_limit — retry with backoff (attempt 1/3, 500ms) · confidence 90% + +JSON artifact: +{ + "incidentReason": "rate_limit", + "riskTier": "low", + "confidence": 0.9, + "recommendedActions": [ + "Retry the request using the provided backoff delay." + ], + "summary": "[low] rate_limit — retry with backoff (attempt 1/3, 500ms)", + "action": "retry", + "schemaVersion": "2.0.0", + "generatedAt": "2026-07-25T00:00:00.000Z" +} +``` diff --git a/examples/pipeline/package.json b/examples/pipeline/package.json new file mode 100644 index 0000000..0bebf7d --- /dev/null +++ b/examples/pipeline/package.json @@ -0,0 +1,18 @@ +{ + "name": "@ark/example-pipeline", + "version": "0.0.0", + "private": true, + "description": "Runnable end-to-end example: sanitize -> classify -> policy -> report.", + "type": "module", + "license": "Apache-2.0", + "scripts": { + "start": "node src/index.ts" + }, + "dependencies": { + "@ark/classify": "workspace:*", + "@ark/core": "workspace:*", + "@ark/policy": "workspace:*", + "@ark/report": "workspace:*", + "@ark/sanitize": "workspace:*" + } +} diff --git a/examples/pipeline/src/index.ts b/examples/pipeline/src/index.ts new file mode 100644 index 0000000..fcb356f --- /dev/null +++ b/examples/pipeline/src/index.ts @@ -0,0 +1,55 @@ +/** + * Runnable end-to-end example for the Agent Reliability Kit. + * + * One runtime failure walked through the whole pipeline: + * + * sanitize -> classify -> policy -> report + * + * Everything here is deterministic (fixed timestamps, no clock reads), so the + * output is stable across runs. From the repo root: + * + * pnpm install + * pnpm demo + */ + +import { classifyEvent } from "@ark/classify"; +import type { RuntimeEvent } from "@ark/core"; +import { evaluatePolicy } from "@ark/policy"; +import { buildIncidentReport, formatHumanSummary } from "@ark/report"; +import { runPreflightGuards } from "@ark/sanitize"; + +// 1. sanitize — clean a messy request payload before it leaves the process: +// HTML tags stripped, extra whitespace collapsed, the empty message dropped. +const sanitized = runPreflightGuards( + { + messages: [ + { role: "user", content: " Refund order #1234 please " }, + { role: "user", content: " " }, + ], + }, + { provider: "openai", stripHtmlTags: true, collapseMergedWhitespace: true }, +); +console.log("1) sanitize →", JSON.stringify(sanitized.messages)); + +// 2. classify — map a runtime failure onto the incident taxonomy (deterministic). +const event: RuntimeEvent = { + timestamp: "2026-07-25T00:00:00.000Z", + app: "checkout-agent", + phase: "error", + provider: "openai", + error: { type: "RateLimitError", status: 429 }, +}; +const classification = classifyEvent(event); +console.log( + "2) classify →", + `${classification.incidentReason} / ${classification.riskTier} / ${classification.confidence}`, +); + +// 3. policy — turn the classification into a concrete recovery action. +const decision = evaluatePolicy(classification, { attempt: 0 }); +console.log("3) policy →", `${decision.action} ${JSON.stringify(decision.retry)}`); + +// 4. report — one-line human summary plus a stable JSON artifact for automation. +const report = buildIncidentReport(classification, decision, { generatedAt: event.timestamp }); +console.log("4) report →", formatHumanSummary(report)); +console.log(`\nJSON artifact:\n${JSON.stringify(report, null, 2)}`); diff --git a/package.json b/package.json index ea3d25f..ddc2515 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "agent-reliability-kit", "version": "0.0.0", "private": true, - "description": "A reliability layer for AI agent products: sanitize, classify, policy, report.", + "description": "A reliability layer for AI agent runtimes: sanitize, classify, policy, report.", "type": "module", "license": "Apache-2.0", "repository": { @@ -19,6 +19,7 @@ }, "scripts": { "build": "pnpm -r --filter \"./packages/*\" run build", + "demo": "pnpm run build && pnpm --filter @ark/example-pipeline start", "typecheck": "tsc --noEmit", "lint": "biome check .", "lint:fix": "biome check --write .", diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 07f1846..240b291 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -60,7 +60,11 @@ export interface RuntimeError { readonly retryable?: boolean; } -/** Compact prior event used for burst / stale-noise detection. */ +/** + * Compact prior event used for burst detection. The `timestamp` is carried for + * correlation and is reserved for a future stale-noise gate (not yet used by + * {@link classifyEvent}). + */ export interface RecentEvent { readonly timestamp: string; readonly phase: IncidentPhase; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a7b33a5..920051d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -78,6 +78,24 @@ importers: specifier: 'catalog:' version: 1.29.0(zod@4.4.3) + examples/pipeline: + dependencies: + '@ark/classify': + specifier: workspace:* + version: link:../../packages/classify + '@ark/core': + specifier: workspace:* + version: link:../../packages/core + '@ark/policy': + specifier: workspace:* + version: link:../../packages/policy + '@ark/report': + specifier: workspace:* + version: link:../../packages/report + '@ark/sanitize': + specifier: workspace:* + version: link:../../packages/sanitize + packages/classify: dependencies: '@ark/core': diff --git a/schemas/runtime-event.v2.json b/schemas/runtime-event.v2.json index 55170b5..d441230 100644 --- a/schemas/runtime-event.v2.json +++ b/schemas/runtime-event.v2.json @@ -73,7 +73,7 @@ "error": { "$ref": "#/$defs/runtimeError" }, "recentWindow": { "type": "array", - "description": "Recent same-app events, newest last, for burst / stale-noise detection.", + "description": "Recent same-app events, newest last, for burst detection (timestamps reserved for a future stale-noise gate).", "items": { "type": "object", "additionalProperties": false,