diff --git a/CHANGELOG.md b/CHANGELOG.md index d3e96f0..804ad58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on Keep a Changelog and this project follows Semantic Versio ## [Unreleased] +- **Legacy Runs API cleanup** — removed the orphaned public `/runs` list/delete routes, their route-specific service, and route-only tests now that Results deletion uses `/results-view/runs/:runId`, while retaining the underlying run/result tables for active benchmark, evaluation, retention, and cleanup flows. - **Datasets editor checkpoint** — added a Datasets page and JSONL dataset-file API for creating, editing, saving, and deleting dataset item files under `INFERHARNESS_BENCHMARK_DATASET_ROOT`, with synced `dataset_manifest` documents, copy-down editing for repeated fields, and clamped long-prompt display. - **Benchmark plan cleanup** — removed the transitional inline `/benchmark/plans/run` execution API and stale `INFERHARNESS_TEST_TEMPLATES_DIR` example so plan execution goes through persisted `benchmark_plan` documents. - **Tool-call assertion metric** — benchmark tool-call templates now include `tool_call_assertion_pass`, a single-turn pass/fail metric requiring exact expected tool selection and structurally matching arguments while keeping assertion failures as quality metrics rather than execution failures. diff --git a/backend/src/api/routes/runs.ts b/backend/src/api/routes/runs.ts deleted file mode 100644 index 1368233..0000000 --- a/backend/src/api/routes/runs.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { FastifyInstance } from 'fastify'; - -import { getDb } from '../../models/db.js'; -import { deleteRun } from '../../services/run-service.js'; - -export function registerRunsRoutes(app: FastifyInstance): void { - app.get('/runs', async () => { - const db = getDb(); - return db.prepare('SELECT * FROM runs ORDER BY started_at DESC').all(); - }); - - app.delete('/runs/:runId', async (request, reply) => { - const { runId } = request.params as { runId: string }; - const deleted = deleteRun(runId); - if (!deleted.ok) { - reply.code(deleted.code === 'RUN_ACTIVE' ? 409 : 404).send({ error: deleted.error }); - return; - } - reply.code(204).send(); - }); -} diff --git a/backend/src/api/server.ts b/backend/src/api/server.ts index 8460423..d21755d 100644 --- a/backend/src/api/server.ts +++ b/backend/src/api/server.ts @@ -6,7 +6,6 @@ import { fileURLToPath } from 'url'; import { getDb, resolvedDbPath, runSchema } from '../models/db.js'; import { registerAuth } from './middleware/auth.js'; import { registerResultsViewRoutes } from './routes/results-view.js'; -import { registerRunsRoutes } from './routes/runs.js'; import { registerInferenceServersRoutes } from './routes/inference-servers.js'; import { registerModelsRoutes } from './routes/models.js'; import { registerSystemRoutes } from './routes/system.js'; @@ -99,7 +98,6 @@ export function createServer() { registerSystemRoutes(app); registerInferenceServersRoutes(app); - registerRunsRoutes(app); registerModelsRoutes(app); registerResultsViewRoutes(app); registerEvalInferenceRoutes(app); diff --git a/backend/src/services/run-service.ts b/backend/src/services/run-service.ts deleted file mode 100644 index 83b873f..0000000 --- a/backend/src/services/run-service.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { getDb } from '../models/db.js'; -import { parseJson } from '../models/repositories.js'; -import { logEvent } from './observability.js'; - -export interface RunRecord { - id: string; - inference_server_id: string; - suite_id: string | null; - test_id: string | null; - profile_id: string | null; - profile_version: string | null; - status: string; - started_at: string; - ended_at: string | null; - environment_snapshot: Record | null; - retention_days: number | null; -} - -export type DeleteRunResult = - | { ok: true } - | { ok: false; code: 'RUN_NOT_FOUND' | 'RUN_ACTIVE'; error: string }; - -export function getRun(id: string): RunRecord | null { - const db = getDb(); - const row = db.prepare('SELECT * FROM runs WHERE id = ?').get(id) as RunRecord | undefined; - if (!row) { - return null; - } - return { - ...row, - environment_snapshot: parseJson(row.environment_snapshot as unknown as string) - }; -} - -export function deleteRun(id: string): DeleteRunResult { - const db = getDb(); - const row = db.prepare('SELECT id, status FROM runs WHERE id = ?').get(id) as { id: string; status: string } | undefined; - if (!row) { - return { ok: false, code: 'RUN_NOT_FOUND', error: 'Run not found' }; - } - if (row.status === 'queued' || row.status === 'running') { - return { ok: false, code: 'RUN_ACTIVE', error: 'Active runs cannot be deleted' }; - } - - const transaction = db.transaction((runId: string) => { - const resultRows = db.prepare('SELECT id FROM test_results WHERE run_id = ?').all(runId) as Array<{ id: string }>; - const resultIds = resultRows.map((result) => result.id); - - if (resultIds.length > 0) { - const placeholders = resultIds.map(() => '?').join(','); - db.prepare(`DELETE FROM metric_samples WHERE test_result_id IN (${placeholders})`).run(...resultIds); - db.prepare(`DELETE FROM evaluation_queue_skips WHERE test_result_id IN (${placeholders})`).run(...resultIds); - db.prepare(`DELETE FROM test_result_documents WHERE test_result_id IN (${placeholders})`).run(...resultIds); - } - - db.prepare('DELETE FROM test_results WHERE run_id = ?').run(runId); - db.prepare('DELETE FROM runs WHERE id = ?').run(runId); - }); - - transaction(id); - logEvent({ - level: 'info', - message: 'run deleted', - meta: { run_id: id } - }); - return { ok: true }; -} diff --git a/backend/tests/integration/runs-history.test.ts b/backend/tests/integration/runs-history.test.ts deleted file mode 100644 index 468e201..0000000 --- a/backend/tests/integration/runs-history.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -import fs from 'fs'; -import path from 'path'; -import { fileURLToPath } from 'url'; - -import { describe, expect, it } from 'vitest'; - -import { createServer } from '../../src/api/server.js'; -import { runSchema } from '../../src/models/db.js'; - - -describe('run history API', () => { - it('lists runs', async () => { - process.env.INFERHARNESS_API_TOKEN = 'test-token'; - process.env.INFERHARNESS_DB_PATH = ':memory:'; - const moduleDir = path.dirname(fileURLToPath(import.meta.url)); - const schemaPath = path.resolve(moduleDir, '../../src/models/schema.sql'); - runSchema(fs.readFileSync(schemaPath, 'utf8')); - const app = createServer(); - const response = await app.inject({ - method: 'GET', - url: '/runs', - headers: { 'x-api-token': 'test-token' } - }); - if (response.statusCode !== 200) { - throw new Error(`list runs failed: ${response.statusCode} ${response.body}`); - } - expect(response.statusCode).toBe(200); - }); -}); diff --git a/backend/tests/integration/runs-single.test.ts b/backend/tests/integration/runs-single.test.ts deleted file mode 100644 index 99f4911..0000000 --- a/backend/tests/integration/runs-single.test.ts +++ /dev/null @@ -1,175 +0,0 @@ -import fs from 'fs'; -import os from 'os'; -import path from 'path'; -import { fileURLToPath } from 'url'; - -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; - -import { createServer } from '../../src/api/server.js'; -import { getDb, resetDbInstance, runSchema } from '../../src/models/db.js'; - -const AUTH_HEADERS = { 'x-api-token': 'test-token' }; -const moduleDir = path.dirname(fileURLToPath(import.meta.url)); -const SCHEMA_PATH = path.resolve(moduleDir, '../../src/models/schema.sql'); - -function seedServer(serverId = 'srv-delete') { - const db = getDb(); - const now = '2026-05-01T00:00:00.000Z'; - db.prepare(` - INSERT INTO inference_servers ( - server_id, display_name, active, archived, created_at, updated_at, runtime, - endpoints, auth, capabilities, discovery, raw - ) VALUES (?, ?, 1, 0, ?, ?, ?, ?, ?, ?, ?, ?) - `).run( - serverId, - 'Delete Server', - now, - now, - JSON.stringify({ api: { api_version: '1.0.0' } }), - JSON.stringify({ base_url: 'http://localhost:8080' }), - JSON.stringify({}), - JSON.stringify({}), - JSON.stringify({ model_list: { normalised: [] } }), - JSON.stringify({}) - ); -} - -function seedRunWithDependencies(input: { runId: string; status?: string; resultId?: string }) { - const db = getDb(); - const now = '2026-05-01T10:00:00.000Z'; - const runId = input.runId; - const resultId = input.resultId ?? `result-${runId}`; - db.prepare(` - INSERT INTO runs ( - id, inference_server_id, suite_id, test_id, profile_id, profile_version, - status, started_at, ended_at, environment_snapshot, retention_days - ) VALUES (?, 'srv-delete', null, 'delete-test', null, null, ?, ?, ?, ?, 30) - `).run( - runId, - input.status ?? 'completed', - now, - now, - JSON.stringify({ effective_config: { model: 'model-delete' } }) - ); - db.prepare(` - INSERT INTO test_results ( - id, run_id, test_id, verdict, failure_reason, metrics, artefacts, raw_events, - repetition_stats, started_at, ended_at - ) VALUES (?, ?, 'delete-test', 'pass', null, ?, ?, '[]', ?, ?, ?) - `).run( - resultId, - runId, - JSON.stringify({ latency_ms: 42 }), - JSON.stringify({ response_body: 'ok' }), - JSON.stringify({ repetitions: 1 }), - now, - now - ); - db.prepare(` - INSERT INTO test_result_documents (test_result_id, run_id, test_id, schema_version, document, created_at) - VALUES (?, ?, 'delete-test', '1.0.0', ?, ?) - `).run(resultId, runId, JSON.stringify({ test: { tags: ['delete'] }, selected_model: { id: 'model-delete' } }), now); - db.prepare(` - INSERT INTO metric_samples (id, test_result_id, repetition_index, total_ms, created_at) - VALUES (?, ?, 0, 42, ?) - `).run(`metric-${runId}`, resultId, now); - db.prepare(` - INSERT INTO evaluation_queue_skips (test_result_id, reason, skipped_at) - VALUES (?, 'not useful', ?) - `).run(resultId, now); - db.prepare(` - INSERT INTO eval_prompts (id, text, tags, created_at) - VALUES (?, 'Prompt', '[]', ?) - `).run(`prompt-${runId}`, now); - db.prepare(` - INSERT INTO evaluations ( - id, prompt_id, model_name, server_id, inference_config, answer_text, - input_tokens, output_tokens, total_tokens, latency_ms, word_count, - estimated_cost, accuracy_score, relevance_score, coherence_score, - completeness_score, helpfulness_score, note, source_test_result_id, created_at - ) VALUES (?, ?, 'model-delete', 'srv-delete', '{}', 'Answer', 1, 1, 2, 42, 1, 0.001, 5, 5, 5, 5, 5, null, ?, ?) - `).run(`evaluation-${runId}`, `prompt-${runId}`, resultId, now); -} - -beforeEach(() => { - process.env.INFERHARNESS_API_TOKEN = 'test-token'; - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'aitb-runs-single-')); - process.env.INFERHARNESS_DB_PATH = path.join(tmpDir, 'test.sqlite'); - resetDbInstance(); - runSchema(fs.readFileSync(SCHEMA_PATH, 'utf8')); -}); - -afterEach(() => { - resetDbInstance(); -}); - -describe('runs API', () => { - it('deletes a completed run and its result-owned data', async () => { - seedServer(); - seedRunWithDependencies({ runId: 'run-delete' }); - const app = createServer(); - - const deleted = await app.inject({ - method: 'DELETE', - url: '/runs/run-delete', - headers: AUTH_HEADERS - }); - - expect(deleted.statusCode).toBe(204); - expect((await app.inject({ method: 'GET', url: '/results-view/runs/run-delete', headers: AUTH_HEADERS })).statusCode).toBe(404); - const listed = await app.inject({ method: 'GET', url: '/runs', headers: AUTH_HEADERS }); - expect(listed.json().some((run: { id: string }) => run.id === 'run-delete')).toBe(false); - const resultsView = await app.inject({ - method: 'POST', - url: '/results-view/query', - headers: AUTH_HEADERS, - payload: { - date_from: '2026-05-01T00:00:00.000Z', - date_to: '2026-05-02T00:00:00.000Z' - } - }); - expect(resultsView.json().history.total).toBe(0); - - const db = getDb(); - expect((db.prepare('SELECT COUNT(1) AS count FROM runs WHERE id = ?').get('run-delete') as { count: number }).count).toBe(0); - expect((db.prepare('SELECT COUNT(1) AS count FROM test_results WHERE run_id = ?').get('run-delete') as { count: number }).count).toBe(0); - expect((db.prepare('SELECT COUNT(1) AS count FROM test_result_documents WHERE run_id = ?').get('run-delete') as { count: number }).count).toBe(0); - expect((db.prepare('SELECT COUNT(1) AS count FROM metric_samples WHERE test_result_id = ?').get('result-run-delete') as { count: number }).count).toBe(0); - expect((db.prepare('SELECT COUNT(1) AS count FROM evaluation_queue_skips WHERE test_result_id = ?').get('result-run-delete') as { count: number }).count).toBe(0); - expect( - (db.prepare('SELECT source_test_result_id FROM evaluations WHERE id = ?').get('evaluation-run-delete') as { source_test_result_id: string | null }).source_test_result_id - ).toBeNull(); - }); - - it('returns 404 when deleting an unknown run', async () => { - seedServer(); - const app = createServer(); - - const response = await app.inject({ - method: 'DELETE', - url: '/runs/missing-run', - headers: AUTH_HEADERS - }); - - expect(response.statusCode).toBe(404); - expect(response.json().error).toBe('Run not found'); - }); - - it.each(['queued', 'running'])('blocks deleting %s runs', async (status) => { - seedServer(); - seedRunWithDependencies({ runId: `run-${status}`, status }); - const app = createServer(); - - const response = await app.inject({ - method: 'DELETE', - url: `/runs/run-${status}`, - headers: AUTH_HEADERS - }); - - expect(response.statusCode).toBe(409); - expect(response.json().error).toBe('Active runs cannot be deleted'); - const db = getDb(); - expect((db.prepare('SELECT COUNT(1) AS count FROM runs WHERE id = ?').get(`run-${status}`) as { count: number }).count).toBe(1); - expect((db.prepare('SELECT COUNT(1) AS count FROM test_results WHERE run_id = ?').get(`run-${status}`) as { count: number }).count).toBe(1); - }); -}); diff --git a/frontend/tests/e2e/results-dashboard.spec.ts b/frontend/tests/e2e/results-dashboard.spec.ts index 5148e5c..0495180 100644 --- a/frontend/tests/e2e/results-dashboard.spec.ts +++ b/frontend/tests/e2e/results-dashboard.spec.ts @@ -347,6 +347,12 @@ test('deletes a run from the run detail drawer after confirmation', async ({ pag }); await page.route('**/results-view/runs/run-dashboard-1', async (route) => { + if (route.request().method() === 'DELETE') { + deleteRequests += 1; + deleted = true; + await route.fulfill({ status: 204 }); + return; + } if (deleted) { await route.fulfill({ status: 404, @@ -375,16 +381,6 @@ test('deletes a run from the run detail drawer after confirmation', async ({ pag }); }); - await page.route('**/runs/run-dashboard-1', async (route) => { - if (route.request().method() !== 'DELETE') { - await route.fallback(); - return; - } - deleteRequests += 1; - deleted = true; - await route.fulfill({ status: 204 }); - }); - page.on('dialog', async (dialog) => { dialogCount += 1; if (dialogCount === 1) {