Repository navigation
Expand file tree
/
Copy pathMakefile
More file actions
93 lines (76 loc) · 3.66 KB
/
Copy pathMakefile
File metadata and controls
93 lines (76 loc) · 3.66 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
.PHONY: help install test quickstart test-contrib test-ts verify-page release-dry-run eval-scale lint build check demo demos eval eval-agents clean
PY ?= python3
help:
@echo "make install editable install with dev extras"
@echo "make test run the test suite"
@echo "make quickstart build the wheel into a fresh venv and run the v2 quickstart end to end (needs the package index)"
@echo "make test-contrib run the contrib/ packages' tests against the installed packages (pip install . ./contrib/*)"
@echo "make lint pyflakes over the package, SDK and tests"
@echo "make build build the sdist and wheel into dist/"
@echo "make check lint + test + build + twine check"
@echo "make verify-page build the browser verifier, one self-contained page: sdk/typescript/dist/verify.html"
@echo "make release-dry-run build every release artifact, SBOM and sum locally, publish nothing, and check them (needs the package index and npm)"
@echo "make demo run the scripted end-to-end demo"
@echo "make demos record the offline demos in docs/demos/ again (scripts/record_demos.py; --check checks them)"
@echo "make eval offline evaluations E1 (integrity), E2 (overhead), E3 (policy gate), E4 (seeded faults, v1 and v2), E6 (findings), E17 (server policy corpora, detection), E18 (agent approvers), E19 (egress observation), E20 (intent, reasoning and action checks), E21 (policy robustness: held-out calls, and shell rewrites), E23 (intent grants), on Linux E9 --quick (signer perf); rewrites eval/results/"
@echo "make eval-agents E5: real Claude Code runs (needs the claude CLI; spends model usage)"
@echo "make eval-scale E7: SQL index over a million-event ledger (contrib/query)"
@echo "E8 (insider attacks) needs root and a Linux system-mode signer; CI runs it, see eval/e8_insider.py"
@echo "E9 (signer perf, gates on Linux) and E15 (kill -9 durability, POSIX): eval/e9_signer_perf.py, eval/e15_durability.py"
@echo "make clean remove build artefacts"
install:
$(PY) -m pip install -e ".[dev]"
test:
$(PY) -m pytest -q
quickstart:
TRACEKIT_QUICKSTART=1 $(PY) -m pytest -q -s -m quickstart tests/test_quickstart.py
eval-scale:
$(PY) contrib/query/e7_sql_scale.py
test-contrib:
$(PY) -m pytest -q contrib
test-ts:
cd sdk/typescript && npm install --no-audit --no-fund && npm test
verify-page:
cd sdk/typescript && npm install --no-audit --no-fund && npm run build
@echo "browser verifier: sdk/typescript/dist/verify.html"
lint:
$(PY) -m pyflakes tracekit tracekit_sdk.py tests eval examples contrib integrations
build:
$(PY) -m pip install -q build twine
$(PY) -m build
$(PY) -m twine check dist/*
check: lint test build
release-dry-run:
$(PY) -m pip install -q -c scripts/release/constraints.txt build
TRACEKIT_RELEASE=1 $(PY) -m pytest -q tests/test_release.py
demo:
$(PY) -m tracekit demo
demos:
$(PY) scripts/record_demos.py
eval:
$(PY) eval/e1_integrity.py
$(PY) eval/e2_perf.py
$(PY) eval/e3_policy.py
$(PY) eval/e4_seeded_faults.py
$(PY) eval/e4_seeded_faults_v2.py
$(PY) eval/e6_findings.py
$(PY) eval/e10_reconcile.py
$(PY) eval/e11_l1_tampering.py
$(PY) eval/e12_cross_tenant.py
$(PY) eval/e14_outage.py --quick
$(PY) eval/e16_doctor.py
$(PY) eval/e17_policy_corpus.py
$(PY) eval/e17_detection.py
$(PY) eval/e17_containment.py
$(PY) eval/e18_agent_approvers.py
$(PY) eval/e19_egress.py
$(PY) eval/e20_alignment.py
$(PY) eval/e21_policy_robustness.py
$(PY) eval/e21_robustness.py
$(PY) eval/e23_grants.py
if [ "$$(uname)" = Linux ]; then $(PY) eval/e9_signer_perf.py --quick; fi
eval-agents:
E5=1 $(PY) eval/e5_agents.py
clean:
rm -rf build dist *.egg-info .pytest_cache
find . -name __pycache__ -type d -prune -exec rm -rf {} +