diff --git a/.github/workflows/toolkit.yml b/.github/workflows/toolkit.yml new file mode 100644 index 0000000..a12ebda --- /dev/null +++ b/.github/workflows/toolkit.yml @@ -0,0 +1,30 @@ +name: toolkit + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + validate: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: 24 + cache: npm + + - name: Install Dependencies + run: npm ci + + - name: Lint + run: npm run lint + + - name: Check formatting + run: npm run format:check + + - name: Verify CLI loads + run: node toolkit/bin/benchmark.js --help diff --git a/.gitignore b/.gitignore index 364b4e8..11fb668 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ node_modules/ target/ dist/ out/ +results/ .idea/ .env \ No newline at end of file diff --git a/README.md b/README.md index 0ae35de..294eef7 100644 --- a/README.md +++ b/README.md @@ -1 +1,288 @@ -# benchmark \ No newline at end of file +# benchmark + +API benchmark platform for comparing backend service implementations. Measures build time, deploy time, and load test performance across different tech stacks, then publishes results to Grafana Cloud for comparison. + +## Prerequisites + +- [Node.js](https://nodejs.org/) >= 22 +- [Docker](https://docs.docker.com/get-docker/) with Compose v2 +- [k6](https://grafana.com/docs/k6/latest/set-up/install-k6/) for load testing +- A [Grafana Cloud](https://grafana.com/products/cloud/) account (for publishing results) + +## Setup + +```bash +npm install +``` + +## Quick start + +```bash +# list configured targets +npm run benchmark -- list + +# run a full benchmark (build + deploy + loadtest + collect) +npm run benchmark -- run my-api + +# run and publish results to Grafana Cloud +npm run benchmark -- run my-api --publish + +# compare results from multiple runs +npm run benchmark -- compare results/go-api-2026-03-04T12-00-00-000Z.json results/java-api-2026-03-04T12-00-00-000Z.json +``` + +## Configuration + +All targets are defined in `benchmark.config.json` at the repo root. Each target points to a directory containing a `docker-compose.yml`. + +```json +{ + "targets": { + "my-go-api": { + "path": "./examples/go-api", + "composeFile": "docker-compose.yml", + "service": "api", + "port": 8080, + "protocol": "http", + "readinessProbe": { + "httpGet": { + "path": "/health", + "port": 8080, + "expectedStatus": 200 + }, + "initialDelayMs": 2000, + "intervalMs": 1000, + "timeoutMs": 120000 + }, + "k6": { + "script": "toolkit/k6/scripts/default-http.js", + "vus": 50, + "duration": "30s", + "env": { + "BASE_URL": "http://localhost:8080" + } + }, + "tags": { + "language": "go", + "framework": "stdlib" + } + } + }, + "defaults": { + "k6": { "vus": 50, "duration": "30s" }, + "readinessProbe": { + "initialDelayMs": 2000, + "intervalMs": 1000, + "timeoutMs": 120000 + } + }, + "grafana": { + "endpoint": "https://otlp-gateway-prod-us-central-0.grafana.net/otlp", + "instanceId": "${GRAFANA_INSTANCE_ID}", + "apiKey": "${GRAFANA_API_KEY}" + }, + "output": { "dir": "./results" } +} +``` + +### Target fields + +| Field | Required | Default | Description | +| ---------------- | -------- | -------------------- | --------------------------------------------------------- | +| `path` | yes | | Path to the project directory containing the compose file | +| `composeFile` | no | `docker-compose.yml` | Compose file name | +| `service` | yes | | Primary service name in the compose file | +| `port` | yes | | Port the service exposes on localhost | +| `protocol` | no | `http` | `http` or `grpc` | +| `readinessProbe` | no | see defaults | How to check if the service is ready | +| `k6` | no | see defaults | k6 load test configuration | +| `tags` | no | `{}` | Metadata labels (language, framework, etc.) | + +### Environment variables + +Grafana Cloud credentials are resolved from environment variables. Create a `.env` file or export them in your shell: + +```bash +export GRAFANA_INSTANCE_ID=your-instance-id +export GRAFANA_API_KEY=your-api-key +``` + +Values in the config using `${VAR_NAME}` syntax are interpolated from the environment at load time. + +## Commands + +### `benchmark run ` + +Runs the full benchmark pipeline: **build** → **deploy** → **loadtest** → **collect** → **cleanup**. + +```bash +npm run benchmark -- run my-api +npm run benchmark -- run my-api --tag "go-v1.22" --publish +npm run benchmark -- run my-api --skip-build --k6-vus 100 --k6-duration 60s +``` + +| Option | Description | +| ------------------- | ---------------------------------------------------------------- | +| `--skip-build` | Skip the Docker build step | +| `--skip-loadtest` | Skip the k6 load test step | +| `--k6-vus ` | Override virtual users count | +| `--k6-duration ` | Override test duration (e.g. `30s`, `1m`) | +| `--tag