diff --git a/.claude/commands/scaffold-implementation.md b/.claude/commands/scaffold-implementation.md new file mode 100644 index 0000000..131a507 --- /dev/null +++ b/.claude/commands/scaffold-implementation.md @@ -0,0 +1,128 @@ +# Scaffold a new benchmark implementation + +Guide the user interactively through creating a new implementation for a benchmark project. + +## Step 1 — Discover available projects + +List directories under `projects/` that contain a `_shared/` subdirectory. Present them to the user and ask which project they want to add an implementation for (e.g., `content-api`). + +## Step 2 — Pick API style + +List the API style subdirectories available under `projects//_shared/` (e.g., `openapi`, `graphql`, `protobuf`). Ask the user to choose which API style this implementation will use. + +## Step 3 — Name the implementation + +Ask the user for the implementation name in kebab-case (e.g., `spring-boot`, `express`, `ktor`, `actix-web`). This becomes the directory name under the project. + +## Step 4 — Specify language and framework + +Ask the user for: +- The programming language (e.g., `java`, `kotlin`, `javascript`, `typescript`, `go`, `rust`) +- The framework name (e.g., `spring-boot`, `express`, `ktor`, `actix-web`, `gin`) + +These are used in benchmark tags and guide code generation. + +## Step 5 — Generate the implementation + +Read the API spec and k6 script from `projects//_shared//` to understand the contract. + +Generate the following files under `projects///`: + +### Dockerfile +- Use an appropriate base image for the language/framework +- Multi-stage build where applicable (build stage + runtime stage) +- The final image should be as small as practical +- Expose port 8080 + +### docker-compose.yml +- Primary service named `api` on port 8080 +- Add a database service if the framework conventionally uses one +- Include healthcheck in the compose file +- Example structure: +```yaml +services: + api: + build: + context: . + dockerfile: Dockerfile + ports: + - "8080:8080" + environment: + - SERVER_PORT=8080 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 10s +``` + +### Application source code +- Implement ALL endpoints defined in the API spec +- For openapi style: + - `GET /health` returning `{"status": "up"}` + - `GET /api/v1/content` with pagination (limit/offset query params) + - `POST /api/v1/content` creating a new content item (return 201) + - `GET /api/v1/content/{id}` returning a single item (404 if not found) + - `PUT /api/v1/content/{id}` updating an item (404 if not found) + - `DELETE /api/v1/content/{id}` deleting an item (return 204, 404 if not found) +- For graphql style: + - Implement all queries and mutations defined in the schema + - Serve at `/graphql` endpoint + - Also serve `GET /health` returning `{"status": "up"}` for the readiness probe +- For protobuf style: + - Implement all RPCs defined in the proto service + - Serve on port 50051 for gRPC + - Also serve `GET /health` on port 8080 for the readiness probe +- Use in-memory storage (a simple map/list) unless the user specifically requests a database +- Generate UUIDs for content IDs +- Track createdAt and updatedAt timestamps +- Return proper HTTP status codes / gRPC status codes as specified +- Follow idiomatic conventions for the chosen language and framework + +### Build files +- Language-appropriate build file (pom.xml, package.json, build.gradle.kts, go.mod, Cargo.toml, etc.) +- Include only the dependencies needed for a minimal implementation + +## Step 6 — Register the target + +Add a target entry to `benchmark.config.json`: + +```json +"/": { + "path": "./projects//", + "composeFile": "docker-compose.yml", + "service": "api", + "port": 8080, + "protocol": "", + "readinessProbe": { + "httpGet": { + "path": "/health", + "port": 8080, + "expectedStatus": 200 + } + }, + "k6": { + "script": "./projects//_shared//k6/.js", + "vus": 50, + "duration": "30s", + "env": { + "BASE_URL": "http://localhost:8080" + } + }, + "tags": { + "project": "", + "language": "", + "framework": "", + "api-style": "" + } +} +``` + +For protobuf targets, adjust: +- `protocol`: `"grpc"` +- `k6.env`: use `GRPC_HOST` and `PROTO_PATH` instead of `BASE_URL` + +## Step 7 — Verify + +Run `npm run benchmark -- list` to confirm the new target appears in the output. diff --git a/benchmark.config.json b/benchmark.config.json index 8c617b8..b843820 100644 --- a/benchmark.config.json +++ b/benchmark.config.json @@ -1,32 +1,5 @@ { - "targets": { - "example-api": { - "path": "./examples/example-api", - "composeFile": "docker-compose.yml", - "service": "api", - "port": 8080, - "protocol": "http", - "readinessProbe": { - "httpGet": { - "path": "/health", - "port": 8080, - "expectedStatus": 200 - } - }, - "k6": { - "script": "toolkit/k6/scripts/default-http.js", - "vus": 50, - "duration": "30s", - "env": { - "BASE_URL": "http://localhost:8080" - } - }, - "tags": { - "language": "example", - "framework": "example" - } - } - }, + "targets": {}, "defaults": { "k6": { "vus": 50, diff --git a/projects/content-api/_shared/graphql/k6/content-api.js b/projects/content-api/_shared/graphql/k6/content-api.js new file mode 100644 index 0000000..e6255db --- /dev/null +++ b/projects/content-api/_shared/graphql/k6/content-api.js @@ -0,0 +1,139 @@ +import http from 'k6/http'; +import { check, group, sleep } from 'k6'; +import { randomString } from 'https://jslib.k6.io/k6-utils/1.4.0/index.js'; + +const BASE_URL = __ENV.BASE_URL || 'http://localhost:8080'; +const GRAPHQL_URL = `${BASE_URL}/graphql`; + +export const options = { + thresholds: { + http_req_duration: ['p(95)<500'], + http_req_failed: ['rate<0.01'], + }, +}; + +const headers = { 'Content-Type': 'application/json' }; + +function graphql(query, variables = {}) { + return http.post(GRAPHQL_URL, JSON.stringify({ query, variables }), { headers }); +} + +export default function () { + let contentId; + + group('create content', () => { + const res = graphql( + `mutation CreateContent($input: CreateContentInput!) { + createContent(input: $input) { + id title body status createdAt updatedAt + } + }`, + { + input: { + title: `Benchmark ${randomString(8)}`, + body: `Load test content body ${randomString(32)}`, + status: 'DRAFT', + }, + } + ); + + check(res, { + 'create: status is 200': (r) => r.status === 200, + 'create: has id': (r) => { + const body = r.json(); + contentId = body.data && body.data.createContent && body.data.createContent.id; + return !!contentId; + }, + 'create: no errors': (r) => !r.json().errors, + }); + }); + + group('get content', () => { + if (!contentId) return; + + const res = graphql( + `query GetContent($id: ID!) { + content(id: $id) { + id title body status createdAt updatedAt + } + }`, + { id: contentId } + ); + + check(res, { + 'get: status is 200': (r) => r.status === 200, + 'get: correct id': (r) => { + const body = r.json(); + return body.data && body.data.content && body.data.content.id === contentId; + }, + }); + }); + + group('list content', () => { + const res = graphql( + `query ListContent($limit: Int, $offset: Int) { + contents(limit: $limit, offset: $offset) { + items { id title status } + total limit offset + } + }`, + { limit: 10, offset: 0 } + ); + + check(res, { + 'list: status is 200': (r) => r.status === 200, + 'list: has items': (r) => { + const body = r.json(); + return body.data && Array.isArray(body.data.contents.items); + }, + }); + }); + + group('update content', () => { + if (!contentId) return; + + const res = graphql( + `mutation UpdateContent($id: ID!, $input: UpdateContentInput!) { + updateContent(id: $id, input: $input) { + id title status updatedAt + } + }`, + { + id: contentId, + input: { + title: `Updated ${randomString(8)}`, + status: 'PUBLISHED', + }, + } + ); + + check(res, { + 'update: status is 200': (r) => r.status === 200, + 'update: status changed': (r) => { + const body = r.json(); + return body.data && body.data.updateContent && body.data.updateContent.status === 'PUBLISHED'; + }, + }); + }); + + group('delete content', () => { + if (!contentId) return; + + const res = graphql( + `mutation DeleteContent($id: ID!) { + deleteContent(id: $id) + }`, + { id: contentId } + ); + + check(res, { + 'delete: status is 200': (r) => r.status === 200, + 'delete: success': (r) => { + const body = r.json(); + return body.data && body.data.deleteContent === true; + }, + }); + }); + + sleep(0.1); +} diff --git a/projects/content-api/_shared/graphql/schema.graphql b/projects/content-api/_shared/graphql/schema.graphql new file mode 100644 index 0000000..ec404c4 --- /dev/null +++ b/projects/content-api/_shared/graphql/schema.graphql @@ -0,0 +1,49 @@ +type Query { + health: HealthResponse! + content(id: ID!): Content + contents(limit: Int = 20, offset: Int = 0): ContentList! +} + +type Mutation { + createContent(input: CreateContentInput!): Content! + updateContent(id: ID!, input: UpdateContentInput!): Content + deleteContent(id: ID!): Boolean! +} + +type HealthResponse { + status: String! +} + +type Content { + id: ID! + title: String! + body: String! + status: ContentStatus! + createdAt: String! + updatedAt: String! +} + +type ContentList { + items: [Content!]! + total: Int! + limit: Int! + offset: Int! +} + +input CreateContentInput { + title: String! + body: String! + status: ContentStatus = DRAFT +} + +input UpdateContentInput { + title: String + body: String + status: ContentStatus +} + +enum ContentStatus { + DRAFT + PUBLISHED + ARCHIVED +} diff --git a/projects/content-api/_shared/openapi/api-spec.yaml b/projects/content-api/_shared/openapi/api-spec.yaml new file mode 100644 index 0000000..86b20bf --- /dev/null +++ b/projects/content-api/_shared/openapi/api-spec.yaml @@ -0,0 +1,200 @@ +openapi: 3.1.0 +info: + title: Content API + description: > + Benchmark API contract for content management. + All implementations must conform to this specification. + version: 1.0.0 + +paths: + /health: + get: + operationId: healthCheck + summary: Health check endpoint + responses: + '200': + description: Service is healthy + content: + application/json: + schema: + $ref: '#/components/schemas/HealthResponse' + + /api/v1/content: + get: + operationId: listContent + summary: List all content items + parameters: + - name: limit + in: query + schema: + type: integer + default: 20 + maximum: 100 + - name: offset + in: query + schema: + type: integer + default: 0 + responses: + '200': + description: Paginated list of content items + content: + application/json: + schema: + $ref: '#/components/schemas/ContentList' + + post: + operationId: createContent + summary: Create a new content item + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CreateContentRequest' + responses: + '201': + description: Content item created + content: + application/json: + schema: + $ref: '#/components/schemas/Content' + '400': + description: Invalid request + + /api/v1/content/{id}: + get: + operationId: getContent + summary: Get a content item by ID + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + '200': + description: Content item + content: + application/json: + schema: + $ref: '#/components/schemas/Content' + '404': + description: Not found + + put: + operationId: updateContent + summary: Update a content item + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateContentRequest' + responses: + '200': + description: Content item updated + content: + application/json: + schema: + $ref: '#/components/schemas/Content' + '404': + description: Not found + + delete: + operationId: deleteContent + summary: Delete a content item + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + '204': + description: Content item deleted + '404': + description: Not found + +components: + schemas: + HealthResponse: + type: object + required: [status] + properties: + status: + type: string + enum: [up] + + Content: + type: object + required: [id, title, body, status, createdAt, updatedAt] + properties: + id: + type: string + format: uuid + title: + type: string + body: + type: string + status: + type: string + enum: [draft, published, archived] + createdAt: + type: string + format: date-time + updatedAt: + type: string + format: date-time + + ContentList: + type: object + required: [items, total, limit, offset] + properties: + items: + type: array + items: + $ref: '#/components/schemas/Content' + total: + type: integer + limit: + type: integer + offset: + type: integer + + CreateContentRequest: + type: object + required: [title, body] + properties: + title: + type: string + minLength: 1 + maxLength: 255 + body: + type: string + status: + type: string + enum: [draft, published] + default: draft + + UpdateContentRequest: + type: object + properties: + title: + type: string + minLength: 1 + maxLength: 255 + body: + type: string + status: + type: string + enum: [draft, published, archived] diff --git a/projects/content-api/_shared/openapi/k6/content-api.js b/projects/content-api/_shared/openapi/k6/content-api.js new file mode 100644 index 0000000..a4edd8c --- /dev/null +++ b/projects/content-api/_shared/openapi/k6/content-api.js @@ -0,0 +1,85 @@ +import http from 'k6/http'; +import { check, group, sleep } from 'k6'; +import { randomString } from 'https://jslib.k6.io/k6-utils/1.4.0/index.js'; + +const BASE_URL = __ENV.BASE_URL || 'http://localhost:8080'; + +export const options = { + thresholds: { + http_req_duration: ['p(95)<500'], + http_req_failed: ['rate<0.01'], + }, +}; + +const headers = { 'Content-Type': 'application/json' }; + +export default function () { + let contentId; + + group('create content', () => { + const payload = JSON.stringify({ + title: `Benchmark ${randomString(8)}`, + body: `Load test content body ${randomString(32)}`, + status: 'draft', + }); + + const res = http.post(`${BASE_URL}/api/v1/content`, payload, { headers }); + + check(res, { + 'create: status is 201': (r) => r.status === 201, + 'create: has id': (r) => { + const body = r.json(); + contentId = body.id; + return !!contentId; + }, + }); + }); + + group('get content', () => { + if (!contentId) return; + + const res = http.get(`${BASE_URL}/api/v1/content/${contentId}`); + + check(res, { + 'get: status is 200': (r) => r.status === 200, + 'get: correct id': (r) => r.json().id === contentId, + }); + }); + + group('list content', () => { + const res = http.get(`${BASE_URL}/api/v1/content?limit=10&offset=0`); + + check(res, { + 'list: status is 200': (r) => r.status === 200, + 'list: has items array': (r) => Array.isArray(r.json().items), + }); + }); + + group('update content', () => { + if (!contentId) return; + + const payload = JSON.stringify({ + title: `Updated ${randomString(8)}`, + status: 'published', + }); + + const res = http.put(`${BASE_URL}/api/v1/content/${contentId}`, payload, { headers }); + + check(res, { + 'update: status is 200': (r) => r.status === 200, + 'update: status changed': (r) => r.json().status === 'published', + }); + }); + + group('delete content', () => { + if (!contentId) return; + + const res = http.del(`${BASE_URL}/api/v1/content/${contentId}`); + + check(res, { + 'delete: status is 204': (r) => r.status === 204, + }); + }); + + sleep(0.1); +} diff --git a/projects/content-api/_shared/protobuf/content.proto b/projects/content-api/_shared/protobuf/content.proto new file mode 100644 index 0000000..7bb91f5 --- /dev/null +++ b/projects/content-api/_shared/protobuf/content.proto @@ -0,0 +1,74 @@ +syntax = "proto3"; + +package content.v1; + +option java_multiple_files = true; +option java_package = "com.labset.benchmark.content.v1"; + +service ContentService { + rpc CheckHealth(HealthRequest) returns (HealthResponse); + rpc ListContent(ListContentRequest) returns (ListContentResponse); + rpc GetContent(GetContentRequest) returns (Content); + rpc CreateContent(CreateContentRequest) returns (Content); + rpc UpdateContent(UpdateContentRequest) returns (Content); + rpc DeleteContent(DeleteContentRequest) returns (DeleteContentResponse); +} + +message HealthRequest {} + +message HealthResponse { + string status = 1; +} + +enum ContentStatus { + CONTENT_STATUS_UNSPECIFIED = 0; + CONTENT_STATUS_DRAFT = 1; + CONTENT_STATUS_PUBLISHED = 2; + CONTENT_STATUS_ARCHIVED = 3; +} + +message Content { + string id = 1; + string title = 2; + string body = 3; + ContentStatus status = 4; + string created_at = 5; + string updated_at = 6; +} + +message ListContentRequest { + int32 limit = 1; + int32 offset = 2; +} + +message ListContentResponse { + repeated Content items = 1; + int32 total = 2; + int32 limit = 3; + int32 offset = 4; +} + +message GetContentRequest { + string id = 1; +} + +message CreateContentRequest { + string title = 1; + string body = 2; + ContentStatus status = 3; +} + +message UpdateContentRequest { + string id = 1; + optional string title = 2; + optional string body = 3; + optional ContentStatus status = 4; +} + +message DeleteContentRequest { + string id = 1; +} + +message DeleteContentResponse { + bool success = 1; +} diff --git a/projects/content-api/_shared/protobuf/k6/content-api.js b/projects/content-api/_shared/protobuf/k6/content-api.js new file mode 100644 index 0000000..25fbeb4 --- /dev/null +++ b/projects/content-api/_shared/protobuf/k6/content-api.js @@ -0,0 +1,97 @@ +import grpc from 'k6/net/grpc'; +import { check, group, sleep } from 'k6'; +import { randomString } from 'https://jslib.k6.io/k6-utils/1.4.0/index.js'; + +const GRPC_HOST = __ENV.GRPC_HOST || 'localhost:50051'; +const PROTO_PATH = __ENV.PROTO_PATH || ''; + +const client = new grpc.Client(); + +export const options = { + thresholds: { + grpc_req_duration: ['p(95)<500'], + }, +}; + +export default function () { + if (PROTO_PATH) { + client.load([], PROTO_PATH); + } + + client.connect(GRPC_HOST, { plaintext: true }); + + let contentId; + + group('create content', () => { + const res = client.invoke('content.v1.ContentService/CreateContent', { + title: `Benchmark ${randomString(8)}`, + body: `Load test content body ${randomString(32)}`, + status: 'CONTENT_STATUS_DRAFT', + }); + + check(res, { + 'create: status is OK': (r) => r && r.status === grpc.StatusOK, + 'create: has id': (r) => { + contentId = r && r.message && r.message.id; + return !!contentId; + }, + }); + }); + + group('get content', () => { + if (!contentId) return; + + const res = client.invoke('content.v1.ContentService/GetContent', { + id: contentId, + }); + + check(res, { + 'get: status is OK': (r) => r && r.status === grpc.StatusOK, + 'get: correct id': (r) => r && r.message && r.message.id === contentId, + }); + }); + + group('list content', () => { + const res = client.invoke('content.v1.ContentService/ListContent', { + limit: 10, + offset: 0, + }); + + check(res, { + 'list: status is OK': (r) => r && r.status === grpc.StatusOK, + 'list: has items': (r) => r && r.message && Array.isArray(r.message.items), + }); + }); + + group('update content', () => { + if (!contentId) return; + + const res = client.invoke('content.v1.ContentService/UpdateContent', { + id: contentId, + title: `Updated ${randomString(8)}`, + status: 'CONTENT_STATUS_PUBLISHED', + }); + + check(res, { + 'update: status is OK': (r) => r && r.status === grpc.StatusOK, + 'update: status changed': (r) => + r && r.message && r.message.status === 'CONTENT_STATUS_PUBLISHED', + }); + }); + + group('delete content', () => { + if (!contentId) return; + + const res = client.invoke('content.v1.ContentService/DeleteContent', { + id: contentId, + }); + + check(res, { + 'delete: status is OK': (r) => r && r.status === grpc.StatusOK, + 'delete: success': (r) => r && r.message && r.message.success === true, + }); + }); + + client.close(); + sleep(0.1); +} diff --git a/toolkit/src/report/json.js b/toolkit/src/report/json.js index ac7c86d..7e8fd91 100644 --- a/toolkit/src/report/json.js +++ b/toolkit/src/report/json.js @@ -7,7 +7,8 @@ export async function writeResults(results, outputDir) { await mkdir(outputDir, { recursive: true }); const timestamp = results.timestamp.replace(/[:.]/g, '-'); - const filename = `${results.target}-${timestamp}.json`; + const safeName = results.target.replace(/\//g, '-'); + const filename = `${safeName}-${timestamp}.json`; const filepath = join(outputDir, filename); await writeFile(filepath, JSON.stringify(results, null, 2), 'utf-8');