From dae0246e9f2f764c06a664f9005d9b8463103746 Mon Sep 17 00:00:00 2001 From: Hasnae Date: Fri, 6 Mar 2026 16:01:16 +1100 Subject: [PATCH] Add content-api benchmark project with multi-style API specs Establish the canonical project structure under projects/ for benchmark targets, starting with content-api. Shared API contracts and k6 load test scripts are provided for three API styles: OpenAPI (REST), GraphQL, and Protobuf (gRPC). An interactive Claude command is added to scaffold new implementations against these specs. Also fixes result filename generation to sanitize slashes in target names and removes the example-api placeholder from the config. Co-Authored-By: Claude Opus 4.6 --- .claude/commands/scaffold-implementation.md | 128 +++++++++++ benchmark.config.json | 29 +-- .../_shared/graphql/k6/content-api.js | 139 ++++++++++++ .../_shared/graphql/schema.graphql | 49 +++++ .../content-api/_shared/openapi/api-spec.yaml | 200 ++++++++++++++++++ .../_shared/openapi/k6/content-api.js | 85 ++++++++ .../_shared/protobuf/content.proto | 74 +++++++ .../_shared/protobuf/k6/content-api.js | 97 +++++++++ toolkit/src/report/json.js | 3 +- 9 files changed, 775 insertions(+), 29 deletions(-) create mode 100644 .claude/commands/scaffold-implementation.md create mode 100644 projects/content-api/_shared/graphql/k6/content-api.js create mode 100644 projects/content-api/_shared/graphql/schema.graphql create mode 100644 projects/content-api/_shared/openapi/api-spec.yaml create mode 100644 projects/content-api/_shared/openapi/k6/content-api.js create mode 100644 projects/content-api/_shared/protobuf/content.proto create mode 100644 projects/content-api/_shared/protobuf/k6/content-api.js diff --git a/.claude/commands/scaffold-implementation.md b/.claude/commands/scaffold-implementation.md new file mode 100644 index 0000000..131a507 --- /dev/null +++ b/.claude/commands/scaffold-implementation.md @@ -0,0 +1,128 @@ +# Scaffold a new benchmark implementation + +Guide the user interactively through creating a new implementation for a benchmark project. + +## Step 1 — Discover available projects + +List directories under `projects/` that contain a `_shared/` subdirectory. Present them to the user and ask which project they want to add an implementation for (e.g., `content-api`). + +## Step 2 — Pick API style + +List the API style subdirectories available under `projects//_shared/` (e.g., `openapi`, `graphql`, `protobuf`). Ask the user to choose which API style this implementation will use. + +## Step 3 — Name the implementation + +Ask the user for the implementation name in kebab-case (e.g., `spring-boot`, `express`, `ktor`, `actix-web`). This becomes the directory name under the project. + +## Step 4 — Specify language and framework + +Ask the user for: +- The programming language (e.g., `java`, `kotlin`, `javascript`, `typescript`, `go`, `rust`) +- The framework name (e.g., `spring-boot`, `express`, `ktor`, `actix-web`, `gin`) + +These are used in benchmark tags and guide code generation. + +## Step 5 — Generate the implementation + +Read the API spec and k6 script from `projects//_shared//` to understand the contract. + +Generate the following files under `projects///`: + +### Dockerfile +- Use an appropriate base image for the language/framework +- Multi-stage build where applicable (build stage + runtime stage) +- The final image should be as small as practical +- Expose port 8080 + +### docker-compose.yml +- Primary service named `api` on port 8080 +- Add a database service if the framework conventionally uses one +- Include healthcheck in the compose file +- Example structure: +```yaml +services: + api: + build: + context: . + dockerfile: Dockerfile + ports: + - "8080:8080" + environment: + - SERVER_PORT=8080 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 10s +``` + +### Application source code +- Implement ALL endpoints defined in the API spec +- For openapi style: + - `GET /health` returning `{"status": "up"}` + - `GET /api/v1/content` with pagination (limit/offset query params) + - `POST /api/v1/content` creating a new content item (return 201) + - `GET /api/v1/content/{id}` returning a single item (404 if not found) + - `PUT /api/v1/content/{id}` updating an item (404 if not found) + - `DELETE /api/v1/content/{id}` deleting an item (return 204, 404 if not found) +- For graphql style: + - Implement all queries and mutations defined in the schema + - Serve at `/graphql` endpoint + - Also serve `GET /health` returning `{"status": "up"}` for the readiness probe +- For protobuf style: + - Implement all RPCs defined in the proto service + - Serve on port 50051 for gRPC + - Also serve `GET /health` on port 8080 for the readiness probe +- Use in-memory storage (a simple map/list) unless the user specifically requests a database +- Generate UUIDs for content IDs +- Track createdAt and updatedAt timestamps +- Return proper HTTP status codes / gRPC status codes as specified +- Follow idiomatic conventions for the chosen language and framework + +### Build files +- Language-appropriate build file (pom.xml, package.json, build.gradle.kts, go.mod, Cargo.toml, etc.) +- Include only the dependencies needed for a minimal implementation + +## Step 6 — Register the target + +Add a target entry to `benchmark.config.json`: + +```json +"/": { + "path": "./projects//", + "composeFile": "docker-compose.yml", + "service": "api", + "port": 8080, + "protocol": "", + "readinessProbe": { + "httpGet": { + "path": "/health", + "port": 8080, + "expectedStatus": 200 + } + }, + "k6": { + "script": "./projects//_shared//k6/.js", + "vus": 50, + "duration": "30s", + "env": { + "BASE_URL": "http://localhost:8080" + } + }, + "tags": { + "project": "", + "language": "", + "framework": "", + "api-style": "" + } +} +``` + +For protobuf targets, adjust: +- `protocol`: `"grpc"` +- `k6.env`: use `GRPC_HOST` and `PROTO_PATH` instead of `BASE_URL` + +## Step 7 — Verify + +Run `npm run benchmark -- list` to confirm the new target appears in the output. diff --git a/benchmark.config.json b/benchmark.config.json index 8c617b8..b843820 100644 --- a/benchmark.config.json +++ b/benchmark.config.json @@ -1,32 +1,5 @@ { - "targets": { - "example-api": { - "path": "./examples/example-api", - "composeFile": "docker-compose.yml", - "service": "api", - "port": 8080, - "protocol": "http", - "readinessProbe": { - "httpGet": { - "path": "/health", - "port": 8080, - "expectedStatus": 200 - } - }, - "k6": { - "script": "toolkit/k6/scripts/default-http.js", - "vus": 50, - "duration": "30s", - "env": { - "BASE_URL": "http://localhost:8080" - } - }, - "tags": { - "language": "example", - "framework": "example" - } - } - }, + "targets": {}, "defaults": { "k6": { "vus": 50, diff --git a/projects/content-api/_shared/graphql/k6/content-api.js b/projects/content-api/_shared/graphql/k6/content-api.js new file mode 100644 index 0000000..e6255db --- /dev/null +++ b/projects/content-api/_shared/graphql/k6/content-api.js @@ -0,0 +1,139 @@ +import http from 'k6/http'; +import { check, group, sleep } from 'k6'; +import { randomString } from 'https://jslib.k6.io/k6-utils/1.4.0/index.js'; + +const BASE_URL = __ENV.BASE_URL || 'http://localhost:8080'; +const GRAPHQL_URL = `${BASE_URL}/graphql`; + +export const options = { + thresholds: { + http_req_duration: ['p(95)<500'], + http_req_failed: ['rate<0.01'], + }, +}; + +const headers = { 'Content-Type': 'application/json' }; + +function graphql(query, variables = {}) { + return http.post(GRAPHQL_URL, JSON.stringify({ query, variables }), { headers }); +} + +export default function () { + let contentId; + + group('create content', () => { + const res = graphql( + `mutation CreateContent($input: CreateContentInput!) { + createContent(input: $input) { + id title body status createdAt updatedAt + } + }`, + { + input: { + title: `Benchmark ${randomString(8)}`, + body: `Load test content body ${randomString(32)}`, + status: 'DRAFT', + }, + } + ); + + check(res, { + 'create: status is 200': (r) => r.status === 200, + 'create: has id': (r) => { + const body = r.json(); + contentId = body.data && body.data.createContent && body.data.createContent.id; + return !!contentId; + }, + 'create: no errors': (r) => !r.json().errors, + }); + }); + + group('get content', () => { + if (!contentId) return; + + const res = graphql( + `query GetContent($id: ID!) { + content(id: $id) { + id title body status createdAt updatedAt + } + }`, + { id: contentId } + ); + + check(res, { + 'get: status is 200': (r) => r.status === 200, + 'get: correct id': (r) => { + const body = r.json(); + return body.data && body.data.content && body.data.content.id === contentId; + }, + }); + }); + + group('list content', () => { + const res = graphql( + `query ListContent($limit: Int, $offset: Int) { + contents(limit: $limit, offset: $offset) { + items { id title status } + total limit offset + } + }`, + { limit: 10, offset: 0 } + ); + + check(res, { + 'list: status is 200': (r) => r.status === 200, + 'list: has items': (r) => { + const body = r.json(); + return body.data && Array.isArray(body.data.contents.items); + }, + }); + }); + + group('update content', () => { + if (!contentId) return; + + const res = graphql( + `mutation UpdateContent($id: ID!, $input: UpdateContentInput!) { + updateContent(id: $id, input: $input) { + id title status updatedAt + } + }`, + { + id: contentId, + input: { + title: `Updated ${randomString(8)}`, + status: 'PUBLISHED', + }, + } + ); + + check(res, { + 'update: status is 200': (r) => r.status === 200, + 'update: status changed': (r) => { + const body = r.json(); + return body.data && body.data.updateContent && body.data.updateContent.status === 'PUBLISHED'; + }, + }); + }); + + group('delete content', () => { + if (!contentId) return; + + const res = graphql( + `mutation DeleteContent($id: ID!) { + deleteContent(id: $id) + }`, + { id: contentId } + ); + + check(res, { + 'delete: status is 200': (r) => r.status === 200, + 'delete: success': (r) => { + const body = r.json(); + return body.data && body.data.deleteContent === true; + }, + }); + }); + + sleep(0.1); +} diff --git a/projects/content-api/_shared/graphql/schema.graphql b/projects/content-api/_shared/graphql/schema.graphql new file mode 100644 index 0000000..ec404c4 --- /dev/null +++ b/projects/content-api/_shared/graphql/schema.graphql @@ -0,0 +1,49 @@ +type Query { + health: HealthResponse! + content(id: ID!): Content + contents(limit: Int = 20, offset: Int = 0): ContentList! +} + +type Mutation { + createContent(input: CreateContentInput!): Content! + updateContent(id: ID!, input: UpdateContentInput!): Content + deleteContent(id: ID!): Boolean! +} + +type HealthResponse { + status: String! +} + +type Content { + id: ID! + title: String! + body: String! + status: ContentStatus! + createdAt: String! + updatedAt: String! +} + +type ContentList { + items: [Content!]! + total: Int! + limit: Int! + offset: Int! +} + +input CreateContentInput { + title: String! + body: String! + status: ContentStatus = DRAFT +} + +input UpdateContentInput { + title: String + body: String + status: ContentStatus +} + +enum ContentStatus { + DRAFT + PUBLISHED + ARCHIVED +} diff --git a/projects/content-api/_shared/openapi/api-spec.yaml b/projects/content-api/_shared/openapi/api-spec.yaml new file mode 100644 index 0000000..86b20bf --- /dev/null +++ b/projects/content-api/_shared/openapi/api-spec.yaml @@ -0,0 +1,200 @@ +openapi: 3.1.0 +info: + title: Content API + description: > + Benchmark API contract for content management. + All implementations must conform to this specification. + version: 1.0.0 + +paths: + /health: + get: + operationId: healthCheck + summary: Health check endpoint + responses: + '200': + description: Service is healthy + content: + application/json: + schema: + $ref: '#/components/schemas/HealthResponse' + + /api/v1/content: + get: + operationId: listContent + summary: List all content items + parameters: + - name: limit + in: query + schema: + type: integer + default: 20 + maximum: 100 + - name: offset + in: query + schema: + type: integer + default: 0 + responses: + '200': + description: Paginated list of content items + content: + application/json: + schema: + $ref: '#/components/schemas/ContentList' + + post: + operationId: createContent + summary: Create a new content item + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CreateContentRequest' + responses: + '201': + description: Content item created + content: + application/json: + schema: + $ref: '#/components/schemas/Content' + '400': + description: Invalid request + + /api/v1/content/{id}: + get: + operationId: getContent + summary: Get a content item by ID + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + '200': + description: Content item + content: + application/json: + schema: + $ref: '#/components/schemas/Content' + '404': + description: Not found + + put: + operationId: updateContent + summary: Update a content item + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateContentRequest' + responses: + '200': + description: Content item updated + content: + application/json: + schema: + $ref: '#/components/schemas/Content' + '404': + description: Not found + + delete: + operationId: deleteContent + summary: Delete a content item + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + '204': + description: Content item deleted + '404': + description: Not found + +components: + schemas: + HealthResponse: + type: object + required: [status] + properties: + status: + type: string + enum: [up] + + Content: + type: object + required: [id, title, body, status, createdAt, updatedAt] + properties: + id: + type: string + format: uuid + title: + type: string + body: + type: string + status: + type: string + enum: [draft, published, archived] + createdAt: + type: string + format: date-time + updatedAt: + type: string + format: date-time + + ContentList: + type: object + required: [items, total, limit, offset] + properties: + items: + type: array + items: + $ref: '#/components/schemas/Content' + total: + type: integer + limit: + type: integer + offset: + type: integer + + CreateContentRequest: + type: object + required: [title, body] + properties: + title: + type: string + minLength: 1 + maxLength: 255 + body: + type: string + status: + type: string + enum: [draft, published] + default: draft + + UpdateContentRequest: + type: object + properties: + title: + type: string + minLength: 1 + maxLength: 255 + body: + type: string + status: + type: string + enum: [draft, published, archived] diff --git a/projects/content-api/_shared/openapi/k6/content-api.js b/projects/content-api/_shared/openapi/k6/content-api.js new file mode 100644 index 0000000..a4edd8c --- /dev/null +++ b/projects/content-api/_shared/openapi/k6/content-api.js @@ -0,0 +1,85 @@ +import http from 'k6/http'; +import { check, group, sleep } from 'k6'; +import { randomString } from 'https://jslib.k6.io/k6-utils/1.4.0/index.js'; + +const BASE_URL = __ENV.BASE_URL || 'http://localhost:8080'; + +export const options = { + thresholds: { + http_req_duration: ['p(95)<500'], + http_req_failed: ['rate<0.01'], + }, +}; + +const headers = { 'Content-Type': 'application/json' }; + +export default function () { + let contentId; + + group('create content', () => { + const payload = JSON.stringify({ + title: `Benchmark ${randomString(8)}`, + body: `Load test content body ${randomString(32)}`, + status: 'draft', + }); + + const res = http.post(`${BASE_URL}/api/v1/content`, payload, { headers }); + + check(res, { + 'create: status is 201': (r) => r.status === 201, + 'create: has id': (r) => { + const body = r.json(); + contentId = body.id; + return !!contentId; + }, + }); + }); + + group('get content', () => { + if (!contentId) return; + + const res = http.get(`${BASE_URL}/api/v1/content/${contentId}`); + + check(res, { + 'get: status is 200': (r) => r.status === 200, + 'get: correct id': (r) => r.json().id === contentId, + }); + }); + + group('list content', () => { + const res = http.get(`${BASE_URL}/api/v1/content?limit=10&offset=0`); + + check(res, { + 'list: status is 200': (r) => r.status === 200, + 'list: has items array': (r) => Array.isArray(r.json().items), + }); + }); + + group('update content', () => { + if (!contentId) return; + + const payload = JSON.stringify({ + title: `Updated ${randomString(8)}`, + status: 'published', + }); + + const res = http.put(`${BASE_URL}/api/v1/content/${contentId}`, payload, { headers }); + + check(res, { + 'update: status is 200': (r) => r.status === 200, + 'update: status changed': (r) => r.json().status === 'published', + }); + }); + + group('delete content', () => { + if (!contentId) return; + + const res = http.del(`${BASE_URL}/api/v1/content/${contentId}`); + + check(res, { + 'delete: status is 204': (r) => r.status === 204, + }); + }); + + sleep(0.1); +} diff --git a/projects/content-api/_shared/protobuf/content.proto b/projects/content-api/_shared/protobuf/content.proto new file mode 100644 index 0000000..7bb91f5 --- /dev/null +++ b/projects/content-api/_shared/protobuf/content.proto @@ -0,0 +1,74 @@ +syntax = "proto3"; + +package content.v1; + +option java_multiple_files = true; +option java_package = "com.labset.benchmark.content.v1"; + +service ContentService { + rpc CheckHealth(HealthRequest) returns (HealthResponse); + rpc ListContent(ListContentRequest) returns (ListContentResponse); + rpc GetContent(GetContentRequest) returns (Content); + rpc CreateContent(CreateContentRequest) returns (Content); + rpc UpdateContent(UpdateContentRequest) returns (Content); + rpc DeleteContent(DeleteContentRequest) returns (DeleteContentResponse); +} + +message HealthRequest {} + +message HealthResponse { + string status = 1; +} + +enum ContentStatus { + CONTENT_STATUS_UNSPECIFIED = 0; + CONTENT_STATUS_DRAFT = 1; + CONTENT_STATUS_PUBLISHED = 2; + CONTENT_STATUS_ARCHIVED = 3; +} + +message Content { + string id = 1; + string title = 2; + string body = 3; + ContentStatus status = 4; + string created_at = 5; + string updated_at = 6; +} + +message ListContentRequest { + int32 limit = 1; + int32 offset = 2; +} + +message ListContentResponse { + repeated Content items = 1; + int32 total = 2; + int32 limit = 3; + int32 offset = 4; +} + +message GetContentRequest { + string id = 1; +} + +message CreateContentRequest { + string title = 1; + string body = 2; + ContentStatus status = 3; +} + +message UpdateContentRequest { + string id = 1; + optional string title = 2; + optional string body = 3; + optional ContentStatus status = 4; +} + +message DeleteContentRequest { + string id = 1; +} + +message DeleteContentResponse { + bool success = 1; +} diff --git a/projects/content-api/_shared/protobuf/k6/content-api.js b/projects/content-api/_shared/protobuf/k6/content-api.js new file mode 100644 index 0000000..25fbeb4 --- /dev/null +++ b/projects/content-api/_shared/protobuf/k6/content-api.js @@ -0,0 +1,97 @@ +import grpc from 'k6/net/grpc'; +import { check, group, sleep } from 'k6'; +import { randomString } from 'https://jslib.k6.io/k6-utils/1.4.0/index.js'; + +const GRPC_HOST = __ENV.GRPC_HOST || 'localhost:50051'; +const PROTO_PATH = __ENV.PROTO_PATH || ''; + +const client = new grpc.Client(); + +export const options = { + thresholds: { + grpc_req_duration: ['p(95)<500'], + }, +}; + +export default function () { + if (PROTO_PATH) { + client.load([], PROTO_PATH); + } + + client.connect(GRPC_HOST, { plaintext: true }); + + let contentId; + + group('create content', () => { + const res = client.invoke('content.v1.ContentService/CreateContent', { + title: `Benchmark ${randomString(8)}`, + body: `Load test content body ${randomString(32)}`, + status: 'CONTENT_STATUS_DRAFT', + }); + + check(res, { + 'create: status is OK': (r) => r && r.status === grpc.StatusOK, + 'create: has id': (r) => { + contentId = r && r.message && r.message.id; + return !!contentId; + }, + }); + }); + + group('get content', () => { + if (!contentId) return; + + const res = client.invoke('content.v1.ContentService/GetContent', { + id: contentId, + }); + + check(res, { + 'get: status is OK': (r) => r && r.status === grpc.StatusOK, + 'get: correct id': (r) => r && r.message && r.message.id === contentId, + }); + }); + + group('list content', () => { + const res = client.invoke('content.v1.ContentService/ListContent', { + limit: 10, + offset: 0, + }); + + check(res, { + 'list: status is OK': (r) => r && r.status === grpc.StatusOK, + 'list: has items': (r) => r && r.message && Array.isArray(r.message.items), + }); + }); + + group('update content', () => { + if (!contentId) return; + + const res = client.invoke('content.v1.ContentService/UpdateContent', { + id: contentId, + title: `Updated ${randomString(8)}`, + status: 'CONTENT_STATUS_PUBLISHED', + }); + + check(res, { + 'update: status is OK': (r) => r && r.status === grpc.StatusOK, + 'update: status changed': (r) => + r && r.message && r.message.status === 'CONTENT_STATUS_PUBLISHED', + }); + }); + + group('delete content', () => { + if (!contentId) return; + + const res = client.invoke('content.v1.ContentService/DeleteContent', { + id: contentId, + }); + + check(res, { + 'delete: status is OK': (r) => r && r.status === grpc.StatusOK, + 'delete: success': (r) => r && r.message && r.message.success === true, + }); + }); + + client.close(); + sleep(0.1); +} diff --git a/toolkit/src/report/json.js b/toolkit/src/report/json.js index ac7c86d..7e8fd91 100644 --- a/toolkit/src/report/json.js +++ b/toolkit/src/report/json.js @@ -7,7 +7,8 @@ export async function writeResults(results, outputDir) { await mkdir(outputDir, { recursive: true }); const timestamp = results.timestamp.replace(/[:.]/g, '-'); - const filename = `${results.target}-${timestamp}.json`; + const safeName = results.target.replace(/\//g, '-'); + const filename = `${safeName}-${timestamp}.json`; const filepath = join(outputDir, filename); await writeFile(filepath, JSON.stringify(results, null, 2), 'utf-8');