diff --git a/docs/batch-queue-stress-test-plan.md b/docs/batch-queue-stress-test-plan.md
new file mode 100644
index 0000000000..0a8c639960
--- /dev/null
+++ b/docs/batch-queue-stress-test-plan.md
@@ -0,0 +1,5011 @@
+# Batch Queue Stress Test Plan
+
+This document outlines a systematic approach to testing the new Batch Queue with Deficit Round Robin (DRR) scheduling.
+
+## Test Environment
+
+**Setup:**
+
+- Single local machine running one webapp instance (single BatchQueue consumer process)
+- 3 organizations, each with 3 projects = **9 tenants total**
+- All tenants using the same task ID: `stress-test-task`
+
+**Note:** In production, multiple webapp instances will run BatchQueue consumers in parallel. This single-instance testing focuses on algorithm behavior rather than horizontal scaling.
+
+## Configuration Options Under Test
+
+| Variable | Default | Description |
+| --------------------------------------- | ------- | ------------------------------------------------------ |
+| `BATCH_QUEUE_DRR_QUANTUM` | 5 | Credits allocated per environment per scheduling round |
+| `BATCH_QUEUE_MAX_DEFICIT` | 50 | Maximum accumulated deficit (prevents starvation) |
+| `BATCH_QUEUE_CONSUMER_COUNT` | 1 | Number of concurrent consumer loops |
+| `BATCH_QUEUE_CONSUMER_INTERVAL_MS` | 100 | Polling interval between consumer iterations |
+| `BATCH_CONCURRENCY_DEFAULT_CONCURRENCY` | 10 | Default concurrent batch items per environment |
+| `BATCH_QUEUE_GLOBAL_RATE_LIMIT` | none | Optional global items/sec limit |
+
+**Per-Org Settings (via database):**
+
+- `batchQueueConcurrencyConfig` - Override concurrency per org
+- `batchRateLimitConfig` - Rate limit per org
+
+---
+
+## Test Series Overview
+
+| Series | Focus | Objective |
+| ------ | ------------------- | ------------------------------------------------------ |
+| A | Baseline | Establish reference metrics with defaults |
+| B | DRR Quantum | Understand how quantum affects fairness and throughput |
+| C | Max Deficit | Understand starvation prevention and catch-up behavior |
+| D | Consumer Count | Test parallelism within single process |
+| E | Consumer Interval | Test polling frequency impact |
+| F | Concurrency Limits | Test per-environment processing limits |
+| G | Global Rate Limiter | Test global throughput caps |
+| H | Asymmetric Load | Test fairness under uneven workloads |
+| I | Combined Tuning | Test optimized configurations |
+
+---
+
+## Series A: Baseline (Default Configuration)
+
+**Objective:** Establish baseline metrics with all default settings.
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+# BATCH_QUEUE_CONSUMER_COUNT not set (default: 1)
+# BATCH_QUEUE_CONSUMER_INTERVAL_MS not set (default: 100)
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+# BATCH_QUEUE_GLOBAL_RATE_LIMIT not set
+```
+
+### Test A1: Fairness Baseline (All 9 Tenants)
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Baseline fairness test"
+```
+
+**Expected:** All tenants should complete in roughly similar timeframes.
+
+
+Results A1 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "Baseline fairness test",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 54.51792024230187,
+ "overallDuration": 49525,
+ "fairnessIndex": 0.9999098436641237,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 49323,
+ "avgItemsPerSecond": 6.082355087890031,
+ "avgBatchDuration": 49238,
+ "minBatchDuration": 49196,
+ "maxBatchDuration": 49322,
+ "p50Duration": 49196,
+ "p95Duration": 49322,
+ "p99Duration": 49322
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 49460,
+ "avgItemsPerSecond": 6.06550748079256,
+ "avgBatchDuration": 49290.666666666664,
+ "minBatchDuration": 49187,
+ "maxBatchDuration": 49429,
+ "p50Duration": 49256,
+ "p95Duration": 49429,
+ "p99Duration": 49429
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 49484,
+ "avgItemsPerSecond": 6.062565677794843,
+ "avgBatchDuration": 36861.666666666664,
+ "minBatchDuration": 15041,
+ "maxBatchDuration": 49472,
+ "p50Duration": 46072,
+ "p95Duration": 49472,
+ "p99Duration": 49472
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 48442,
+ "avgItemsPerSecond": 6.192973039924033,
+ "avgBatchDuration": 48347,
+ "minBatchDuration": 48223,
+ "maxBatchDuration": 48442,
+ "p50Duration": 48376,
+ "p95Duration": 48442,
+ "p99Duration": 48442
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 48446,
+ "avgItemsPerSecond": 6.192461709945094,
+ "avgBatchDuration": 48102.333333333336,
+ "minBatchDuration": 47417,
+ "maxBatchDuration": 48446,
+ "p50Duration": 48444,
+ "p95Duration": 48446,
+ "p99Duration": 48446
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 48288,
+ "avgItemsPerSecond": 6.21272365805169,
+ "avgBatchDuration": 36139.333333333336,
+ "minBatchDuration": 16087,
+ "maxBatchDuration": 48218,
+ "p50Duration": 44113,
+ "p95Duration": 48218,
+ "p99Duration": 48218
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 49148,
+ "avgItemsPerSecond": 6.104012370798404,
+ "avgBatchDuration": 49074.666666666664,
+ "minBatchDuration": 49061,
+ "maxBatchDuration": 49099,
+ "p50Duration": 49064,
+ "p95Duration": 49099,
+ "p99Duration": 49099
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 49178,
+ "avgItemsPerSecond": 6.1002887470006915,
+ "avgBatchDuration": 48484.333333333336,
+ "minBatchDuration": 47154,
+ "maxBatchDuration": 49176,
+ "p50Duration": 49123,
+ "p95Duration": 49176,
+ "p99Duration": 49176
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 49454,
+ "avgItemsPerSecond": 6.066243377684312,
+ "avgBatchDuration": 49372.666666666664,
+ "minBatchDuration": 49251,
+ "maxBatchDuration": 49454,
+ "p50Duration": 49413,
+ "p95Duration": 49454,
+ "p99Duration": 49454
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T14:23:40.510Z",
+ "end": "2025-12-15T14:24:30.035Z"
+ }
+}
+```
+
+**Observations:**
+
+- Overall throughput: **54.52 items/sec** with 9 tenants competing
+- Fairness index: **0.9999** (nearly perfect fairness)
+- Notable patterns: All tenants completed within 1-2 seconds of each other (48-50s range). Per-tenant throughput ~6 items/sec each. **DRR is working excellently for symmetric load.**
+
+
+
+### Test A2: Throughput Baseline (Single Tenant)
+
+**Command:**
+
+```bash
+pnpm stress throughput --batch-sizes 100,500,1000 --batch-count 3
+```
+
+**Expected:** Establish throughput ceiling for single tenant.
+
+
+Results A2 (click to expand)
+
+```json
+[
+ {
+ "scenario": "throughput-100",
+ "config": {
+ "tenantCount": 1,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 300,
+ "totalBatches": 3,
+ "overallThroughput": 82.57638315441784,
+ "overallDuration": 3633,
+ "fairnessIndex": 1,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 3570,
+ "avgItemsPerSecond": 84.03361344537815,
+ "avgBatchDuration": 3558.6666666666665,
+ "minBatchDuration": 3538,
+ "maxBatchDuration": 3570,
+ "p50Duration": 3568,
+ "p95Duration": 3570,
+ "p99Duration": 3570
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T14:41:19.119Z",
+ "end": "2025-12-15T14:41:22.752Z"
+ }
+ },
+ {
+ "scenario": "throughput-500",
+ "config": {
+ "tenantCount": 1,
+ "batchSize": 500,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 1500,
+ "totalBatches": 3,
+ "overallThroughput": 97.48488984207448,
+ "overallDuration": 15387,
+ "fairnessIndex": 1,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 1500,
+ "totalBatches": 3,
+ "totalDuration": 15384,
+ "avgItemsPerSecond": 97.50390015600624,
+ "avgBatchDuration": 15369,
+ "minBatchDuration": 15356,
+ "maxBatchDuration": 15384,
+ "p50Duration": 15367,
+ "p95Duration": 15384,
+ "p99Duration": 15384
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T14:41:23.069Z",
+ "end": "2025-12-15T14:41:38.456Z"
+ }
+ },
+ {
+ "scenario": "throughput-1000",
+ "config": {
+ "tenantCount": 1,
+ "batchSize": 1000,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 3000,
+ "totalBatches": 3,
+ "overallThroughput": 88.60796880999499,
+ "overallDuration": 33857,
+ "fairnessIndex": 1,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 3000,
+ "totalBatches": 3,
+ "totalDuration": 33839,
+ "avgItemsPerSecond": 88.65510210112592,
+ "avgBatchDuration": 33109,
+ "minBatchDuration": 32731,
+ "maxBatchDuration": 33827,
+ "p50Duration": 32769,
+ "p95Duration": 33827,
+ "p99Duration": 33827
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T14:41:38.860Z",
+ "end": "2025-12-15T14:42:12.717Z"
+ }
+ }
+]
+```
+
+**Observations:**
+
+- Max throughput achieved: **97.48 items/sec** (at batch size 500)
+- Scaling behavior: **Non-linear.** 100 items → 82.58/sec, 500 items → 97.48/sec (peak), 1000 items → 88.61/sec. Sweet spot around 500 items per batch. Larger batches may introduce overhead or hit concurrency limits.
+
+
+
+---
+
+## Series B: DRR Quantum Variations
+
+**Objective:** Understand how quantum size affects fairness vs. throughput tradeoff.
+
+**Theory:**
+
+- Lower quantum = more frequent tenant switching = better fairness, possibly lower throughput
+- Higher quantum = longer bursts per tenant = potentially higher throughput, worse fairness
+
+### Test B1: Very Low Quantum (quantum=1)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=1
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Low quantum=1"
+```
+
+
+Results B1 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "Low quantum=1",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 66.08899985313556,
+ "overallDuration": 40854,
+ "fairnessIndex": 0.9997008998554137,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 40769,
+ "avgItemsPerSecond": 7.358532218106895,
+ "avgBatchDuration": 36850.333333333336,
+ "minBatchDuration": 29016,
+ "maxBatchDuration": 40769,
+ "p50Duration": 40766,
+ "p95Duration": 40769,
+ "p99Duration": 40769
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 40771,
+ "avgItemsPerSecond": 7.358171249172206,
+ "avgBatchDuration": 40408.333333333336,
+ "minBatchDuration": 39684,
+ "maxBatchDuration": 40771,
+ "p50Duration": 40770,
+ "p95Duration": 40771,
+ "p99Duration": 40771
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 40754,
+ "avgItemsPerSecond": 7.361240614418216,
+ "avgBatchDuration": 40106,
+ "minBatchDuration": 39076,
+ "maxBatchDuration": 40753,
+ "p50Duration": 40489,
+ "p95Duration": 40753,
+ "p99Duration": 40753
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 40770,
+ "avgItemsPerSecond": 7.358351729212656,
+ "avgBatchDuration": 40717.333333333336,
+ "minBatchDuration": 40616,
+ "maxBatchDuration": 40769,
+ "p50Duration": 40767,
+ "p95Duration": 40769,
+ "p99Duration": 40769
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 40767,
+ "avgItemsPerSecond": 7.358893222459342,
+ "avgBatchDuration": 40610,
+ "minBatchDuration": 40299,
+ "maxBatchDuration": 40766,
+ "p50Duration": 40765,
+ "p95Duration": 40766,
+ "p99Duration": 40766
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 40780,
+ "avgItemsPerSecond": 7.356547327121138,
+ "avgBatchDuration": 40681.666666666664,
+ "minBatchDuration": 40497,
+ "maxBatchDuration": 40778,
+ "p50Duration": 40770,
+ "p95Duration": 40778,
+ "p99Duration": 40778
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 40771,
+ "avgItemsPerSecond": 7.358171249172206,
+ "avgBatchDuration": 40766.333333333336,
+ "minBatchDuration": 40764,
+ "maxBatchDuration": 40769,
+ "p50Duration": 40766,
+ "p95Duration": 40769,
+ "p99Duration": 40769
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 38628,
+ "avgItemsPerSecond": 7.766387076731904,
+ "avgBatchDuration": 34753,
+ "minBatchDuration": 28057,
+ "maxBatchDuration": 38627,
+ "p50Duration": 37575,
+ "p95Duration": 38627,
+ "p99Duration": 38627
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 40754,
+ "avgItemsPerSecond": 7.361240614418216,
+ "avgBatchDuration": 40018.333333333336,
+ "minBatchDuration": 39630,
+ "maxBatchDuration": 40754,
+ "p50Duration": 39671,
+ "p95Duration": 40754,
+ "p99Duration": 40754
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T14:48:32.814Z",
+ "end": "2025-12-15T14:49:13.668Z"
+ }
+}
+```
+
+**Observations:**
+
+- Fairness index: **0.9997** (vs baseline: **0.9999**) - nearly identical
+- Throughput: **66.09 items/sec** (vs baseline: **54.52**) - +21% improvement!
+- Context switching overhead visible?: **Not significantly.** Despite switching tenants every 1 item, throughput actually improved. The tighter scheduling may have better utilized concurrency slots.
+
+
+
+### Test B2: Medium Quantum (quantum=10)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=10
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Medium quantum=10"
+```
+
+
+Results B2 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "Medium quantum=10",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 64.68615237182559,
+ "overallDuration": 41740,
+ "fairnessIndex": 0.9998055065601579,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 40743,
+ "avgItemsPerSecond": 7.363228039172373,
+ "avgBatchDuration": 37656,
+ "minBatchDuration": 31484,
+ "maxBatchDuration": 40743,
+ "p50Duration": 40741,
+ "p95Duration": 40743,
+ "p99Duration": 40743
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 40745,
+ "avgItemsPerSecond": 7.362866609399926,
+ "avgBatchDuration": 36601.666666666664,
+ "minBatchDuration": 28318,
+ "maxBatchDuration": 40745,
+ "p50Duration": 40742,
+ "p95Duration": 40745,
+ "p99Duration": 40745
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 41664,
+ "avgItemsPerSecond": 7.200460829493087,
+ "avgBatchDuration": 38524.333333333336,
+ "minBatchDuration": 32253,
+ "maxBatchDuration": 41660,
+ "p50Duration": 41660,
+ "p95Duration": 41660,
+ "p99Duration": 41660
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 41443,
+ "avgItemsPerSecond": 7.238858190768043,
+ "avgBatchDuration": 36661.333333333336,
+ "minBatchDuration": 32251,
+ "maxBatchDuration": 41443,
+ "p50Duration": 36290,
+ "p95Duration": 41443,
+ "p99Duration": 41443
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 40436,
+ "avgItemsPerSecond": 7.419131467009596,
+ "avgBatchDuration": 40406,
+ "minBatchDuration": 40349,
+ "maxBatchDuration": 40436,
+ "p50Duration": 40433,
+ "p95Duration": 40436,
+ "p99Duration": 40436
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 41044,
+ "avgItemsPerSecond": 7.309229119968814,
+ "avgBatchDuration": 39122,
+ "minBatchDuration": 35972,
+ "maxBatchDuration": 41040,
+ "p50Duration": 40354,
+ "p95Duration": 41040,
+ "p99Duration": 41040
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 41683,
+ "avgItemsPerSecond": 7.197178705947269,
+ "avgBatchDuration": 41325,
+ "minBatchDuration": 40636,
+ "maxBatchDuration": 41683,
+ "p50Duration": 41656,
+ "p95Duration": 41683,
+ "p99Duration": 41683
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 41412,
+ "avgItemsPerSecond": 7.244277021153288,
+ "avgBatchDuration": 40959.666666666664,
+ "minBatchDuration": 40735,
+ "maxBatchDuration": 41406,
+ "p50Duration": 40738,
+ "p95Duration": 41406,
+ "p99Duration": 41406
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 39914,
+ "avgItemsPerSecond": 7.516159743448415,
+ "avgBatchDuration": 36645,
+ "minBatchDuration": 32034,
+ "maxBatchDuration": 39914,
+ "p50Duration": 37987,
+ "p95Duration": 39914,
+ "p99Duration": 39914
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T14:51:13.796Z",
+ "end": "2025-12-15T14:51:55.536Z"
+ }
+}
+```
+
+**Observations:**
+
+- Fairness index: **0.9998** (excellent)
+- Throughput: **64.69 items/sec** (vs quantum=1: 66.09, slightly lower)
+- Comparison to baseline: **+18% throughput vs baseline.** Higher quantum didn't provide expected gains; the overhead theory is not supported by evidence.
+
+
+
+### Test B3: High Quantum (quantum=25)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=25
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "High quantum=25"
+```
+
+
+Results B3 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "High quantum=25",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 84.3644544431946,
+ "overallDuration": 32004,
+ "fairnessIndex": 0.9999195340273302,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 31137,
+ "avgItemsPerSecond": 9.634839579920994,
+ "avgBatchDuration": 31135.333333333332,
+ "minBatchDuration": 31134,
+ "maxBatchDuration": 31137,
+ "p50Duration": 31135,
+ "p95Duration": 31137,
+ "p99Duration": 31137
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 31135,
+ "avgItemsPerSecond": 9.635458487233016,
+ "avgBatchDuration": 30287.333333333332,
+ "minBatchDuration": 29792,
+ "maxBatchDuration": 31133,
+ "p50Duration": 29937,
+ "p95Duration": 31133,
+ "p99Duration": 31133
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 31197,
+ "avgItemsPerSecond": 9.616309260505819,
+ "avgBatchDuration": 24641,
+ "minBatchDuration": 18973,
+ "maxBatchDuration": 31197,
+ "p50Duration": 23753,
+ "p95Duration": 31197,
+ "p99Duration": 31197
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 31105,
+ "avgItemsPerSecond": 9.644751647645073,
+ "avgBatchDuration": 29303,
+ "minBatchDuration": 25964,
+ "maxBatchDuration": 31105,
+ "p50Duration": 30840,
+ "p95Duration": 31105,
+ "p99Duration": 31105
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 31120,
+ "avgItemsPerSecond": 9.640102827763496,
+ "avgBatchDuration": 31006.333333333332,
+ "minBatchDuration": 30835,
+ "maxBatchDuration": 31120,
+ "p50Duration": 31064,
+ "p95Duration": 31120,
+ "p99Duration": 31120
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 31075,
+ "avgItemsPerSecond": 9.654062751407883,
+ "avgBatchDuration": 24953.333333333332,
+ "minBatchDuration": 21079,
+ "maxBatchDuration": 31073,
+ "p50Duration": 22708,
+ "p95Duration": 31073,
+ "p99Duration": 31073
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 31965,
+ "avgItemsPerSecond": 9.385265133740027,
+ "avgBatchDuration": 25924.666666666668,
+ "minBatchDuration": 22904,
+ "maxBatchDuration": 31964,
+ "p50Duration": 22906,
+ "p95Duration": 31964,
+ "p99Duration": 31964
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 31106,
+ "avgItemsPerSecond": 9.644441586832123,
+ "avgBatchDuration": 26298.333333333332,
+ "minBatchDuration": 16867,
+ "maxBatchDuration": 31106,
+ "p50Duration": 30922,
+ "p95Duration": 31106,
+ "p99Duration": 31106
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 30886,
+ "avgItemsPerSecond": 9.713138638865505,
+ "avgBatchDuration": 30883,
+ "minBatchDuration": 30881,
+ "maxBatchDuration": 30884,
+ "p50Duration": 30884,
+ "p95Duration": 30884,
+ "p99Duration": 30884
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T14:54:10.280Z",
+ "end": "2025-12-15T14:54:42.284Z"
+ }
+}
+```
+
+**Observations:**
+
+- Fairness index: **0.9999** (best in B series!)
+- Throughput: **84.36 items/sec** (**BEST in B series**, +55% vs baseline)
+- Evidence of tenant starvation?: **None.** All tenants completed in 30-32s range. Higher quantum=25 actually provided better throughput AND fairness!
+
+
+
+### Test B4: Very High Quantum (quantum=50)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=50
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Very high quantum=50"
+```
+
+
+Results B4 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "Very high quantum=50",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 51.97605251506343,
+ "overallDuration": 51947,
+ "fairnessIndex": 0.9997441540697334,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 51416,
+ "avgItemsPerSecond": 5.834759607904155,
+ "avgBatchDuration": 51002,
+ "minBatchDuration": 50774,
+ "maxBatchDuration": 51416,
+ "p50Duration": 50816,
+ "p95Duration": 51416,
+ "p99Duration": 51416
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 51847,
+ "avgItemsPerSecond": 5.786255713927518,
+ "avgBatchDuration": 51840.666666666664,
+ "minBatchDuration": 51838,
+ "maxBatchDuration": 51843,
+ "p50Duration": 51841,
+ "p95Duration": 51843,
+ "p99Duration": 51843
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 51252,
+ "avgItemsPerSecond": 5.853430110044486,
+ "avgBatchDuration": 51034.333333333336,
+ "minBatchDuration": 50778,
+ "maxBatchDuration": 51250,
+ "p50Duration": 51075,
+ "p95Duration": 51250,
+ "p99Duration": 51250
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 50778,
+ "avgItemsPerSecond": 5.908070424199456,
+ "avgBatchDuration": 50768.666666666664,
+ "minBatchDuration": 50765,
+ "maxBatchDuration": 50776,
+ "p50Duration": 50765,
+ "p95Duration": 50776,
+ "p99Duration": 50776
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 49692,
+ "avgItemsPerSecond": 6.037189084762135,
+ "avgBatchDuration": 49689.666666666664,
+ "minBatchDuration": 49687,
+ "maxBatchDuration": 49692,
+ "p50Duration": 49690,
+ "p95Duration": 49692,
+ "p99Duration": 49692
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 51843,
+ "avgItemsPerSecond": 5.786702158439905,
+ "avgBatchDuration": 51842,
+ "minBatchDuration": 51841,
+ "maxBatchDuration": 51843,
+ "p50Duration": 51842,
+ "p95Duration": 51843,
+ "p99Duration": 51843
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 50821,
+ "avgItemsPerSecond": 5.9030715649042715,
+ "avgBatchDuration": 50794.666666666664,
+ "minBatchDuration": 50779,
+ "maxBatchDuration": 50821,
+ "p50Duration": 50784,
+ "p95Duration": 50821,
+ "p99Duration": 50821
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 49689,
+ "avgItemsPerSecond": 6.037553583288052,
+ "avgBatchDuration": 49682,
+ "minBatchDuration": 49678,
+ "maxBatchDuration": 49689,
+ "p50Duration": 49679,
+ "p95Duration": 49689,
+ "p99Duration": 49689
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 51838,
+ "avgItemsPerSecond": 5.787260310968787,
+ "avgBatchDuration": 51483.333333333336,
+ "minBatchDuration": 50775,
+ "maxBatchDuration": 51838,
+ "p50Duration": 51837,
+ "p95Duration": 51838,
+ "p99Duration": 51838
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T14:58:47.844Z",
+ "end": "2025-12-15T14:59:39.791Z"
+ }
+}
+```
+
+**Observations:**
+
+- Fairness index: **0.9997** (still excellent)
+- Throughput: **51.98 items/sec** (**WORST in B series**, -5% vs baseline)
+- Notable patterns: **Diminishing returns / counterproductive.** Very high quantum=50 actually hurt throughput. With 9 tenants each getting 50-item bursts, tenants wait too long for their turn. **Sweet spot appears to be quantum=25.**
+
+
+
+---
+
+## Series C: Max Deficit Variations
+
+**Objective:** Understand how max deficit cap affects catch-up behavior for starved tenants.
+
+**Theory:**
+
+- Lower max deficit = limits how much a starved tenant can "catch up"
+- Higher max deficit = allows more aggressive catch-up, but could starve other tenants
+
+### Test C1: Low Max Deficit (maxDeficit=10)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=10
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Low maxDeficit=10"
+```
+
+
+Results C1 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "Low maxDeficit=10",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 52.51385782359234,
+ "overallDuration": 51415,
+ "fairnessIndex": 0.9997739979483051,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 49624,
+ "avgItemsPerSecond": 6.0454618732871195,
+ "avgBatchDuration": 49357.333333333336,
+ "minBatchDuration": 48832,
+ "maxBatchDuration": 49624,
+ "p50Duration": 49616,
+ "p95Duration": 49624,
+ "p99Duration": 49624
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 51230,
+ "avgItemsPerSecond": 5.8559437829396845,
+ "avgBatchDuration": 50431,
+ "minBatchDuration": 49201,
+ "maxBatchDuration": 51230,
+ "p50Duration": 50862,
+ "p95Duration": 51230,
+ "p99Duration": 51230
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 51410,
+ "avgItemsPerSecond": 5.83544057576347,
+ "avgBatchDuration": 49448.333333333336,
+ "minBatchDuration": 45734,
+ "maxBatchDuration": 51410,
+ "p50Duration": 51201,
+ "p95Duration": 51410,
+ "p99Duration": 51410
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 49206,
+ "avgItemsPerSecond": 6.096817461285209,
+ "avgBatchDuration": 46922.333333333336,
+ "minBatchDuration": 42366,
+ "maxBatchDuration": 49201,
+ "p50Duration": 49200,
+ "p95Duration": 49201,
+ "p99Duration": 49201
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 50661,
+ "avgItemsPerSecond": 5.921714928643334,
+ "avgBatchDuration": 48617.666666666664,
+ "minBatchDuration": 46575,
+ "maxBatchDuration": 50661,
+ "p50Duration": 48617,
+ "p95Duration": 50661,
+ "p99Duration": 50661
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 50189,
+ "avgItemsPerSecond": 5.977405407559425,
+ "avgBatchDuration": 49945.666666666664,
+ "minBatchDuration": 49823,
+ "maxBatchDuration": 50189,
+ "p50Duration": 49825,
+ "p95Duration": 50189,
+ "p99Duration": 50189
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 51232,
+ "avgItemsPerSecond": 5.8557151780137415,
+ "avgBatchDuration": 50892.333333333336,
+ "minBatchDuration": 50636,
+ "maxBatchDuration": 51229,
+ "p50Duration": 50812,
+ "p95Duration": 51229,
+ "p99Duration": 51229
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 49639,
+ "avgItemsPerSecond": 6.0436350450250815,
+ "avgBatchDuration": 44099,
+ "minBatchDuration": 41326,
+ "maxBatchDuration": 49638,
+ "p50Duration": 41333,
+ "p95Duration": 49638,
+ "p99Duration": 49638
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 50193,
+ "avgItemsPerSecond": 5.976929053852131,
+ "avgBatchDuration": 49979.666666666664,
+ "minBatchDuration": 49623,
+ "maxBatchDuration": 50193,
+ "p50Duration": 50123,
+ "p95Duration": 50193,
+ "p99Duration": 50193
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:01:17.027Z",
+ "end": "2025-12-15T15:02:08.442Z"
+ }
+}
+```
+
+**Observations:**
+
+- Fairness index: **0.9998** (excellent)
+- Throughput: **52.51 items/sec** (similar to baseline)
+- Recovery behavior: **Constrained.** Low max deficit limits catch-up ability. Total duration 51.4s, consistent with limited burst capacity.
+
+
+
+### Test C2: Medium Max Deficit (maxDeficit=25)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=25
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Medium maxDeficit=25"
+```
+
+
+Results C2 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "Medium maxDeficit=25",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 66.32928806564142,
+ "overallDuration": 40706,
+ "fairnessIndex": 0.9992454865624668,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 39801,
+ "avgItemsPerSecond": 7.537499057812618,
+ "avgBatchDuration": 35339.333333333336,
+ "minBatchDuration": 28824,
+ "maxBatchDuration": 39800,
+ "p50Duration": 37394,
+ "p95Duration": 39800,
+ "p99Duration": 39800
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 39487,
+ "avgItemsPerSecond": 7.597437131207739,
+ "avgBatchDuration": 36988.666666666664,
+ "minBatchDuration": 34055,
+ "maxBatchDuration": 39486,
+ "p50Duration": 37425,
+ "p95Duration": 39486,
+ "p99Duration": 39486
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 37449,
+ "avgItemsPerSecond": 8.010894816951053,
+ "avgBatchDuration": 36409.333333333336,
+ "minBatchDuration": 34337,
+ "maxBatchDuration": 37447,
+ "p50Duration": 37444,
+ "p95Duration": 37447,
+ "p99Duration": 37447
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 40648,
+ "avgItemsPerSecond": 7.380436921865774,
+ "avgBatchDuration": 39841.666666666664,
+ "minBatchDuration": 39419,
+ "maxBatchDuration": 40647,
+ "p50Duration": 39459,
+ "p95Duration": 40647,
+ "p99Duration": 40647
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 38760,
+ "avgItemsPerSecond": 7.739938080495356,
+ "avgBatchDuration": 36378.333333333336,
+ "minBatchDuration": 33012,
+ "maxBatchDuration": 38750,
+ "p50Duration": 37373,
+ "p95Duration": 38750,
+ "p99Duration": 38750
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 37706,
+ "avgItemsPerSecond": 7.956293428101628,
+ "avgBatchDuration": 37609.333333333336,
+ "minBatchDuration": 37424,
+ "maxBatchDuration": 37706,
+ "p50Duration": 37698,
+ "p95Duration": 37706,
+ "p99Duration": 37706
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 40621,
+ "avgItemsPerSecond": 7.385342556805593,
+ "avgBatchDuration": 38921,
+ "minBatchDuration": 36642,
+ "maxBatchDuration": 40621,
+ "p50Duration": 39500,
+ "p95Duration": 40621,
+ "p99Duration": 40621
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 38757,
+ "avgItemsPerSecond": 7.740537193281214,
+ "avgBatchDuration": 38744,
+ "minBatchDuration": 38730,
+ "maxBatchDuration": 38757,
+ "p50Duration": 38745,
+ "p95Duration": 38757,
+ "p99Duration": 38757
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 38751,
+ "avgItemsPerSecond": 7.741735697143299,
+ "avgBatchDuration": 37674.666666666664,
+ "minBatchDuration": 35558,
+ "maxBatchDuration": 38740,
+ "p50Duration": 38726,
+ "p95Duration": 38740,
+ "p99Duration": 38740
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:03:24.382Z",
+ "end": "2025-12-15T15:04:05.088Z"
+ }
+}
+```
+
+**Observations:**
+
+- Fairness index: **0.9992** (slightly lower but still very good)
+- Throughput: **66.33 items/sec** (+22% vs baseline)
+- Recovery behavior: **Better.** More headroom for catch-up. Duration dropped from 51.4s (C1) to 40.7s.
+
+
+
+### Test C3: High Max Deficit (maxDeficit=100)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=100
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "High maxDeficit=100"
+```
+
+
+Results C3 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "High maxDeficit=100",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 73.73624272878718,
+ "overallDuration": 36617,
+ "fairnessIndex": 0.9988716605255289,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 32809,
+ "avgItemsPerSecond": 9.143832484988875,
+ "avgBatchDuration": 30391.666666666668,
+ "minBatchDuration": 28722,
+ "maxBatchDuration": 32751,
+ "p50Duration": 29702,
+ "p95Duration": 32751,
+ "p99Duration": 32751
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 36540,
+ "avgItemsPerSecond": 8.210180623973727,
+ "avgBatchDuration": 28375.666666666668,
+ "minBatchDuration": 13115,
+ "maxBatchDuration": 36537,
+ "p50Duration": 35475,
+ "p95Duration": 36537,
+ "p99Duration": 36537
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 36337,
+ "avgItemsPerSecond": 8.256047554833916,
+ "avgBatchDuration": 28524.666666666668,
+ "minBatchDuration": 13114,
+ "maxBatchDuration": 36322,
+ "p50Duration": 36138,
+ "p95Duration": 36322,
+ "p99Duration": 36322
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 35478,
+ "avgItemsPerSecond": 8.45594452900389,
+ "avgBatchDuration": 27529.333333333332,
+ "minBatchDuration": 23556,
+ "maxBatchDuration": 35475,
+ "p50Duration": 23557,
+ "p95Duration": 35475,
+ "p99Duration": 35475
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 35453,
+ "avgItemsPerSecond": 8.461907313908554,
+ "avgBatchDuration": 35291,
+ "minBatchDuration": 35039,
+ "maxBatchDuration": 35438,
+ "p50Duration": 35396,
+ "p95Duration": 35438,
+ "p99Duration": 35438
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 36430,
+ "avgItemsPerSecond": 8.23497117760088,
+ "avgBatchDuration": 34851.333333333336,
+ "minBatchDuration": 31705,
+ "maxBatchDuration": 36425,
+ "p50Duration": 36424,
+ "p95Duration": 36425,
+ "p99Duration": 36425
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 36500,
+ "avgItemsPerSecond": 8.21917808219178,
+ "avgBatchDuration": 31197,
+ "minBatchDuration": 27638,
+ "maxBatchDuration": 36495,
+ "p50Duration": 29458,
+ "p95Duration": 36495,
+ "p99Duration": 36495
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 36573,
+ "avgItemsPerSecond": 8.202772537117545,
+ "avgBatchDuration": 27671.666666666668,
+ "minBatchDuration": 23222,
+ "maxBatchDuration": 36570,
+ "p50Duration": 23223,
+ "p95Duration": 36570,
+ "p99Duration": 36570
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 35514,
+ "avgItemsPerSecond": 8.44737286703835,
+ "avgBatchDuration": 35127.666666666664,
+ "minBatchDuration": 34370,
+ "maxBatchDuration": 35511,
+ "p50Duration": 35502,
+ "p95Duration": 35511,
+ "p99Duration": 35511
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:06:48.615Z",
+ "end": "2025-12-15T15:07:25.232Z"
+ }
+}
+```
+
+**Observations:**
+
+- Fairness index: **0.9989** (slightly lower due to more aggressive catch-up)
+- Throughput: **73.74 items/sec** (**+35% vs baseline**, best in C series)
+- Any evidence of new tenant starvation during catch-up?: **Minor variance.** Some tenants took 13-37s for batches (wider spread) but all finished within reasonable bounds. **Higher max deficit = better throughput but slightly more variance.**
+
+
+
+---
+
+## Series D: Consumer Count Variations
+
+**Objective:** Test parallelism within a single process.
+
+**Theory:**
+
+- More consumers = higher potential throughput (limited by concurrency limits)
+- Diminishing returns expected as consumers contend for the same work
+
+### Test D1: Single Consumer (baseline)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=1
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Single consumer"
+```
+
+
+Results D1 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "Single consumer",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 41.326108917255944,
+ "overallDuration": 65334,
+ "fairnessIndex": 0.9996736075271871,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 62086,
+ "avgItemsPerSecond": 4.832007215797443,
+ "avgBatchDuration": 60490.333333333336,
+ "minBatchDuration": 59021,
+ "maxBatchDuration": 62084,
+ "p50Duration": 60366,
+ "p95Duration": 62084,
+ "p99Duration": 62084
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 65000,
+ "avgItemsPerSecond": 4.615384615384616,
+ "avgBatchDuration": 63631.333333333336,
+ "minBatchDuration": 61952,
+ "maxBatchDuration": 64991,
+ "p50Duration": 63951,
+ "p95Duration": 64991,
+ "p99Duration": 64991
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 62539,
+ "avgItemsPerSecond": 4.797006667839269,
+ "avgBatchDuration": 60631,
+ "minBatchDuration": 58991,
+ "maxBatchDuration": 62536,
+ "p50Duration": 60366,
+ "p95Duration": 62536,
+ "p99Duration": 62536
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 64125,
+ "avgItemsPerSecond": 4.678362573099415,
+ "avgBatchDuration": 61768.666666666664,
+ "minBatchDuration": 58220,
+ "maxBatchDuration": 64118,
+ "p50Duration": 62968,
+ "p95Duration": 64118,
+ "p99Duration": 64118
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 62536,
+ "avgItemsPerSecond": 4.797236791608034,
+ "avgBatchDuration": 55644,
+ "minBatchDuration": 48107,
+ "maxBatchDuration": 62516,
+ "p50Duration": 56309,
+ "p95Duration": 62516,
+ "p99Duration": 62516
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 65257,
+ "avgItemsPerSecond": 4.597207962364191,
+ "avgBatchDuration": 62698.666666666664,
+ "minBatchDuration": 60933,
+ "maxBatchDuration": 65257,
+ "p50Duration": 61906,
+ "p95Duration": 65257,
+ "p99Duration": 65257
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 62654,
+ "avgItemsPerSecond": 4.788201870590864,
+ "avgBatchDuration": 60963.666666666664,
+ "minBatchDuration": 59342,
+ "maxBatchDuration": 62654,
+ "p50Duration": 60895,
+ "p95Duration": 62654,
+ "p99Duration": 62654
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 62966,
+ "avgItemsPerSecond": 4.76447606644856,
+ "avgBatchDuration": 61417.333333333336,
+ "minBatchDuration": 60031,
+ "maxBatchDuration": 62961,
+ "p50Duration": 61260,
+ "p95Duration": 62961,
+ "p99Duration": 62961
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 64690,
+ "avgItemsPerSecond": 4.637501932292472,
+ "avgBatchDuration": 59325,
+ "minBatchDuration": 51023,
+ "maxBatchDuration": 64686,
+ "p50Duration": 62266,
+ "p95Duration": 64686,
+ "p99Duration": 64686
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:09:10.445Z",
+ "end": "2025-12-15T15:10:15.779Z"
+ }
+}
+```
+
+**Observations:**
+
+- Throughput: **41.33 items/sec** (lowest in D series, -24% vs baseline)
+- Fairness: **0.9997** (excellent)
+- **Single consumer is a bottleneck.** Total duration 65.3s. Each tenant averaging only ~4.6 items/sec.
+
+
+
+### Test D2: Multiple Consumers (count=3)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "3 consumers"
+```
+
+
+Results D2 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "3 consumers",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 57.55211663895639,
+ "overallDuration": 46914,
+ "fairnessIndex": 0.9992021047885379,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 46746,
+ "avgItemsPerSecond": 6.417661404184315,
+ "avgBatchDuration": 46353,
+ "minBatchDuration": 45585,
+ "maxBatchDuration": 46746,
+ "p50Duration": 46728,
+ "p95Duration": 46746,
+ "p99Duration": 46746
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 46812,
+ "avgItemsPerSecond": 6.40861317610869,
+ "avgBatchDuration": 46805,
+ "minBatchDuration": 46794,
+ "maxBatchDuration": 46812,
+ "p50Duration": 46809,
+ "p95Duration": 46812,
+ "p99Duration": 46812
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 46606,
+ "avgItemsPerSecond": 6.436939449856242,
+ "avgBatchDuration": 38880,
+ "minBatchDuration": 33105,
+ "maxBatchDuration": 46604,
+ "p50Duration": 36931,
+ "p95Duration": 46604,
+ "p99Duration": 46604
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 46821,
+ "avgItemsPerSecond": 6.407381303261357,
+ "avgBatchDuration": 46059,
+ "minBatchDuration": 45679,
+ "maxBatchDuration": 46810,
+ "p50Duration": 45688,
+ "p95Duration": 46810,
+ "p99Duration": 46810
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 43729,
+ "avgItemsPerSecond": 6.860435866358709,
+ "avgBatchDuration": 42251,
+ "minBatchDuration": 39304,
+ "maxBatchDuration": 43729,
+ "p50Duration": 43720,
+ "p95Duration": 43729,
+ "p99Duration": 43729
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 46597,
+ "avgItemsPerSecond": 6.438182715625469,
+ "avgBatchDuration": 45484.333333333336,
+ "minBatchDuration": 44415,
+ "maxBatchDuration": 46567,
+ "p50Duration": 45471,
+ "p95Duration": 46567,
+ "p99Duration": 46567
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 46808,
+ "avgItemsPerSecond": 6.409160827209024,
+ "avgBatchDuration": 45405,
+ "minBatchDuration": 43679,
+ "maxBatchDuration": 46761,
+ "p50Duration": 45775,
+ "p95Duration": 46761,
+ "p99Duration": 46761
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 46811,
+ "avgItemsPerSecond": 6.408750080109376,
+ "avgBatchDuration": 45293,
+ "minBatchDuration": 44526,
+ "maxBatchDuration": 46784,
+ "p50Duration": 44569,
+ "p95Duration": 46784,
+ "p99Duration": 46784
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 43732,
+ "avgItemsPerSecond": 6.859965242842769,
+ "avgBatchDuration": 41508,
+ "minBatchDuration": 39455,
+ "maxBatchDuration": 43581,
+ "p50Duration": 41488,
+ "p95Duration": 43581,
+ "p99Duration": 43581
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:11:42.929Z",
+ "end": "2025-12-15T15:12:29.843Z"
+ }
+}
+```
+
+**Observations:**
+
+- Throughput: **57.55 items/sec** (vs single: **1.39x** improvement)
+- Fairness impact: **0.9992** (very minor degradation)
+- Duration dropped from 65.3s to 46.9s. **3 consumers is a good balance.**
+
+
+
+### Test D3: Many Consumers (count=5)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=5
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "5 consumers"
+```
+
+
+Results D3 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "5 consumers",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 63.80263717566993,
+ "overallDuration": 42318,
+ "fairnessIndex": 0.9999000645903268,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 41203,
+ "avgItemsPerSecond": 7.281023226464092,
+ "avgBatchDuration": 40931,
+ "minBatchDuration": 40810,
+ "maxBatchDuration": 41167,
+ "p50Duration": 40816,
+ "p95Duration": 41167,
+ "p99Duration": 41167
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 41814,
+ "avgItemsPerSecond": 7.1746305065289135,
+ "avgBatchDuration": 37657.333333333336,
+ "minBatchDuration": 35575,
+ "maxBatchDuration": 41814,
+ "p50Duration": 35583,
+ "p95Duration": 41814,
+ "p99Duration": 41814
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 41795,
+ "avgItemsPerSecond": 7.177892092355545,
+ "avgBatchDuration": 37201.666666666664,
+ "minBatchDuration": 34298,
+ "maxBatchDuration": 41795,
+ "p50Duration": 35512,
+ "p95Duration": 41795,
+ "p99Duration": 41795
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 41837,
+ "avgItemsPerSecond": 7.170686234672658,
+ "avgBatchDuration": 40580.333333333336,
+ "minBatchDuration": 38694,
+ "maxBatchDuration": 41833,
+ "p50Duration": 41214,
+ "p95Duration": 41833,
+ "p99Duration": 41833
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 41829,
+ "avgItemsPerSecond": 7.1720576633436135,
+ "avgBatchDuration": 39891.666666666664,
+ "minBatchDuration": 37128,
+ "maxBatchDuration": 41786,
+ "p50Duration": 40761,
+ "p95Duration": 41786,
+ "p99Duration": 41786
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 42246,
+ "avgItemsPerSecond": 7.10126402499645,
+ "avgBatchDuration": 34807.333333333336,
+ "minBatchDuration": 28353,
+ "maxBatchDuration": 42154,
+ "p50Duration": 33915,
+ "p95Duration": 42154,
+ "p99Duration": 42154
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 41654,
+ "avgItemsPerSecond": 7.202189465597542,
+ "avgBatchDuration": 40819.333333333336,
+ "minBatchDuration": 40181,
+ "maxBatchDuration": 41653,
+ "p50Duration": 40624,
+ "p95Duration": 41653,
+ "p99Duration": 41653
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 41210,
+ "avgItemsPerSecond": 7.279786459597185,
+ "avgBatchDuration": 40886.333333333336,
+ "minBatchDuration": 40630,
+ "maxBatchDuration": 41210,
+ "p50Duration": 40819,
+ "p95Duration": 41210,
+ "p99Duration": 41210
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 40813,
+ "avgItemsPerSecond": 7.350599073824517,
+ "avgBatchDuration": 37843.666666666664,
+ "minBatchDuration": 31921,
+ "maxBatchDuration": 40810,
+ "p50Duration": 40800,
+ "p95Duration": 40810,
+ "p99Duration": 40810
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:13:47.964Z",
+ "end": "2025-12-15T15:14:30.282Z"
+ }
+}
+```
+
+**Observations:**
+
+- Throughput: **63.80 items/sec** (vs 3 consumers: **1.11x** improvement)
+- Fairness: **0.9999** (nearly perfect!)
+- Diminishing returns?: **Yes, starting to see it.** Going from 3→5 consumers added ~11% throughput. Still beneficial but less dramatic.
+
+
+
+### Test D4: High Consumer Count (count=10)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=10
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "10 consumers"
+```
+
+
+Results D4 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "10 consumers",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 114.94252873563218,
+ "overallDuration": 23490,
+ "fairnessIndex": 0.9887990604258773,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 23004,
+ "avgItemsPerSecond": 13.041210224308816,
+ "avgBatchDuration": 21971,
+ "minBatchDuration": 19947,
+ "maxBatchDuration": 23004,
+ "p50Duration": 22962,
+ "p95Duration": 23004,
+ "p99Duration": 23004
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 22777,
+ "avgItemsPerSecond": 13.171181454976512,
+ "avgBatchDuration": 21004.666666666668,
+ "minBatchDuration": 18477,
+ "maxBatchDuration": 22775,
+ "p50Duration": 21762,
+ "p95Duration": 22775,
+ "p99Duration": 22775
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 21772,
+ "avgItemsPerSecond": 13.77916590115745,
+ "avgBatchDuration": 21014.666666666668,
+ "minBatchDuration": 19505,
+ "maxBatchDuration": 21772,
+ "p50Duration": 21767,
+ "p95Duration": 21772,
+ "p99Duration": 21772
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 22339,
+ "avgItemsPerSecond": 13.42942835399973,
+ "avgBatchDuration": 21297.333333333332,
+ "minBatchDuration": 20240,
+ "maxBatchDuration": 22339,
+ "p50Duration": 21313,
+ "p95Duration": 22339,
+ "p99Duration": 22339
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 22597,
+ "avgItemsPerSecond": 13.276098597158914,
+ "avgBatchDuration": 20592.333333333332,
+ "minBatchDuration": 17413,
+ "maxBatchDuration": 22597,
+ "p50Duration": 21767,
+ "p95Duration": 22597,
+ "p99Duration": 22597
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 21762,
+ "avgItemsPerSecond": 13.785497656465399,
+ "avgBatchDuration": 21700,
+ "minBatchDuration": 21578,
+ "maxBatchDuration": 21762,
+ "p50Duration": 21760,
+ "p95Duration": 21762,
+ "p99Duration": 21762
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 22275,
+ "avgItemsPerSecond": 13.468013468013467,
+ "avgBatchDuration": 22239.333333333332,
+ "minBatchDuration": 22218,
+ "maxBatchDuration": 22275,
+ "p50Duration": 22225,
+ "p95Duration": 22275,
+ "p99Duration": 22275
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 16719,
+ "avgItemsPerSecond": 17.94365691727974,
+ "avgBatchDuration": 15697,
+ "minBatchDuration": 13656,
+ "maxBatchDuration": 16719,
+ "p50Duration": 16716,
+ "p95Duration": 16719,
+ "p99Duration": 16719
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 23387,
+ "avgItemsPerSecond": 12.827639286783254,
+ "avgBatchDuration": 23160.333333333332,
+ "minBatchDuration": 22770,
+ "maxBatchDuration": 23387,
+ "p50Duration": 23324,
+ "p95Duration": 23387,
+ "p99Duration": 23387
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:16:28.939Z",
+ "end": "2025-12-15T15:16:52.429Z"
+ }
+}
+```
+
+**Observations:**
+
+- Throughput: **114.94 items/sec** (**BEST IN ALL TESTS SO FAR!** 2.78x vs single consumer)
+- Fairness: **0.9888** (noticeably lower - some tenants got 17s while others got 23s)
+- CPU impact: **Not measured, but likely higher.**
+- Contention visible?: **Yes, fairness degradation suggests some contention/scheduling variance.** One tenant (org-3:efaelbvnogkhjnrdfsmi) finished in 16.7s while others took 21-23s. **10 consumers is great for throughput but comes at fairness cost.**
+
+
+
+---
+
+## Series E: Consumer Interval Variations
+
+**Objective:** Test polling frequency impact on throughput and latency.
+
+**Theory:**
+
+- Lower interval = more responsive, higher CPU usage
+- Higher interval = lower CPU, higher latency for new work
+
+### Test E1: Fast Polling (interval=20ms)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_QUEUE_CONSUMER_INTERVAL_MS=20
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Fast polling 20ms"
+```
+
+
+Results E1 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "Fast polling 20ms",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 70.23019898556379,
+ "overallDuration": 38445,
+ "fairnessIndex": 0.9979118686098942,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 38383,
+ "avgItemsPerSecond": 7.815960190709429,
+ "avgBatchDuration": 32665.333333333332,
+ "minBatchDuration": 28219,
+ "maxBatchDuration": 38381,
+ "p50Duration": 31396,
+ "p95Duration": 38381,
+ "p99Duration": 38381
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 37376,
+ "avgItemsPerSecond": 8.026541095890412,
+ "avgBatchDuration": 37178.333333333336,
+ "minBatchDuration": 36801,
+ "maxBatchDuration": 37376,
+ "p50Duration": 37358,
+ "p95Duration": 37376,
+ "p99Duration": 37376
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 37361,
+ "avgItemsPerSecond": 8.029763657289687,
+ "avgBatchDuration": 29490.666666666668,
+ "minBatchDuration": 23952,
+ "maxBatchDuration": 37361,
+ "p50Duration": 27159,
+ "p95Duration": 37361,
+ "p99Duration": 37361
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 37997,
+ "avgItemsPerSecond": 7.895360160012633,
+ "avgBatchDuration": 36532,
+ "minBatchDuration": 34226,
+ "maxBatchDuration": 37994,
+ "p50Duration": 37376,
+ "p95Duration": 37994,
+ "p99Duration": 37994
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 37889,
+ "avgItemsPerSecond": 7.917865343503392,
+ "avgBatchDuration": 36989,
+ "minBatchDuration": 35364,
+ "maxBatchDuration": 37867,
+ "p50Duration": 37736,
+ "p95Duration": 37867,
+ "p99Duration": 37867
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 37359,
+ "avgItemsPerSecond": 8.030193527664016,
+ "avgBatchDuration": 36533.666666666664,
+ "minBatchDuration": 35927,
+ "maxBatchDuration": 37357,
+ "p50Duration": 36317,
+ "p95Duration": 37357,
+ "p99Duration": 37357
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 38245,
+ "avgItemsPerSecond": 7.844162635638646,
+ "avgBatchDuration": 34851.333333333336,
+ "minBatchDuration": 28081,
+ "maxBatchDuration": 38237,
+ "p50Duration": 38236,
+ "p95Duration": 38237,
+ "p99Duration": 38237
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 38090,
+ "avgItemsPerSecond": 7.876082961407193,
+ "avgBatchDuration": 36777.666666666664,
+ "minBatchDuration": 34984,
+ "maxBatchDuration": 38019,
+ "p50Duration": 37330,
+ "p95Duration": 38019,
+ "p99Duration": 38019
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 33054,
+ "avgItemsPerSecond": 9.076057360682519,
+ "avgBatchDuration": 24749.666666666668,
+ "minBatchDuration": 17350,
+ "maxBatchDuration": 33048,
+ "p50Duration": 23851,
+ "p95Duration": 33048,
+ "p99Duration": 33048
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:18:51.094Z",
+ "end": "2025-12-15T15:19:29.539Z"
+ }
+}
+```
+
+**Observations:**
+
+- Throughput: **70.23 items/sec** (good improvement over baseline)
+- Fairness: **0.9979** (slightly lower)
+- CPU usage (if observed): **Expected higher due to more frequent polling.** Faster polling didn't dramatically improve throughput - suggests bottleneck is elsewhere (processing time, not poll latency).
+
+
+
+### Test E2: Medium Polling (interval=50ms)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_QUEUE_CONSUMER_INTERVAL_MS=50
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Medium polling 50ms"
+```
+
+
+Results E2 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "Medium polling 50ms",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 93.56806210146937,
+ "overallDuration": 28856,
+ "fairnessIndex": 0.9994094367115308,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 28584,
+ "avgItemsPerSecond": 10.495382031905962,
+ "avgBatchDuration": 26949,
+ "minBatchDuration": 25606,
+ "maxBatchDuration": 28582,
+ "p50Duration": 26659,
+ "p95Duration": 28582,
+ "p99Duration": 28582
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 28733,
+ "avgItemsPerSecond": 10.440956391605472,
+ "avgBatchDuration": 28390,
+ "minBatchDuration": 27709,
+ "maxBatchDuration": 28732,
+ "p50Duration": 28729,
+ "p95Duration": 28732,
+ "p99Duration": 28732
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 26660,
+ "avgItemsPerSecond": 11.252813203300825,
+ "avgBatchDuration": 24539,
+ "minBatchDuration": 20303,
+ "maxBatchDuration": 26660,
+ "p50Duration": 26654,
+ "p95Duration": 26660,
+ "p99Duration": 26660
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 28729,
+ "avgItemsPerSecond": 10.442410108252986,
+ "avgBatchDuration": 28355.333333333332,
+ "minBatchDuration": 27708,
+ "maxBatchDuration": 28727,
+ "p50Duration": 28631,
+ "p95Duration": 28727,
+ "p99Duration": 28727
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 28732,
+ "avgItemsPerSecond": 10.44131978282055,
+ "avgBatchDuration": 23195.666666666668,
+ "minBatchDuration": 13149,
+ "maxBatchDuration": 28729,
+ "p50Duration": 27709,
+ "p95Duration": 28729,
+ "p99Duration": 28729
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 28683,
+ "avgItemsPerSecond": 10.459156991946449,
+ "avgBatchDuration": 28325,
+ "minBatchDuration": 27667,
+ "maxBatchDuration": 28680,
+ "p50Duration": 28628,
+ "p95Duration": 28680,
+ "p99Duration": 28680
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 28728,
+ "avgItemsPerSecond": 10.442773600668337,
+ "avgBatchDuration": 28049,
+ "minBatchDuration": 26791,
+ "maxBatchDuration": 28728,
+ "p50Duration": 28628,
+ "p95Duration": 28728,
+ "p99Duration": 28728
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 28829,
+ "avgItemsPerSecond": 10.406188213257483,
+ "avgBatchDuration": 28452.666666666668,
+ "minBatchDuration": 27805,
+ "maxBatchDuration": 28829,
+ "p50Duration": 28724,
+ "p95Duration": 28829,
+ "p99Duration": 28829
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 28824,
+ "avgItemsPerSecond": 10.407993338884264,
+ "avgBatchDuration": 28786.666666666668,
+ "minBatchDuration": 28727,
+ "maxBatchDuration": 28817,
+ "p50Duration": 28816,
+ "p95Duration": 28817,
+ "p99Duration": 28817
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:20:41.913Z",
+ "end": "2025-12-15T15:21:10.769Z"
+ }
+}
+```
+
+**Observations:**
+
+- Throughput: **93.57 items/sec** (**BEST in E series!** +72% vs baseline)
+- Fairness: **0.9994** (excellent)
+- **50ms polling with 3 consumers is a sweet spot.** Much better than 20ms (70.23) - suggests 20ms introduced contention overhead.
+
+
+
+### Test E3: Default Polling (interval=100ms)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_QUEUE_CONSUMER_INTERVAL_MS=100
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Default polling 100ms"
+```
+
+
+Results E3 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "Default polling 100ms",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 86.52182272639877,
+ "overallDuration": 31206,
+ "fairnessIndex": 0.9996560386487722,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 31119,
+ "avgItemsPerSecond": 9.640412609659693,
+ "avgBatchDuration": 24846,
+ "minBatchDuration": 12442,
+ "maxBatchDuration": 31116,
+ "p50Duration": 30980,
+ "p95Duration": 31116,
+ "p99Duration": 31116
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 30083,
+ "avgItemsPerSecond": 9.972409666589103,
+ "avgBatchDuration": 28193.666666666668,
+ "minBatchDuration": 24794,
+ "maxBatchDuration": 30082,
+ "p50Duration": 29705,
+ "p95Duration": 30082,
+ "p99Duration": 30082
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 30774,
+ "avgItemsPerSecond": 9.748488984207448,
+ "avgBatchDuration": 28711.333333333332,
+ "minBatchDuration": 26655,
+ "maxBatchDuration": 30774,
+ "p50Duration": 28705,
+ "p95Duration": 30774,
+ "p99Duration": 30774
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 29722,
+ "avgItemsPerSecond": 10.093533409595585,
+ "avgBatchDuration": 29380.666666666668,
+ "minBatchDuration": 28699,
+ "maxBatchDuration": 29722,
+ "p50Duration": 29721,
+ "p95Duration": 29722,
+ "p99Duration": 29722
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 30761,
+ "avgItemsPerSecond": 9.752608822860115,
+ "avgBatchDuration": 28703,
+ "minBatchDuration": 27637,
+ "maxBatchDuration": 30759,
+ "p50Duration": 27713,
+ "p95Duration": 30759,
+ "p99Duration": 30759
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 29721,
+ "avgItemsPerSecond": 10.093873019077419,
+ "avgBatchDuration": 29032.333333333332,
+ "minBatchDuration": 27674,
+ "maxBatchDuration": 29719,
+ "p50Duration": 29704,
+ "p95Duration": 29719,
+ "p99Duration": 29719
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 30089,
+ "avgItemsPerSecond": 9.97042108411712,
+ "avgBatchDuration": 23863.666666666668,
+ "minBatchDuration": 12440,
+ "maxBatchDuration": 30087,
+ "p50Duration": 29064,
+ "p95Duration": 30087,
+ "p99Duration": 30087
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 29704,
+ "avgItemsPerSecond": 10.099649878804202,
+ "avgBatchDuration": 23475,
+ "minBatchDuration": 20361,
+ "maxBatchDuration": 29701,
+ "p50Duration": 20363,
+ "p95Duration": 29701,
+ "p99Duration": 29701
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 31114,
+ "avgItemsPerSecond": 9.641961817831202,
+ "avgBatchDuration": 30969.333333333332,
+ "minBatchDuration": 30943,
+ "maxBatchDuration": 30984,
+ "p50Duration": 30981,
+ "p95Duration": 30984,
+ "p99Duration": 30984
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:22:27.627Z",
+ "end": "2025-12-15T15:22:58.833Z"
+ }
+}
+```
+
+**Observations:**
+
+- Throughput: **86.52 items/sec** (+59% vs baseline)
+- Fairness: **0.9997** (excellent)
+- 100ms default is solid but slightly lower than 50ms (93.57). **100ms is a good default for balance.**
+
+
+
+### Test E4: Slow Polling (interval=250ms)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_QUEUE_CONSUMER_INTERVAL_MS=250
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Slow polling 250ms"
+```
+
+
+Results E4 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "Slow polling 250ms",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 47.45583970471922,
+ "overallDuration": 56895,
+ "fairnessIndex": 0.9993684226112225,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 56835,
+ "avgItemsPerSecond": 5.278437582475587,
+ "avgBatchDuration": 56487.666666666664,
+ "minBatchDuration": 56107,
+ "maxBatchDuration": 56835,
+ "p50Duration": 56521,
+ "p95Duration": 56835,
+ "p99Duration": 56835
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 56711,
+ "avgItemsPerSecond": 5.289979016416568,
+ "avgBatchDuration": 48760.666666666664,
+ "minBatchDuration": 34046,
+ "maxBatchDuration": 56710,
+ "p50Duration": 55526,
+ "p95Duration": 56710,
+ "p99Duration": 56710
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 55715,
+ "avgItemsPerSecond": 5.384546351969847,
+ "avgBatchDuration": 55026.666666666664,
+ "minBatchDuration": 54680,
+ "maxBatchDuration": 55715,
+ "p50Duration": 54685,
+ "p95Duration": 55715,
+ "p99Duration": 55715
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 56114,
+ "avgItemsPerSecond": 5.346259400506113,
+ "avgBatchDuration": 47434,
+ "minBatchDuration": 42815,
+ "maxBatchDuration": 56109,
+ "p50Duration": 43378,
+ "p95Duration": 56109,
+ "p99Duration": 56109
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 52049,
+ "avgItemsPerSecond": 5.763799496628177,
+ "avgBatchDuration": 51173,
+ "minBatchDuration": 49551,
+ "maxBatchDuration": 52046,
+ "p50Duration": 51922,
+ "p95Duration": 52046,
+ "p99Duration": 52046
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 55787,
+ "avgItemsPerSecond": 5.377596931184685,
+ "avgBatchDuration": 55069.666666666664,
+ "minBatchDuration": 53713,
+ "maxBatchDuration": 55787,
+ "p50Duration": 55709,
+ "p95Duration": 55787,
+ "p99Duration": 55787
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 55790,
+ "avgItemsPerSecond": 5.377307761247535,
+ "avgBatchDuration": 47576,
+ "minBatchDuration": 31153,
+ "maxBatchDuration": 55788,
+ "p50Duration": 55787,
+ "p95Duration": 55788,
+ "p99Duration": 55788
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 56010,
+ "avgItemsPerSecond": 5.356186395286556,
+ "avgBatchDuration": 47534.333333333336,
+ "minBatchDuration": 30885,
+ "maxBatchDuration": 56010,
+ "p50Duration": 55708,
+ "p95Duration": 56010,
+ "p99Duration": 56010
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 55784,
+ "avgItemsPerSecond": 5.377886132224293,
+ "avgBatchDuration": 55374.333333333336,
+ "minBatchDuration": 54732,
+ "maxBatchDuration": 55784,
+ "p50Duration": 55607,
+ "p95Duration": 55784,
+ "p99Duration": 55784
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:24:13.441Z",
+ "end": "2025-12-15T15:25:10.336Z"
+ }
+}
+```
+
+**Observations:**
+
+- Throughput: **47.46 items/sec** (-13% vs baseline, worst in E series)
+- Fairness: **0.9994** (good)
+- Latency impact: **Significant.** Duration 56.9s vs 28.9s for 50ms. **250ms is too slow - introduces unnecessary latency between scheduling rounds.**
+
+
+
+---
+
+## Series F: Concurrency Limit Variations
+
+**Objective:** Test per-environment processing concurrency limits.
+
+**Theory:**
+
+- Lower concurrency = throttled processing per env, better resource control
+- Higher concurrency = faster per-env processing, potentially starving smaller tenants
+
+### Test F1: Low Concurrency (concurrency=2)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=2
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Low concurrency=2"
+```
+
+
+Results F1 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "Low concurrency=2",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 90.7288551362613,
+ "overallDuration": 29759,
+ "fairnessIndex": 0.9931064638844554,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 28794,
+ "avgItemsPerSecond": 10.418837257762034,
+ "avgBatchDuration": 27016,
+ "minBatchDuration": 23890,
+ "maxBatchDuration": 28794,
+ "p50Duration": 28364,
+ "p95Duration": 28794,
+ "p99Duration": 28794
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 29532,
+ "avgItemsPerSecond": 10.158472165786264,
+ "avgBatchDuration": 27720,
+ "minBatchDuration": 26284,
+ "maxBatchDuration": 29529,
+ "p50Duration": 27347,
+ "p95Duration": 29529,
+ "p99Duration": 29529
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 29439,
+ "avgItemsPerSecond": 10.19056353816366,
+ "avgBatchDuration": 22083.333333333332,
+ "minBatchDuration": 10765,
+ "maxBatchDuration": 29423,
+ "p50Duration": 26062,
+ "p95Duration": 29423,
+ "p99Duration": 29423
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 29529,
+ "avgItemsPerSecond": 10.15950421619425,
+ "avgBatchDuration": 22252.666666666668,
+ "minBatchDuration": 17683,
+ "maxBatchDuration": 29396,
+ "p50Duration": 19679,
+ "p95Duration": 29396,
+ "p99Duration": 29396
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 28474,
+ "avgItemsPerSecond": 10.535927512818711,
+ "avgBatchDuration": 26113.666666666668,
+ "minBatchDuration": 24815,
+ "maxBatchDuration": 28474,
+ "p50Duration": 25052,
+ "p95Duration": 28474,
+ "p99Duration": 28474
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 22890,
+ "avgItemsPerSecond": 13.10615989515072,
+ "avgBatchDuration": 22201.666666666668,
+ "minBatchDuration": 20834,
+ "maxBatchDuration": 22887,
+ "p50Duration": 22884,
+ "p95Duration": 22887,
+ "p99Duration": 22887
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 27952,
+ "avgItemsPerSecond": 10.732684602175159,
+ "avgBatchDuration": 21568.333333333332,
+ "minBatchDuration": 18289,
+ "maxBatchDuration": 27826,
+ "p50Duration": 18590,
+ "p95Duration": 27826,
+ "p99Duration": 27826
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 28473,
+ "avgItemsPerSecond": 10.536297545042672,
+ "avgBatchDuration": 26193.333333333332,
+ "minBatchDuration": 25054,
+ "maxBatchDuration": 28470,
+ "p50Duration": 25056,
+ "p95Duration": 28470,
+ "p99Duration": 28470
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 29668,
+ "avgItemsPerSecond": 10.111905082917621,
+ "avgBatchDuration": 28195.666666666668,
+ "minBatchDuration": 25936,
+ "maxBatchDuration": 29562,
+ "p50Duration": 29089,
+ "p95Duration": 29562,
+ "p99Duration": 29562
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:27:02.038Z",
+ "end": "2025-12-15T15:27:31.797Z"
+ }
+}
+```
+
+**Observations:**
+
+- Throughput: **90.73 items/sec** (+66% vs baseline)
+- Fairness index: **0.9931** (slightly degraded)
+- Processing time variance: **Higher.** One tenant (org-2:giomqjmqmqbcngusxqfo) finished in 22.9s while others took 28-30s. **Low concurrency=2 paradoxically gave high throughput but with fairness cost.** The system may be "batching up" work more efficiently but creating uneven distribution.
+
+
+
+### Test F2: Medium Concurrency (concurrency=5)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=5
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Medium concurrency=5"
+```
+
+
+Results F2 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "Medium concurrency=5",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 58.25368400612742,
+ "overallDuration": 46349,
+ "fairnessIndex": 0.9998661349420435,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 46278,
+ "avgItemsPerSecond": 6.482561908466226,
+ "avgBatchDuration": 45090,
+ "minBatchDuration": 43007,
+ "maxBatchDuration": 46275,
+ "p50Duration": 45988,
+ "p95Duration": 46275,
+ "p99Duration": 46275
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 45205,
+ "avgItemsPerSecond": 6.63643402278509,
+ "avgBatchDuration": 44818,
+ "minBatchDuration": 44147,
+ "maxBatchDuration": 45164,
+ "p50Duration": 45143,
+ "p95Duration": 45164,
+ "p99Duration": 45164
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 45206,
+ "avgItemsPerSecond": 6.636287218510818,
+ "avgBatchDuration": 44442,
+ "minBatchDuration": 43086,
+ "maxBatchDuration": 45206,
+ "p50Duration": 45034,
+ "p95Duration": 45206,
+ "p99Duration": 45206
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 45210,
+ "avgItemsPerSecond": 6.635700066357001,
+ "avgBatchDuration": 44481.666666666664,
+ "minBatchDuration": 43090,
+ "maxBatchDuration": 45210,
+ "p50Duration": 45145,
+ "p95Duration": 45210,
+ "p99Duration": 45210
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 45209,
+ "avgItemsPerSecond": 6.635846844654825,
+ "avgBatchDuration": 45130.333333333336,
+ "minBatchDuration": 44982,
+ "maxBatchDuration": 45207,
+ "p50Duration": 45202,
+ "p95Duration": 45207,
+ "p99Duration": 45207
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 46279,
+ "avgItemsPerSecond": 6.482421832796733,
+ "avgBatchDuration": 45474,
+ "minBatchDuration": 44160,
+ "maxBatchDuration": 46277,
+ "p50Duration": 45985,
+ "p95Duration": 46277,
+ "p99Duration": 46277
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 45209,
+ "avgItemsPerSecond": 6.635846844654825,
+ "avgBatchDuration": 44499.333333333336,
+ "minBatchDuration": 43086,
+ "maxBatchDuration": 45206,
+ "p50Duration": 45206,
+ "p95Duration": 45206,
+ "p99Duration": 45206
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 46278,
+ "avgItemsPerSecond": 6.482561908466226,
+ "avgBatchDuration": 45212,
+ "minBatchDuration": 44153,
+ "maxBatchDuration": 46276,
+ "p50Duration": 45207,
+ "p95Duration": 46276,
+ "p99Duration": 46276
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 46262,
+ "avgItemsPerSecond": 6.484803942760797,
+ "avgBatchDuration": 44808,
+ "minBatchDuration": 42006,
+ "maxBatchDuration": 46209,
+ "p50Duration": 46209,
+ "p95Duration": 46209,
+ "p99Duration": 46209
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:28:48.965Z",
+ "end": "2025-12-15T15:29:35.314Z"
+ }
+}
+```
+
+**Observations:**
+
+- Throughput: **58.25 items/sec** (+7% vs baseline)
+- Fairness index: **0.9999** (nearly perfect!)
+- **Concurrency=5 gives best fairness but modest throughput.** All tenants completed in 45-46s range (very tight clustering).
+
+
+
+### Test F3: Default Concurrency (concurrency=10)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Default concurrency=10"
+```
+
+
+Results F3 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "Default concurrency=10",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 88.71073728479432,
+ "overallDuration": 30436,
+ "fairnessIndex": 0.9986319639223729,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 29265,
+ "avgItemsPerSecond": 10.251153254741158,
+ "avgBatchDuration": 22707.666666666668,
+ "minBatchDuration": 12771,
+ "maxBatchDuration": 29256,
+ "p50Duration": 26096,
+ "p95Duration": 29256,
+ "p99Duration": 29256
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 30322,
+ "avgItemsPerSecond": 9.893806477145308,
+ "avgBatchDuration": 23069,
+ "minBatchDuration": 12719,
+ "maxBatchDuration": 30322,
+ "p50Duration": 26166,
+ "p95Duration": 30322,
+ "p99Duration": 30322
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 29049,
+ "avgItemsPerSecond": 10.327377878756584,
+ "avgBatchDuration": 26282.333333333332,
+ "minBatchDuration": 21766,
+ "maxBatchDuration": 29041,
+ "p50Duration": 28040,
+ "p95Duration": 29041,
+ "p99Duration": 29041
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 29051,
+ "avgItemsPerSecond": 10.326666896148154,
+ "avgBatchDuration": 22590.333333333332,
+ "minBatchDuration": 11757,
+ "maxBatchDuration": 29043,
+ "p50Duration": 26971,
+ "p95Duration": 29043,
+ "p99Duration": 29043
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 30247,
+ "avgItemsPerSecond": 9.918339008827322,
+ "avgBatchDuration": 22704.333333333332,
+ "minBatchDuration": 12811,
+ "maxBatchDuration": 30217,
+ "p50Duration": 25085,
+ "p95Duration": 30217,
+ "p99Duration": 30217
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 30216,
+ "avgItemsPerSecond": 9.928514694201748,
+ "avgBatchDuration": 23965.333333333332,
+ "minBatchDuration": 12747,
+ "maxBatchDuration": 30216,
+ "p50Duration": 28933,
+ "p95Duration": 30216,
+ "p99Duration": 30216
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 29163,
+ "avgItemsPerSecond": 10.287007509515483,
+ "avgBatchDuration": 29106,
+ "minBatchDuration": 29005,
+ "maxBatchDuration": 29163,
+ "p50Duration": 29150,
+ "p95Duration": 29163,
+ "p99Duration": 29163
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 26728,
+ "avgItemsPerSecond": 11.224184375935348,
+ "avgBatchDuration": 24337.666666666668,
+ "minBatchDuration": 21607,
+ "maxBatchDuration": 26726,
+ "p50Duration": 24680,
+ "p95Duration": 26726,
+ "p99Duration": 26726
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 29268,
+ "avgItemsPerSecond": 10.25010250102501,
+ "avgBatchDuration": 28511.333333333332,
+ "minBatchDuration": 27111,
+ "maxBatchDuration": 29268,
+ "p50Duration": 29155,
+ "p95Duration": 29268,
+ "p99Duration": 29268
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:31:05.764Z",
+ "end": "2025-12-15T15:31:36.200Z"
+ }
+}
+```
+
+**Observations:**
+
+- Throughput: **88.71 items/sec** (+63% vs baseline)
+- Fairness index: **0.9986** (very good)
+- **Default concurrency=10 is solid.** Good balance of throughput and fairness. Duration 30.4s.
+
+
+
+### Test F4: High Concurrency (concurrency=25)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=25
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "High concurrency=25"
+```
+
+
+Results F4 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "High concurrency=25",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 92.11873080859776,
+ "overallDuration": 29310,
+ "fairnessIndex": 0.9985800387045317,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 28196,
+ "avgItemsPerSecond": 10.63980706483189,
+ "avgBatchDuration": 21146.333333333332,
+ "minBatchDuration": 9134,
+ "maxBatchDuration": 28193,
+ "p50Duration": 26112,
+ "p95Duration": 28193,
+ "p99Duration": 28193
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 26122,
+ "avgItemsPerSecond": 11.484572391087971,
+ "avgBatchDuration": 24917.333333333332,
+ "minBatchDuration": 22512,
+ "maxBatchDuration": 26122,
+ "p50Duration": 26118,
+ "p95Duration": 26122,
+ "p99Duration": 26122
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 27157,
+ "avgItemsPerSecond": 11.046875575358102,
+ "avgBatchDuration": 20760.333333333332,
+ "minBatchDuration": 17529,
+ "maxBatchDuration": 27154,
+ "p50Duration": 17598,
+ "p95Duration": 27154,
+ "p99Duration": 27154
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 29234,
+ "avgItemsPerSecond": 10.262023671067935,
+ "avgBatchDuration": 27852.666666666668,
+ "minBatchDuration": 27155,
+ "maxBatchDuration": 29232,
+ "p50Duration": 27171,
+ "p95Duration": 29232,
+ "p99Duration": 29232
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 29230,
+ "avgItemsPerSecond": 10.263427984946972,
+ "avgBatchDuration": 28536,
+ "minBatchDuration": 27154,
+ "maxBatchDuration": 29230,
+ "p50Duration": 29224,
+ "p95Duration": 29230,
+ "p99Duration": 29230
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 28199,
+ "avgItemsPerSecond": 10.63867513032377,
+ "avgBatchDuration": 28189.666666666668,
+ "minBatchDuration": 28179,
+ "maxBatchDuration": 28195,
+ "p50Duration": 28195,
+ "p95Duration": 28195,
+ "p99Duration": 28195
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 29231,
+ "avgItemsPerSecond": 10.26307687044576,
+ "avgBatchDuration": 27848.333333333332,
+ "minBatchDuration": 27149,
+ "maxBatchDuration": 29231,
+ "p50Duration": 27165,
+ "p95Duration": 29231,
+ "p99Duration": 29231
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 28194,
+ "avgItemsPerSecond": 10.640561821664184,
+ "avgBatchDuration": 27132.666666666668,
+ "minBatchDuration": 26095,
+ "maxBatchDuration": 28190,
+ "p50Duration": 27113,
+ "p95Duration": 28190,
+ "p99Duration": 28190
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 29234,
+ "avgItemsPerSecond": 10.262023671067935,
+ "avgBatchDuration": 28871,
+ "minBatchDuration": 28201,
+ "maxBatchDuration": 29232,
+ "p50Duration": 29180,
+ "p95Duration": 29232,
+ "p99Duration": 29232
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:32:57.803Z",
+ "end": "2025-12-15T15:33:27.113Z"
+ }
+}
+```
+
+**Observations:**
+
+- Throughput: **92.12 items/sec** (+69% vs baseline, best in F series)
+- Fairness index: **0.9986** (very good)
+- Resource contention?: **Minimal.** Duration 29.3s, only marginally better than concurrency=10 (30.4s). **Diminishing returns above concurrency=10.** Higher concurrency helps but gains are flattening.
+
+
+
+---
+
+## Series G: Global Rate Limiter
+
+**Objective:** Test global throughput caps across all consumers.
+
+**Theory:**
+
+- Rate limiting provides predictable resource consumption
+- Should not affect fairness, only overall throughput
+
+### Test G1: Moderate Rate Limit (50 items/sec)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+BATCH_QUEUE_GLOBAL_RATE_LIMIT=50
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Rate limit 50/sec"
+```
+
+
+Results G1 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "Rate limit 50/sec",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 45.432364670447086,
+ "overallDuration": 59429,
+ "fairnessIndex": 0.9985621839167762,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 54093,
+ "avgItemsPerSecond": 5.546004104043037,
+ "avgBatchDuration": 54091,
+ "minBatchDuration": 54089,
+ "maxBatchDuration": 54093,
+ "p50Duration": 54091,
+ "p95Duration": 54093,
+ "p99Duration": 54093
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 54109,
+ "avgItemsPerSecond": 5.544364153837624,
+ "avgBatchDuration": 53601,
+ "minBatchDuration": 52695,
+ "maxBatchDuration": 54109,
+ "p50Duration": 53999,
+ "p95Duration": 54109,
+ "p99Duration": 54109
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 59343,
+ "avgItemsPerSecond": 5.055356149840756,
+ "avgBatchDuration": 57304.333333333336,
+ "minBatchDuration": 55866,
+ "maxBatchDuration": 59029,
+ "p50Duration": 57018,
+ "p95Duration": 59029,
+ "p99Duration": 59029
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 58371,
+ "avgItemsPerSecond": 5.139538469445443,
+ "avgBatchDuration": 57164.333333333336,
+ "minBatchDuration": 56262,
+ "maxBatchDuration": 58093,
+ "p50Duration": 57138,
+ "p95Duration": 58093,
+ "p99Duration": 58093
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 58285,
+ "avgItemsPerSecond": 5.147121901003689,
+ "avgBatchDuration": 55634.333333333336,
+ "minBatchDuration": 50470,
+ "maxBatchDuration": 58284,
+ "p50Duration": 58149,
+ "p95Duration": 58284,
+ "p99Duration": 58284
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 55174,
+ "avgItemsPerSecond": 5.437343676369304,
+ "avgBatchDuration": 41686.333333333336,
+ "minBatchDuration": 28166,
+ "maxBatchDuration": 55157,
+ "p50Duration": 41736,
+ "p95Duration": 55157,
+ "p99Duration": 55157
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 58284,
+ "avgItemsPerSecond": 5.147210212065061,
+ "avgBatchDuration": 57015.666666666664,
+ "minBatchDuration": 54788,
+ "maxBatchDuration": 58246,
+ "p50Duration": 58013,
+ "p95Duration": 58246,
+ "p99Duration": 58246
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 59296,
+ "avgItemsPerSecond": 5.059363194819212,
+ "avgBatchDuration": 58258.666666666664,
+ "minBatchDuration": 57226,
+ "maxBatchDuration": 59291,
+ "p50Duration": 58259,
+ "p95Duration": 59291,
+ "p99Duration": 59291
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 59408,
+ "avgItemsPerSecond": 5.049824939402101,
+ "avgBatchDuration": 58836.333333333336,
+ "minBatchDuration": 58371,
+ "maxBatchDuration": 59107,
+ "p50Duration": 59031,
+ "p95Duration": 59107,
+ "p99Duration": 59107
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:35:48.593Z",
+ "end": "2025-12-15T15:36:48.022Z"
+ }
+}
+```
+
+**Observations:**
+
+- Actual throughput: **45.43 items/sec** (target: 50) - ~91% of limit
+- Fairness preserved?: **Yes, 0.9986** (excellent)
+- **Rate limiter is working correctly.** Duration 59.4s shows throttling is effective. Some overhead explains why we don't hit exactly 50/sec.
+
+
+
+### Test G2: Higher Rate Limit (100 items/sec)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+BATCH_QUEUE_GLOBAL_RATE_LIMIT=100
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Rate limit 100/sec"
+```
+
+
+Results G2 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "Rate limit 100/sec",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 81.93487694595333,
+ "overallDuration": 32953,
+ "fairnessIndex": 0.9994851661042276,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 31984,
+ "avgItemsPerSecond": 9.379689844922462,
+ "avgBatchDuration": 30240.333333333332,
+ "minBatchDuration": 28799,
+ "maxBatchDuration": 31980,
+ "p50Duration": 29942,
+ "p95Duration": 31980,
+ "p99Duration": 31980
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 31947,
+ "avgItemsPerSecond": 9.3905531035778,
+ "avgBatchDuration": 26446,
+ "minBatchDuration": 23696,
+ "maxBatchDuration": 31945,
+ "p50Duration": 23697,
+ "p95Duration": 31945,
+ "p99Duration": 31945
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 32914,
+ "avgItemsPerSecond": 9.114662453667133,
+ "avgBatchDuration": 31870.666666666668,
+ "minBatchDuration": 29821,
+ "maxBatchDuration": 32912,
+ "p50Duration": 32879,
+ "p95Duration": 32912,
+ "p99Duration": 32912
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 30770,
+ "avgItemsPerSecond": 9.749756256093598,
+ "avgBatchDuration": 26862,
+ "minBatchDuration": 22381,
+ "maxBatchDuration": 30768,
+ "p50Duration": 27437,
+ "p95Duration": 30768,
+ "p99Duration": 30768
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 30770,
+ "avgItemsPerSecond": 9.749756256093598,
+ "avgBatchDuration": 29516.666666666668,
+ "minBatchDuration": 27853,
+ "maxBatchDuration": 30769,
+ "p50Duration": 29928,
+ "p95Duration": 30769,
+ "p99Duration": 30769
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 30771,
+ "avgItemsPerSecond": 9.749439407234084,
+ "avgBatchDuration": 28237.333333333332,
+ "minBatchDuration": 25781,
+ "maxBatchDuration": 30769,
+ "p50Duration": 28162,
+ "p95Duration": 30769,
+ "p99Duration": 30769
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 31890,
+ "avgItemsPerSecond": 9.40733772342427,
+ "avgBatchDuration": 23515.666666666668,
+ "minBatchDuration": 18184,
+ "maxBatchDuration": 31888,
+ "p50Duration": 20475,
+ "p95Duration": 31888,
+ "p99Duration": 31888
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 30956,
+ "avgItemsPerSecond": 9.691174570357928,
+ "avgBatchDuration": 28886,
+ "minBatchDuration": 25781,
+ "maxBatchDuration": 30947,
+ "p50Duration": 29930,
+ "p95Duration": 30947,
+ "p99Duration": 30947
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 31976,
+ "avgItemsPerSecond": 9.382036527395547,
+ "avgBatchDuration": 29608.666666666668,
+ "minBatchDuration": 26943,
+ "maxBatchDuration": 31974,
+ "p50Duration": 29909,
+ "p95Duration": 31974,
+ "p99Duration": 31974
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:37:49.150Z",
+ "end": "2025-12-15T15:38:22.103Z"
+ }
+}
+```
+
+**Observations:**
+
+- Actual throughput: **81.93 items/sec** (limit was 100)
+- Is this limit binding or above natural throughput?: **Above natural throughput.** Without rate limiting (E2 with same settings), throughput was 93.57/sec. The 100/sec limit is barely constraining. **Use rate limits below ~90/sec to have meaningful impact.**
+
+
+
+### Test G3: Low Rate Limit (20 items/sec)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+BATCH_QUEUE_GLOBAL_RATE_LIMIT=20
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Rate limit 20/sec"
+```
+
+
+Results G3 (click to expand)
+
+```json
+{
+ "scenario": "fairness-9t",
+ "description": "Rate limit 20/sec",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 100,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 2700,
+ "totalBatches": 27,
+ "overallThroughput": 21.692671090900326,
+ "overallDuration": 124466,
+ "fairnessIndex": 0.997793763355806,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 108861,
+ "avgItemsPerSecond": 2.755807865075647,
+ "avgBatchDuration": 99844,
+ "minBatchDuration": 93281,
+ "maxBatchDuration": 108850,
+ "p50Duration": 97401,
+ "p95Duration": 108850,
+ "p99Duration": 108850
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 111633,
+ "avgItemsPerSecond": 2.687377388406654,
+ "avgBatchDuration": 85438,
+ "minBatchDuration": 71861,
+ "maxBatchDuration": 111516,
+ "p50Duration": 72937,
+ "p95Duration": 111516,
+ "p99Duration": 111516
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 123298,
+ "avgItemsPerSecond": 2.433129491151519,
+ "avgBatchDuration": 96355,
+ "minBatchDuration": 43494,
+ "maxBatchDuration": 123297,
+ "p50Duration": 122274,
+ "p95Duration": 123297,
+ "p99Duration": 123297
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 121568,
+ "avgItemsPerSecond": 2.4677546722821795,
+ "avgBatchDuration": 92002,
+ "minBatchDuration": 53253,
+ "maxBatchDuration": 121568,
+ "p50Duration": 101185,
+ "p95Duration": 121568,
+ "p99Duration": 121568
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 123851,
+ "avgItemsPerSecond": 2.422265464146434,
+ "avgBatchDuration": 118623.66666666667,
+ "minBatchDuration": 111599,
+ "maxBatchDuration": 123784,
+ "p50Duration": 120488,
+ "p95Duration": 123784,
+ "p99Duration": 123784
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 120270,
+ "avgItemsPerSecond": 2.4943876278373662,
+ "avgBatchDuration": 113171.33333333333,
+ "minBatchDuration": 109360,
+ "maxBatchDuration": 120269,
+ "p50Duration": 109885,
+ "p95Duration": 120269,
+ "p99Duration": 120269
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 123169,
+ "avgItemsPerSecond": 2.4356778085394866,
+ "avgBatchDuration": 118514,
+ "minBatchDuration": 113405,
+ "maxBatchDuration": 123164,
+ "p50Duration": 118973,
+ "p95Duration": 123164,
+ "p99Duration": 123164
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 124246,
+ "avgItemsPerSecond": 2.414564653992885,
+ "avgBatchDuration": 119631.33333333333,
+ "minBatchDuration": 114504,
+ "maxBatchDuration": 124241,
+ "p50Duration": 120149,
+ "p95Duration": 124241,
+ "p99Duration": 124241
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 300,
+ "totalBatches": 3,
+ "totalDuration": 122197,
+ "avgItemsPerSecond": 2.4550520880218007,
+ "avgBatchDuration": 120645.33333333333,
+ "minBatchDuration": 118022,
+ "maxBatchDuration": 122197,
+ "p50Duration": 121717,
+ "p95Duration": 122197,
+ "p99Duration": 122197
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:39:42.414Z",
+ "end": "2025-12-15T15:41:46.880Z"
+ }
+}
+```
+
+**Observations:**
+
+- Actual throughput: **21.69 items/sec** (target: 20) - slightly above limit
+- Fairness index: **0.9978** (good but slight variance)
+- Duration 124.5s (2+ minutes) - shows throttling working hard. **Strict rate limiting works but creates fairness variance** - some tenants saw 72-124s batch durations (wide spread).
+
+
+
+---
+
+## Series H: Asymmetric Load Testing
+
+**Objective:** Test fairness when tenants have dramatically different workloads.
+
+### Test H1: Asymmetric with Default Config
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress asymmetric --small-size 50 --large-size 800 --batch-count 3 -d "Asymmetric baseline"
+```
+
+
+Results H1 (click to expand)
+
+```json
+{
+ "scenario": "asymmetric-combined",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 0,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 10350,
+ "totalBatches": 27,
+ "overallThroughput": 67.60816590394816,
+ "overallDuration": 107281,
+ "fairnessIndex": 0.9978993490004432,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 150,
+ "totalBatches": 3,
+ "totalDuration": 16361,
+ "avgItemsPerSecond": 9.168143756494102,
+ "avgBatchDuration": 14312.666666666666,
+ "minBatchDuration": 12271,
+ "maxBatchDuration": 16360,
+ "p50Duration": 14307,
+ "p95Duration": 16360,
+ "p99Duration": 16360
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 150,
+ "totalBatches": 3,
+ "totalDuration": 16361,
+ "avgItemsPerSecond": 9.168143756494102,
+ "avgBatchDuration": 13968.333333333334,
+ "minBatchDuration": 11240,
+ "maxBatchDuration": 16358,
+ "p50Duration": 14307,
+ "p95Duration": 16358,
+ "p99Duration": 16358
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 150,
+ "totalBatches": 3,
+ "totalDuration": 16361,
+ "avgItemsPerSecond": 9.168143756494102,
+ "avgBatchDuration": 12117.333333333334,
+ "minBatchDuration": 8977,
+ "maxBatchDuration": 16359,
+ "p50Duration": 11016,
+ "p95Duration": 16359,
+ "p99Duration": 16359
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 150,
+ "totalBatches": 3,
+ "totalDuration": 13966,
+ "avgItemsPerSecond": 10.740369468709723,
+ "avgBatchDuration": 13624.333333333334,
+ "minBatchDuration": 12945,
+ "maxBatchDuration": 13964,
+ "p50Duration": 13964,
+ "p95Duration": 13964,
+ "p99Duration": 13964
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 150,
+ "totalBatches": 3,
+ "totalDuration": 16116,
+ "avgItemsPerSecond": 9.307520476545047,
+ "avgBatchDuration": 14416.666666666666,
+ "minBatchDuration": 11021,
+ "maxBatchDuration": 16116,
+ "p50Duration": 16113,
+ "p95Duration": 16116,
+ "p99Duration": 16116
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 2400,
+ "totalBatches": 3,
+ "totalDuration": 107273,
+ "avgItemsPerSecond": 22.37282447586998,
+ "avgBatchDuration": 106213,
+ "minBatchDuration": 105126,
+ "maxBatchDuration": 107273,
+ "p50Duration": 106240,
+ "p95Duration": 107273,
+ "p99Duration": 107273
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 2400,
+ "totalBatches": 3,
+ "totalDuration": 106584,
+ "avgItemsPerSecond": 22.51745102454402,
+ "avgBatchDuration": 105934,
+ "minBatchDuration": 105097,
+ "maxBatchDuration": 106574,
+ "p50Duration": 106131,
+ "p95Duration": 106574,
+ "p99Duration": 106574
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 2400,
+ "totalBatches": 3,
+ "totalDuration": 107116,
+ "avgItemsPerSecond": 22.40561634116285,
+ "avgBatchDuration": 106374.33333333333,
+ "minBatchDuration": 105308,
+ "maxBatchDuration": 107100,
+ "p50Duration": 106715,
+ "p95Duration": 107100,
+ "p99Duration": 107100
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 2400,
+ "totalBatches": 3,
+ "totalDuration": 107046,
+ "avgItemsPerSecond": 22.42026792220167,
+ "avgBatchDuration": 104503.66666666667,
+ "minBatchDuration": 102401,
+ "maxBatchDuration": 106989,
+ "p50Duration": 104121,
+ "p95Duration": 106989,
+ "p99Duration": 106989
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:44:50.480Z",
+ "end": "2025-12-15T15:46:37.770Z"
+ }
+}
+```
+
+**Observations:**
+
+- Small batch completion times: **14-17s** (150 items × 5 tenants)
+- Large batch completion times: **105-107s** (2400 items × 4 tenants)
+- Were small batches starved?: **NO!** Small batches finished ~6x faster than large batches, exactly as expected. **DRR correctly prioritizes smaller workloads and doesn't let large batches monopolize.** Fairness index 0.9979 is excellent given asymmetric load.
+
+
+
+### Test H2: Asymmetric with Low Quantum (better fairness?)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=2
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress asymmetric --small-size 50 --large-size 800 --batch-count 3 -d "Asymmetric low quantum"
+```
+
+
+Results H2 (click to expand)
+
+```json
+{
+ "scenario": "asymmetric-combined",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 0,
+ "batchCount": 3
+ },
+ "results": {
+ "totalItemsProcessed": 10350,
+ "totalBatches": 27,
+ "overallThroughput": 58.418499446364066,
+ "overallDuration": 127189,
+ "fairnessIndex": 0.9991864215649569,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 150,
+ "totalBatches": 3,
+ "totalDuration": 16648,
+ "avgItemsPerSecond": 9.01009130225853,
+ "avgBatchDuration": 15360,
+ "minBatchDuration": 14080,
+ "maxBatchDuration": 16647,
+ "p50Duration": 15353,
+ "p95Duration": 16647,
+ "p99Duration": 16647
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 150,
+ "totalBatches": 3,
+ "totalDuration": 18118,
+ "avgItemsPerSecond": 8.279059498840931,
+ "avgBatchDuration": 16403.333333333332,
+ "minBatchDuration": 15020,
+ "maxBatchDuration": 18115,
+ "p50Duration": 16075,
+ "p95Duration": 18115,
+ "p99Duration": 18115
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 150,
+ "totalBatches": 3,
+ "totalDuration": 16307,
+ "avgItemsPerSecond": 9.198503710063163,
+ "avgBatchDuration": 15531.333333333334,
+ "minBatchDuration": 14206,
+ "maxBatchDuration": 16305,
+ "p50Duration": 16083,
+ "p95Duration": 16305,
+ "p99Duration": 16305
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 150,
+ "totalBatches": 3,
+ "totalDuration": 16646,
+ "avgItemsPerSecond": 9.011173855580921,
+ "avgBatchDuration": 14090,
+ "minBatchDuration": 11256,
+ "maxBatchDuration": 16645,
+ "p50Duration": 14369,
+ "p95Duration": 16645,
+ "p99Duration": 16645
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 150,
+ "totalBatches": 3,
+ "totalDuration": 17696,
+ "avgItemsPerSecond": 8.476491862567812,
+ "avgBatchDuration": 16322,
+ "minBatchDuration": 14600,
+ "maxBatchDuration": 17694,
+ "p50Duration": 16672,
+ "p95Duration": 17694,
+ "p99Duration": 17694
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 2400,
+ "totalBatches": 3,
+ "totalDuration": 125841,
+ "avgItemsPerSecond": 19.07168569861969,
+ "avgBatchDuration": 125238.66666666667,
+ "minBatchDuration": 124354,
+ "maxBatchDuration": 125800,
+ "p50Duration": 125562,
+ "p95Duration": 125800,
+ "p99Duration": 125800
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 2400,
+ "totalBatches": 3,
+ "totalDuration": 127140,
+ "avgItemsPerSecond": 18.876828692779615,
+ "avgBatchDuration": 126407.66666666667,
+ "minBatchDuration": 125398,
+ "maxBatchDuration": 127140,
+ "p50Duration": 126685,
+ "p95Duration": 127140,
+ "p99Duration": 127140
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 2400,
+ "totalBatches": 3,
+ "totalDuration": 125854,
+ "avgItemsPerSecond": 19.069715702321737,
+ "avgBatchDuration": 125322.66666666667,
+ "minBatchDuration": 124426,
+ "maxBatchDuration": 125849,
+ "p50Duration": 125693,
+ "p95Duration": 125849,
+ "p99Duration": 125849
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 2400,
+ "totalBatches": 3,
+ "totalDuration": 126467,
+ "avgItemsPerSecond": 18.977282611274088,
+ "avgBatchDuration": 124552.33333333333,
+ "minBatchDuration": 123402,
+ "maxBatchDuration": 126202,
+ "p50Duration": 124053,
+ "p95Duration": 126202,
+ "p99Duration": 126202
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:50:21.351Z",
+ "end": "2025-12-15T15:52:28.745Z"
+ }
+}
+```
+
+**Observations:**
+
+- Small batch completion times: **14-18s** (slightly slower than H1's 14-17s)
+- Large batch completion times: **123-127s** (slower than H1's 105-107s)
+- Improvement over H1?: **Mixed.** Fairness improved (0.9992 vs 0.9979) but overall throughput dropped (58.42 vs 67.61 items/sec). **Low quantum=2 improves fairness but at throughput cost.** For asymmetric loads, default quantum=5 is better balance.
+
+
+
+### Test H3: Burst Test (All Tenants Simultaneous)
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress burst --batch-size 500 -d "Burst test baseline"
+```
+
+
+Results H3 (click to expand)
+
+```json
+{
+ "scenario": "burst-9t-500i",
+ "config": {
+ "tenantCount": 9,
+ "batchSize": 500,
+ "batchCount": 1
+ },
+ "results": {
+ "totalItemsProcessed": 4500,
+ "totalBatches": 9,
+ "overallThroughput": 68.83786388459714,
+ "overallDuration": 65371,
+ "fairnessIndex": 0.9999479889573228,
+ "perTenant": [
+ {
+ "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh",
+ "totalItems": 500,
+ "totalBatches": 1,
+ "totalDuration": 64508,
+ "avgItemsPerSecond": 7.750976623054505,
+ "avgBatchDuration": 64508,
+ "minBatchDuration": 64508,
+ "maxBatchDuration": 64508,
+ "p50Duration": 64508,
+ "p95Duration": 64508,
+ "p99Duration": 64508
+ },
+ {
+ "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf",
+ "totalItems": 500,
+ "totalBatches": 1,
+ "totalDuration": 64504,
+ "avgItemsPerSecond": 7.751457273967506,
+ "avgBatchDuration": 64504,
+ "minBatchDuration": 64504,
+ "maxBatchDuration": 64504,
+ "p50Duration": 64504,
+ "p95Duration": 64504,
+ "p99Duration": 64504
+ },
+ {
+ "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj",
+ "totalItems": 500,
+ "totalBatches": 1,
+ "totalDuration": 64404,
+ "avgItemsPerSecond": 7.7634929507484,
+ "avgBatchDuration": 64404,
+ "minBatchDuration": 64404,
+ "maxBatchDuration": 64404,
+ "p50Duration": 64404,
+ "p95Duration": 64404,
+ "p99Duration": 64404
+ },
+ {
+ "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb",
+ "totalItems": 500,
+ "totalBatches": 1,
+ "totalDuration": 64504,
+ "avgItemsPerSecond": 7.751457273967506,
+ "avgBatchDuration": 64504,
+ "minBatchDuration": 64504,
+ "maxBatchDuration": 64504,
+ "p50Duration": 64504,
+ "p95Duration": 64504,
+ "p99Duration": 64504
+ },
+ {
+ "tenantId": "org-2:proj_zgysghtkiezoakvjscin",
+ "totalItems": 500,
+ "totalBatches": 1,
+ "totalDuration": 64507,
+ "avgItemsPerSecond": 7.7510967801943975,
+ "avgBatchDuration": 64507,
+ "minBatchDuration": 64507,
+ "maxBatchDuration": 64507,
+ "p50Duration": 64507,
+ "p95Duration": 64507,
+ "p99Duration": 64507
+ },
+ {
+ "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo",
+ "totalItems": 500,
+ "totalBatches": 1,
+ "totalDuration": 64391,
+ "avgItemsPerSecond": 7.7650603345187985,
+ "avgBatchDuration": 64391,
+ "minBatchDuration": 64391,
+ "maxBatchDuration": 64391,
+ "p50Duration": 64391,
+ "p95Duration": 64391,
+ "p99Duration": 64391
+ },
+ {
+ "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs",
+ "totalItems": 500,
+ "totalBatches": 1,
+ "totalDuration": 63375,
+ "avgItemsPerSecond": 7.889546351084813,
+ "avgBatchDuration": 63375,
+ "minBatchDuration": 63375,
+ "maxBatchDuration": 63375,
+ "p50Duration": 63375,
+ "p95Duration": 63375,
+ "p99Duration": 63375
+ },
+ {
+ "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi",
+ "totalItems": 500,
+ "totalBatches": 1,
+ "totalDuration": 64497,
+ "avgItemsPerSecond": 7.752298556522009,
+ "avgBatchDuration": 64497,
+ "minBatchDuration": 64497,
+ "maxBatchDuration": 64497,
+ "p50Duration": 64497,
+ "p95Duration": 64497,
+ "p99Duration": 64497
+ },
+ {
+ "tenantId": "org-3:proj_ytivyoceocenyxuprmga",
+ "totalItems": 500,
+ "totalBatches": 1,
+ "totalDuration": 65316,
+ "avgItemsPerSecond": 7.655092167309695,
+ "avgBatchDuration": 65316,
+ "minBatchDuration": 65316,
+ "maxBatchDuration": 65316,
+ "p50Duration": 65316,
+ "p95Duration": 65316,
+ "p99Duration": 65316
+ }
+ ]
+ },
+ "timestamps": {
+ "start": "2025-12-15T15:53:49.834Z",
+ "end": "2025-12-15T15:54:55.205Z"
+ }
+}
+```
+
+**Observations:**
+
+- Fairness under burst: **0.9999** (nearly perfect!)
+- Throughput: **68.84 items/sec**
+- Any tenant significantly slower?: **All tenants finished within 2s of each other (63-65s range).** **Excellent burst handling!** Even under simultaneous 4500-item burst from 9 tenants, DRR maintained perfect fairness. This is the ideal scenario for DRR.
+
+
+
+---
+
+## Series I: Combined/Optimized Configurations
+
+**Objective:** Test promising combinations based on earlier results.
+
+### Test I1: Throughput-Optimized
+
+Based on Series B-F results, configure for maximum throughput.
+
+**Configuration (adjust based on findings):**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=10
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=5
+BATCH_QUEUE_CONSUMER_INTERVAL_MS=50
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=25
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 5 --tenants all -d "Throughput optimized"
+```
+
+
+Results I1 (click to expand)
+
+```json
+// Paste results here
+```
+
+**Observations:**
+
+- Throughput: **\_** items/sec
+- Fairness tradeoff: **\_**
+
+
+
+### Test I2: Fairness-Optimized
+
+Based on Series B-F results, configure for best fairness.
+
+**Configuration (adjust based on findings):**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=2
+BATCH_QUEUE_MAX_DEFICIT=25
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_QUEUE_CONSUMER_INTERVAL_MS=50
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=5
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 5 --tenants all -d "Fairness optimized"
+```
+
+
+Results I2 (click to expand)
+
+```json
+// Paste results here
+```
+
+**Observations:**
+
+- Fairness index: **\_**
+- Throughput cost: **\_**
+
+
+
+### Test I3: Balanced Configuration
+
+Best balance of throughput and fairness.
+
+**Configuration (adjust based on findings):**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_QUEUE_CONSUMER_INTERVAL_MS=50
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 5 --tenants all -d "Balanced configuration"
+```
+
+
+Results I3 (click to expand)
+
+```json
+// Paste results here
+```
+
+**Observations:**
+
+- Throughput: **\_** items/sec
+- Fairness index: **\_**
+- Recommended for production?: **\_**
+
+
+
+---
+
+## Per-Org Concurrency Testing (Optional Series J)
+
+If time permits, test different concurrency limits per organization by updating the database.
+
+### Setting Per-Org Concurrency
+
+```sql
+-- Set org-1 to high concurrency (50)
+UPDATE "Organization"
+SET "batchQueueConcurrencyConfig" = '{"default": 50}'
+WHERE slug = 'org-1';
+
+-- Set org-2 to medium concurrency (10)
+UPDATE "Organization"
+SET "batchQueueConcurrencyConfig" = '{"default": 10}'
+WHERE slug = 'org-2';
+
+-- Set org-3 to low concurrency (2)
+UPDATE "Organization"
+SET "batchQueueConcurrencyConfig" = '{"default": 2}'
+WHERE slug = 'org-3';
+```
+
+### Test J1: Mixed Org Concurrency Limits
+
+**Configuration:**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=50
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 # fallback
+```
+
+**Command:**
+
+```bash
+pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Mixed org concurrency"
+```
+
+
+Results J1 (click to expand)
+
+```json
+// Paste results here
+```
+
+**Observations:**
+
+- Org-1 (high concurrency) throughput: **\_**
+- Org-2 (medium concurrency) throughput: **\_**
+- Org-3 (low concurrency) throughput: **\_**
+- Does DRR still maintain fairness?: **\_**
+
+
+
+---
+
+## Results Summary Table
+
+| Test | Config Summary | Throughput (items/sec) | Fairness Index | Notes |
+| ---- | ---------------------- | ---------------------- | -------------- | ---------------------------------- |
+| A1 | Baseline | 54.52 | 0.9999 | Reference point |
+| A2 | Throughput baseline | 97.48 (peak @ 500) | 1.0000 | Single tenant max |
+| B1 | quantum=1 | 66.09 | 0.9997 | +21% vs baseline |
+| B2 | quantum=10 | 64.69 | 0.9998 | +18% |
+| B3 | quantum=25 | **84.36** | **0.9999** | **Best B series** |
+| B4 | quantum=50 | 51.98 | 0.9997 | Too high, hurts perf |
+| C1 | maxDeficit=10 | 52.51 | 0.9998 | Limited catch-up |
+| C2 | maxDeficit=25 | 66.33 | 0.9992 | +22% |
+| C3 | maxDeficit=100 | **73.74** | 0.9989 | **Best C**, slight fairness cost |
+| D1 | consumers=1 | 41.33 | 0.9997 | Bottleneck |
+| D2 | consumers=3 | 57.55 | 0.9992 | 1.39x vs D1 |
+| D3 | consumers=5 | 63.80 | 0.9999 | Good balance |
+| D4 | consumers=10 | **114.94** | 0.9888 | **Best throughput**, fairness cost |
+| E1 | interval=20ms | 70.23 | 0.9979 | Contention overhead |
+| E2 | interval=50ms | **93.57** | **0.9994** | **Best E series** |
+| E3 | interval=100ms | 86.52 | 0.9997 | Good default |
+| E4 | interval=250ms | 47.46 | 0.9994 | Too slow |
+| F1 | concurrency=2 | 90.73 | 0.9931 | High but unfair |
+| F2 | concurrency=5 | 58.25 | **0.9999** | **Best fairness** |
+| F3 | concurrency=10 | 88.71 | 0.9986 | Good balance |
+| F4 | concurrency=25 | **92.12** | 0.9986 | **Best F series** |
+| F5 | concurrency=50 | _not tested_ | _not tested_ | |
+| G1 | rateLimit=50 | 45.43 | 0.9986 | Effective throttle |
+| G2 | rateLimit=100 | 81.93 | 0.9995 | Not binding |
+| G3 | rateLimit=20 | 21.69 | 0.9978 | Strict throttle |
+| H1 | asymmetric baseline | 67.61 | 0.9979 | Small batches not starved |
+| H2 | asymmetric low quantum | 58.42 | 0.9992 | Better fairness, lower throughput |
+| H3 | burst test | 68.84 | **0.9999** | **Perfect burst fairness** |
+| I1 | throughput optimized | _pending_ | _pending_ | |
+| I2 | fairness optimized | _pending_ | _pending_ | |
+| I3 | balanced | _pending_ | _pending_ | |
+
+---
+
+## Recommended Execution Order
+
+1. **Series A** (Baseline) - Establish reference point
+2. **Series B** (DRR Quantum) - Understand core fairness mechanism
+3. **Series C** (Max Deficit) - Understand starvation prevention
+4. **Series D** (Consumer Count) - Find optimal parallelism
+5. **Series E** (Consumer Interval) - Find optimal polling frequency
+6. **Series F** (Concurrency) - Find optimal per-env limits
+7. **Series G** (Global Rate Limit) - Test throttling mechanism
+8. **Series H** (Asymmetric) - Validate fairness under realistic conditions
+9. **Series I** (Combined) - Test optimized configurations
+
+## Between Tests
+
+After each configuration change:
+
+1. Restart the webapp to pick up new env vars
+2. Wait ~5 seconds for BatchQueue to initialize
+3. Consider clearing Redis state between tests if needed: `redis-cli FLUSHDB`
+
+---
+
+## Key Metrics to Watch
+
+1. **Throughput** (`overallThroughput`) - Higher is better, all else equal
+2. **Fairness Index** (`fairnessIndex`) - Closer to 1.0 is better
+3. **Per-Tenant Variance** - Look at min/max batch durations across tenants
+4. **P95/P99 Latencies** - Watch for tail latencies indicating starvation
+
+## Notes and Observations
+
+### Key Findings from Series A-H
+
+#### 1. DRR Fairness is Excellent
+
+- **Fairness Index consistently 0.99+** across all tests
+- Burst scenario (H3) achieved **0.9999 fairness** with 9 tenants competing simultaneously
+- Asymmetric load (H1) showed small batches complete quickly without being starved by large batches
+
+#### 2. Throughput vs Fairness Tradeoffs
+
+| Configuration | Effect on Throughput | Effect on Fairness |
+| --------------------- | -------------------- | ------------------ |
+| Higher quantum (25) | ↑ Significant | → Neutral |
+| Higher quantum (50) | ↓ Hurts | → Neutral |
+| Higher maxDeficit | ↑ Moderate | ↓ Slight |
+| More consumers | ↑↑ Major | ↓ Moderate (at 10) |
+| Faster polling (50ms) | ↑ Best | → Neutral |
+| Higher concurrency | ↑ Moderate | → Neutral |
+
+#### 3. Recommended Default Configuration
+
+Based on test results, the optimal balanced configuration is:
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=25 # Sweet spot from B series
+BATCH_QUEUE_MAX_DEFICIT=50 # Default is fine (100 gains throughput but costs fairness)
+BATCH_QUEUE_CONSUMER_COUNT=3 # Good balance (10 is faster but less fair)
+BATCH_QUEUE_CONSUMER_INTERVAL_MS=50 # Best from E series
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 # Good balance from F series
+```
+
+**Expected performance:** ~90-100 items/sec with 0.999+ fairness
+
+#### 4. Configuration Recommendations by Use Case
+
+**High-throughput priority (fairness acceptable at 0.98+):**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=25
+BATCH_QUEUE_MAX_DEFICIT=100
+BATCH_QUEUE_CONSUMER_COUNT=10
+BATCH_QUEUE_CONSUMER_INTERVAL_MS=50
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=25
+```
+
+**Strict fairness priority (throughput can be lower):**
+
+```env
+BATCH_QUEUE_DRR_QUANTUM=5
+BATCH_QUEUE_MAX_DEFICIT=25
+BATCH_QUEUE_CONSUMER_COUNT=3
+BATCH_QUEUE_CONSUMER_INTERVAL_MS=100
+BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=5
+```
+
+#### 5. Surprising Results
+
+1. **quantum=1 outperformed quantum=10**: Lower quantum didn't add expected overhead. May be due to better slot utilization.
+
+2. **quantum=50 was worst**: Too much time between tenant switches caused overall slowdown.
+
+3. **concurrency=2 had high throughput but poor fairness**: Some tenants finished much faster than others.
+
+4. **20ms polling was worse than 50ms**: Too-fast polling created contention.
+
+#### 6. Rate Limiter Behavior
+
+- Rate limiter effectively caps throughput
+- At 50/sec limit, achieved ~45/sec (91% efficiency)
+- At 20/sec limit, achieved ~22/sec (108% - slight overshoot)
+- Rate limiting preserved fairness well (0.997-0.999)
+
+#### 7. Asymmetric Load Handling
+
+- **Excellent!** Small batches (150 items) completed in 14-17s
+- Large batches (2400 items) completed in 105-127s
+- DRR prevented large batches from starving small ones
+- Lower quantum (2) improved fairness but hurt overall throughput by ~13%
+
+### Scaling Observations
+
+- Single consumer: ~41 items/sec
+- 3 consumers: ~58 items/sec (1.4x)
+- 5 consumers: ~64 items/sec (1.5x)
+- 10 consumers: ~115 items/sec (2.8x)
+
+**Diminishing returns start around 5 consumers, with fairness degradation at 10.**
+
+### Production Recommendations
+
+1. **Start with defaults** - They're well-tuned for most cases
+2. **Consider quantum=25** for throughput boost without fairness cost
+3. **Use 3-5 consumers** for good parallelism without contention
+4. **50ms polling** is optimal for most workloads
+5. **Monitor fairness index** in production metrics
+6. **Use global rate limiter** if you need to cap resource usage
+
+---
+
+**Document created:** $(date)
+**Last updated:**
diff --git a/docs/docs.json b/docs/docs.json
index 13b8b7706d..30d99fd058 100644
--- a/docs/docs.json
+++ b/docs/docs.json
@@ -270,6 +270,13 @@
"management/tasks/batch-trigger"
]
},
+ {
+ "group": "Batches API",
+ "pages": [
+ "management/batches/create",
+ "management/batches/stream-items"
+ ]
+ },
{
"group": "Runs API",
"pages": [
@@ -698,4 +705,4 @@
"destination": "/migrating-from-v3"
}
]
-}
+}
\ No newline at end of file
diff --git a/docs/limits.mdx b/docs/limits.mdx
index db2a18ae6d..c1149c8343 100644
--- a/docs/limits.mdx
+++ b/docs/limits.mdx
@@ -75,20 +75,44 @@ Additional bundles are available for $10/month per 100 concurrent connections. C
## Task payloads and outputs
-| Limit | Details |
-| :--------------------- | :-------------------------------------------- |
-| Single trigger payload | Must not exceed 3MB |
-| Batch trigger payload | The total of all payloads must not exceed 5MB |
-| Task outputs | Must not exceed 10MB |
+| Limit | Details |
+| :--------------------- | :----------------------------------------------------------------- |
+| Single trigger payload | Must not exceed 3MB |
+| Batch trigger payload | Each item can be up to 3MB (SDK 4.3.1+). Prior: 1MB total combined |
+| Task outputs | Must not exceed 10MB |
Payloads and outputs that exceed 512KB will be offloaded to object storage and a presigned URL will be provided to download the data when calling `runs.retrieve`. You don't need to do anything to handle this in your tasks however, as we will transparently upload/download these during operation.
## Batch size
-A single batch can have a maximum of 500 items.
+A single batch can have a maximum of 1,000 items with SDK 4.3.1+. Prior versions are limited to 500 items.
+## Batch trigger rate limits
+
+Batch triggering uses a token bucket algorithm to rate limit the number of runs you can trigger per environment. Each run in a batch consumes one token.
+
+| Pricing tier | Bucket size | Refill rate |
+| :----------- | :---------- | :-------------------- |
+| Free | 1,200 runs | 100 runs every 10 sec |
+| Hobby | 5,000 runs | 500 runs every 5 sec |
+| Pro | 5,000 runs | 500 runs every 5 sec |
+
+**How it works**: You can burst up to your bucket size, then tokens refill at the specified rate. For example, a Free user can trigger 1,200 runs immediately, then must wait for tokens to refill (100 runs become available every 10 seconds).
+
+## Batch processing concurrency
+
+The number of batches that can be processed concurrently per environment.
+
+| Pricing tier | Limit |
+| :----------- | :-------------------- |
+| Free | 1 concurrent batch |
+| Hobby | 10 concurrent batches |
+| Pro | 10 concurrent batches |
+
+This limits how many batches can have their items actively being processed into runs at the same time.
+
## Log retention
| Pricing tier | Limit |
diff --git a/docs/management/batches/create.mdx b/docs/management/batches/create.mdx
new file mode 100644
index 0000000000..f9a11cb2e4
--- /dev/null
+++ b/docs/management/batches/create.mdx
@@ -0,0 +1,5 @@
+---
+title: "Create batch"
+openapi: "openapi POST /api/v3/batches"
+---
+
diff --git a/docs/management/batches/stream-items.mdx b/docs/management/batches/stream-items.mdx
new file mode 100644
index 0000000000..66f074bc42
--- /dev/null
+++ b/docs/management/batches/stream-items.mdx
@@ -0,0 +1,5 @@
+---
+title: "Stream batch items"
+openapi: "openapi POST /api/v3/batches/{batchId}/items"
+---
+
diff --git a/docs/openapi.yml b/docs/openapi.yml
index d5dd12ff46..922e8d7f28 100644
--- a/docs/openapi.yml
+++ b/docs/openapi.yml
@@ -43,6 +43,159 @@ paths:
schema:
$ref: "#/components/schemas/Error"
+ /api/v3/batches:
+ post:
+ operationId: createBatch
+ externalDocs:
+ description: Find more info here
+ url: "https://trigger.dev/docs/triggering"
+ tags:
+ - Batches
+ summary: Create a batch (Phase 1)
+ description: |
+ Phase 1 of 2-phase batch API. Creates a batch record and optionally blocks the parent run for batchTriggerAndWait.
+ After creating a batch, stream items via POST /api/v3/batches/{batchId}/items.
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/CreateBatchRequest"
+ responses:
+ "202":
+ description: Batch successfully created
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/CreateBatchResponse"
+ headers:
+ x-trigger-jwt-claims:
+ description: JWT claims for the batch
+ schema:
+ type: string
+ x-trigger-jwt:
+ description: JWT token for browser clients
+ schema:
+ type: string
+ "400":
+ description: Invalid request (e.g., runCount <= 0 or exceeds maximum)
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Error"
+ "401":
+ description: Unauthorized - API key is missing or invalid
+ "422":
+ description: Validation error
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Error"
+ "429":
+ description: Rate limit exceeded
+ headers:
+ X-RateLimit-Limit:
+ description: Maximum number of requests allowed
+ schema:
+ type: integer
+ X-RateLimit-Remaining:
+ description: Number of requests remaining
+ schema:
+ type: integer
+ X-RateLimit-Reset:
+ description: Unix timestamp when the rate limit resets
+ schema:
+ type: integer
+ Retry-After:
+ description: Seconds to wait before retrying
+ schema:
+ type: integer
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Error"
+ "500":
+ description: Internal server error
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Error"
+
+ /api/v3/batches/{batchId}/items:
+ post:
+ operationId: streamBatchItems
+ externalDocs:
+ description: Find more info here
+ url: "https://trigger.dev/docs/triggering"
+ tags:
+ - Batches
+ summary: Stream batch items (Phase 2)
+ description: |
+ Phase 2 of 2-phase batch API. Accepts an NDJSON stream of batch items and enqueues them.
+ Each line in the body should be a valid BatchItemNDJSON object.
+ The stream is processed with backpressure - items are enqueued as they arrive.
+ The batch is sealed when the stream completes successfully.
+ parameters:
+ - name: batchId
+ in: path
+ required: true
+ description: The batch ID returned from POST /api/v3/batches
+ schema:
+ type: string
+ requestBody:
+ required: true
+ content:
+ application/x-ndjson:
+ schema:
+ type: string
+ description: |
+ NDJSON (newline-delimited JSON) stream where each line is a BatchItemNDJSON object.
+ Example:
+ {"index":0,"task":"my-task","payload":{"key":"value1"}}
+ {"index":1,"task":"my-task","payload":{"key":"value2"}}
+ application/ndjson:
+ schema:
+ type: string
+ description: |
+ NDJSON (newline-delimited JSON) stream where each line is a BatchItemNDJSON object.
+ responses:
+ "200":
+ description: Items successfully processed
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/StreamBatchItemsResponse"
+ "400":
+ description: Invalid request (e.g., invalid JSON, item exceeds maximum size)
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Error"
+ "401":
+ description: Unauthorized - API key is missing or invalid
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Error"
+ "415":
+ description: Unsupported Media Type - Content-Type must be application/x-ndjson or application/ndjson
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Error"
+ "422":
+ description: Validation error
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Error"
+ "500":
+ description: Internal server error
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Error"
+
components:
schemas:
Error:
@@ -130,6 +283,91 @@ components:
type: object
additionalProperties: true
description: A JSON object that represents the deserialized payload or context.
+ CreateBatchRequest:
+ type: object
+ required:
+ - runCount
+ properties:
+ runCount:
+ type: integer
+ minimum: 1
+ description: Expected number of items in the batch. Must be a positive integer.
+ parentRunId:
+ type: string
+ description: Parent run ID (friendly ID) for batchTriggerAndWait.
+ resumeParentOnCompletion:
+ type: boolean
+ description: Whether to resume parent on completion. Set to true for batchTriggerAndWait.
+ idempotencyKey:
+ type: string
+ description: Idempotency key for the batch. If provided and a batch with this key already exists, the existing batch will be returned.
+ CreateBatchResponse:
+ type: object
+ required:
+ - id
+ - runCount
+ - isCached
+ properties:
+ id:
+ type: string
+ description: The batch ID (friendly ID). Use this to stream items via POST /api/v3/batches/{batchId}/items.
+ runCount:
+ type: integer
+ description: The expected run count.
+ isCached:
+ type: boolean
+ description: Whether this response came from a cached/idempotent batch.
+ idempotencyKey:
+ type: string
+ description: The idempotency key if provided.
+ BatchItemNDJSON:
+ type: object
+ required:
+ - index
+ - task
+ properties:
+ index:
+ type: integer
+ minimum: 0
+ description: Zero-based index of this item. Used for idempotency and ordering.
+ task:
+ type: string
+ description: The task identifier to trigger.
+ payload:
+ description: The payload for this task run. Can be any JSON value.
+ options:
+ type: object
+ additionalProperties: true
+ description: Options for this specific item.
+ StreamBatchItemsResponse:
+ type: object
+ required:
+ - id
+ - itemsAccepted
+ - itemsDeduplicated
+ - sealed
+ properties:
+ id:
+ type: string
+ description: The batch ID.
+ itemsAccepted:
+ type: integer
+ description: Number of items successfully accepted.
+ itemsDeduplicated:
+ type: integer
+ description: Number of items that were deduplicated (already enqueued).
+ sealed:
+ type: boolean
+ description: |
+ Whether the batch was sealed and is ready for processing.
+ If false, the batch needs more items before processing can start.
+ Clients should check this field and retry with missing items if needed.
+ enqueuedCount:
+ type: integer
+ description: Total items currently enqueued. Only present when sealed=false to help with retries.
+ expectedCount:
+ type: integer
+ description: Expected total item count. Only present when sealed=false to help with retries.
securitySchemes:
BearerAuth:
type: http
diff --git a/docs/self-hosting/env/webapp.mdx b/docs/self-hosting/env/webapp.mdx
index 49e561179b..eb4c1ed7f4 100644
--- a/docs/self-hosting/env/webapp.mdx
+++ b/docs/self-hosting/env/webapp.mdx
@@ -101,7 +101,9 @@ mode: "wide"
| `TASK_PAYLOAD_MAXIMUM_SIZE` | No | 3145728 (3MB) | Max task payload size. |
| `BATCH_TASK_PAYLOAD_MAXIMUM_SIZE` | No | 1000000 (1MB) | Max batch payload size. |
| `TASK_RUN_METADATA_MAXIMUM_SIZE` | No | 262144 (256KB) | Max metadata size. |
-| `MAX_BATCH_V2_TRIGGER_ITEMS` | No | 500 | Max batch size. |
+| `MAX_BATCH_V2_TRIGGER_ITEMS` | No | 500 | Max batch size (legacy v2 API). |
+| `STREAMING_BATCH_MAX_ITEMS` | No | 1000 | Max items in streaming batch (v3 API, requires SDK 4.3.1+). |
+| `STREAMING_BATCH_ITEM_MAXIMUM_SIZE` | No | 3145728 (3MB) | Max size per item in streaming batch. |
| `MAXIMUM_DEV_QUEUE_SIZE` | No | — | Max dev queue size. |
| `MAXIMUM_DEPLOYED_QUEUE_SIZE` | No | — | Max deployed queue size. |
| **OTel limits** | | | |
diff --git a/docs/snippets/rate-limit-hit-use-batchtrigger.mdx b/docs/snippets/rate-limit-hit-use-batchtrigger.mdx
index cf209d35ad..009ba46c1d 100644
--- a/docs/snippets/rate-limit-hit-use-batchtrigger.mdx
+++ b/docs/snippets/rate-limit-hit-use-batchtrigger.mdx
@@ -1 +1 @@
-The most common cause of hitting the API rate limit is if you’re calling `trigger()` on a task in a loop, instead of doing this use `batchTrigger()` which will trigger multiple tasks in a single API call. You can have up to 500 tasks in a single batch trigger call.
+The most common cause of hitting the API rate limit is if you're calling `trigger()` on a task in a loop, instead of doing this use `batchTrigger()` which will trigger multiple tasks in a single API call. You can have up to 1,000 tasks in a single batch trigger call with SDK 4.3.1+ (500 in prior versions).
diff --git a/docs/triggering.mdx b/docs/triggering.mdx
index 599fe67e99..464eff79c7 100644
--- a/docs/triggering.mdx
+++ b/docs/triggering.mdx
@@ -198,8 +198,8 @@ Triggers a single run of a task with the payload you pass in, and any options yo
If you need to call `trigger()` on a task in a loop, use
- [`batchTrigger()`](#yourTask-batchtrigger) instead which will trigger up to 500 runs in a single
- call.
+ [`batchTrigger()`](#yourTask-batchtrigger) instead which will trigger up to 1,000 runs in a single
+ call with SDK 4.3.1+ (500 runs in prior versions).
```ts ./trigger/my-task.ts
@@ -971,6 +971,44 @@ await yourTask.trigger(payload, { machine: "large-1x" });
If you don't specify a machine it will use the machine preset for your task (or the default for your project). For more information read [the machines guide](/machines).
+## Streaming batch triggering
+
+This feature is only available with SDK 4.3.1+
+
+For large batches, you can pass an `AsyncIterable` or `ReadableStream` instead of an array. This allows you to generate items on-demand without loading them all into memory upfront.
+
+```ts /trigger/my-task.ts
+import { task } from "@trigger.dev/sdk";
+import { myOtherTask } from "~/trigger/my-other-task";
+
+export const myTask = task({
+ id: "my-task",
+ run: async (payload: { userIds: string[] }) => {
+ // Use an async generator to stream items
+ async function* generateItems() {
+ for (const userId of payload.userIds) {
+ yield { payload: { userId } };
+ }
+ }
+
+ const batchHandle = await myOtherTask.batchTrigger(generateItems());
+
+ return { batchId: batchHandle.batchId };
+ },
+});
+```
+
+This works with all batch trigger methods:
+
+- `yourTask.batchTrigger()`
+- `yourTask.batchTriggerAndWait()`
+- `batch.triggerByTask()`
+- `batch.triggerByTaskAndWait()`
+- `tasks.batchTrigger()`
+
+Streaming is especially useful when generating batches from database queries, API pagination, or
+file processing where you don't want to load all items into memory at once.
+
## Large Payloads
We recommend keeping your task payloads as small as possible. We currently have a hard limit on task payloads above 10MB.
@@ -1049,7 +1087,3 @@ export const myTask = task({
```
-
-### Batch Triggering
-
-When using triggering a batch, the total size of all payloads cannot exceed 1MB. This means if you are doing a batch of 100 runs, each payload should be less than 100KB. The max batch size is 500 runs.
diff --git a/docs/v3-openapi.yaml b/docs/v3-openapi.yaml
index 67299f946f..d406ce6c93 100644
--- a/docs/v3-openapi.yaml
+++ b/docs/v3-openapi.yaml
@@ -1301,7 +1301,7 @@ paths:
post:
operationId: batch_trigger_task_v1
summary: Batch trigger tasks
- description: Batch trigger tasks with up to 500 payloads.
+ description: Batch trigger tasks with up to 1,000 payloads with SDK 4.3.1+ (500 in prior versions).
requestBody:
required: true
content: