diff --git a/package-lock.json b/package-lock.json index 3d83d0061c..05a2ba02ef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,7 +36,7 @@ "@adobe/spacecat-shared-ticket-client": "^1.0.4", "@adobe/spacecat-shared-tier-client": "1.5.1", "@adobe/spacecat-shared-tokowaka-client": "1.20.0", - "@adobe/spacecat-shared-user-manager-client": "1.3.1", + "@adobe/spacecat-shared-user-manager-client": "1.5.0", "@adobe/spacecat-shared-utils": "1.124.1", "@adobe/spacecat-shared-vault-secrets": "1.3.5", "@aws-sdk/client-cloudfront": "3.1045.0", @@ -827,7 +827,6 @@ "resolved": "https://registry.npmjs.org/@adobe/helix-universal/-/helix-universal-5.4.1.tgz", "integrity": "sha512-wgmjwo0xJkYhFQUmv6GTPvCFjDYBoT7zP3OuAxLN+FlHgS6kDkbJOtKxwQn9SrWbhoIfM8GdCnRDpBn6BmkASw==", "license": "Apache-2.0", - "peer": true, "dependencies": { "@adobe/fetch": "4.3.0", "aws4": "1.13.2" @@ -10733,9 +10732,9 @@ } }, "node_modules/@adobe/spacecat-shared-user-manager-client": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/@adobe/spacecat-shared-user-manager-client/-/spacecat-shared-user-manager-client-1.3.1.tgz", - "integrity": "sha512-aZmgVJnJPlIR7VXgx3tYW4UMU76dkZ29AToxoFf9fmVo68dvbAQd0Tqzn5+0W66jI+icz2qNxJp5JZk32Y24ZQ==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@adobe/spacecat-shared-user-manager-client/-/spacecat-shared-user-manager-client-1.5.0.tgz", + "integrity": "sha512-ut7smhF4L592MrCrV6ZG9RQqN1CRH9tKtzePa6w3xbPyn+65ZUdrXI6MmbFg+TpXI1rL8Mygg9Jm6rBAWAC4BA==", "license": "Apache-2.0", "dependencies": { "openapi-fetch": "0.17.0" @@ -11581,6 +11580,7 @@ "integrity": "sha512-pEHZqRkAlHfnfAU9tK+WpKv/gBNjGJrHMgA3A0iYRGyswBS2t0pfez+lWlwktb3Bqa0ovh7w/QJTFwp3fDxLNg==", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "@smithy/types": "^4.14.3", "tslib": "^2.6.2" @@ -13491,6 +13491,7 @@ "integrity": "sha512-wfAWZ6oIrsDOFyYm9bDQNva/WCmvIrVqP3dSCePN5YYWCGWWXkikn5YC0wPSxF92M8kQFPfdVpMaTTV1mRk4Lw==", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "@aws-sdk/core": "^3.974.21", "@smithy/core": "^3.24.6", @@ -13507,6 +13508,7 @@ "integrity": "sha512-bBmkG0Dnhfq0/T4Z0PpUr7HkncBVaWvvCbvafeaUM+yC9wa8GGjLJmonq0QL17REB9WivgGeYgWQ5A80Uw5UnQ==", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "mnemonist": "0.38.3", "tslib": "^2.6.2" @@ -13570,6 +13572,7 @@ "integrity": "sha512-FMgyzUq3Jh+ONRYxryBRNdBd+FUX8PwRl07ccQknNdoms6KCeAEusCkl6whqpDrPQ6OH0ddeSifKyqYSs2DLIw==", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "@aws-sdk/endpoint-cache": "^3.972.8", "@aws-sdk/types": "^3.973.13", @@ -13587,6 +13590,7 @@ "integrity": "sha512-pEHZqRkAlHfnfAU9tK+WpKv/gBNjGJrHMgA3A0iYRGyswBS2t0pfez+lWlwktb3Bqa0ovh7w/QJTFwp3fDxLNg==", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "@smithy/types": "^4.14.3", "tslib": "^2.6.2" @@ -14242,8 +14246,7 @@ "version": "2.7.0", "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.7.0.tgz", "integrity": "sha512-qn6tAIZEw5i/wiESBF4nQxZkl86aY4KoO0IkUa2Lh+rya64oTOdJQFlZuMwI1Qz9VBJQrQC4QlSA2DNek5gCOA==", - "license": "(Apache-2.0 AND BSD-3-Clause)", - "peer": true + "license": "(Apache-2.0 AND BSD-3-Clause)" }, "node_modules/@cfworker/json-schema": { "version": "4.1.1", @@ -14267,7 +14270,6 @@ "resolved": "https://registry.npmjs.org/@connectrpc/connect/-/connect-2.1.1.tgz", "integrity": "sha512-JzhkaTvM73m2K1URT6tv53k2RwngSmCXLZJgK580qNQOXRzZRR/BCMfZw3h+90JpnG6XksP5bYT+cz0rpUzUWQ==", "license": "Apache-2.0", - "peer": true, "peerDependencies": { "@bufbuild/protobuf": "^2.7.0" } @@ -15245,7 +15247,6 @@ "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.3.80.tgz", "integrity": "sha512-vcJDV2vk1AlCwSh3aBm/urQ1ZrlXFFBocv11bz/NBUfLWD5/UDNMzwPdaAd2dKvNmTWa9FM2lirLU3+JCf4cRA==", "license": "MIT", - "peer": true, "dependencies": { "@cfworker/json-schema": "^4.0.2", "ansi-styles": "^5.0.0", @@ -15531,7 +15532,6 @@ "resolved": "https://registry.npmjs.org/@octokit/core/-/core-7.0.6.tgz", "integrity": "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==", "license": "MIT", - "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", @@ -15738,7 +15738,6 @@ "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", "devOptional": true, "license": "Apache-2.0", - "peer": true, "engines": { "node": ">=8.0.0" } @@ -15903,7 +15902,6 @@ "integrity": "sha512-r86ut4T1e8vNwB35CqCcKd45yzqH6/6Wzvpk2/cZB8PsPLlZFTvrh8yfOS3CYZYcUmAx4hHTZJ8AO8Dj8nrdhw==", "devOptional": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/resources": "2.6.1", @@ -17921,7 +17919,6 @@ "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.6.tgz", "integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==", "license": "MIT", - "peer": true, "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^5.0.0", @@ -18169,7 +18166,6 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -18216,7 +18212,6 @@ "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -18665,7 +18660,6 @@ "resolved": "https://registry.npmjs.org/aws-xray-sdk-core/-/aws-xray-sdk-core-3.12.0.tgz", "integrity": "sha512-lwalRdxXRy+Sn49/vN7W507qqmBRk5Fy2o0a9U6XTjL9IV+oR5PUiiptoBrOcaYCiVuGld8OEbNqhm6wvV3m6A==", "license": "Apache-2.0", - "peer": true, "dependencies": { "@aws-sdk/types": "^3.4.1", "@smithy/service-error-classification": "^2.0.4", @@ -19415,7 +19409,6 @@ "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=18" } @@ -21645,7 +21638,6 @@ "integrity": "sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -25983,7 +25975,6 @@ "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==", "dev": true, "license": "MIT", - "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -27040,6 +27031,7 @@ "integrity": "sha512-2K9QYubXx/NAjv4VLq1d1Ly8pWNC5L3BrixtdkyTegXWJIqY+zLNDhhX/A+ZwWt70tB1S8H4BE8FLYEFyNoOBw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "obliterator": "^1.6.1" } @@ -27050,7 +27042,6 @@ "integrity": "sha512-UczzB+0nnwGotYSgllfARAqWCJ5e/skuV2K/l+Zyck/H6pJIhLXuBnz+6vn2i211o7DtbE78HQtsYEKICHGI+g==", "dev": true, "license": "MIT", - "peer": true, "funding": { "type": "opencollective", "url": "https://opencollective.com/mobx" @@ -29615,7 +29606,6 @@ "dev": true, "inBundle": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -30089,7 +30079,8 @@ "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-1.6.1.tgz", "integrity": "sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/on-finished": { "version": "2.4.1", @@ -30143,7 +30134,6 @@ "resolved": "https://registry.npmjs.org/openai/-/openai-5.12.2.tgz", "integrity": "sha512-xqzHHQch5Tws5PcKR2xsZGX9xtch+JQFz5zb14dGqlshmmDAFBFEWmeIpf7wVqWV+w7Emj7jRgkNJakyKE0tYQ==", "license": "Apache-2.0", - "peer": true, "bin": { "openai": "bin/cli" }, @@ -31347,7 +31337,6 @@ "integrity": "sha512-DGrYcCWK7tvYMnWh79yrPHt+vdx9tY+1gPZa7nJQtO/p8bLTDaHp4dzwEhQB7pZ4Xe3ok4XKuEPrVuc+wlpkmw==", "devOptional": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -31358,7 +31347,6 @@ "integrity": "sha512-ibrK8llX2a4eOskq1mXKu/TGZj9qzomO+sNfO98M6d9zIPOEhlBkMkBUBLd1vgS0gQsLDBzA+8jJBVXDnfHmJg==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -32107,7 +32095,6 @@ "integrity": "sha512-WRgl5GcypwramYX4HV+eQGzUbD7UUbljVmS+5G1uMwX/wLgYuJAxGeerXJDMO2xshng4+FXqCgyB5QfClV6WjA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@semantic-release/commit-analyzer": "^13.0.1", "@semantic-release/error": "^4.0.0", @@ -33472,7 +33459,6 @@ "integrity": "sha512-ADu2dF53esUzzM4I0ewxhxFtsDd6v4V6dNkg3vG0iFKhnt06sJneTZnRvujAosZwW0XD58IKgGMQoqri4wHRqg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@emotion/is-prop-valid": "1.4.0", "css-to-react-native": "3.2.0", @@ -34366,7 +34352,6 @@ "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -34494,7 +34479,6 @@ "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", "license": "MIT", - "peer": true, "dependencies": { "@types/unist": "^3.0.0", "bail": "^2.0.0", @@ -35301,7 +35285,6 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", "license": "MIT", - "peer": true, "engines": { "node": ">=10.0.0" }, @@ -35594,7 +35577,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } @@ -35604,7 +35586,6 @@ "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.0.tgz", "integrity": "sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ==", "license": "ISC", - "peer": true, "peerDependencies": { "zod": "^3.25 || ^4" } diff --git a/package.json b/package.json index 59c4f7085e..9b3a11626b 100644 --- a/package.json +++ b/package.json @@ -101,7 +101,7 @@ "@adobe/spacecat-shared-ticket-client": "^1.0.4", "@adobe/spacecat-shared-tier-client": "1.5.1", "@adobe/spacecat-shared-tokowaka-client": "1.20.0", - "@adobe/spacecat-shared-user-manager-client": "1.3.1", + "@adobe/spacecat-shared-user-manager-client": "1.5.0", "@adobe/spacecat-shared-utils": "1.124.1", "@adobe/spacecat-shared-vault-secrets": "1.3.5", "@aws-sdk/client-cloudfront": "3.1045.0", diff --git a/src/controllers/serenity.js b/src/controllers/serenity.js index a121db07b1..307546b4b8 100644 --- a/src/controllers/serenity.js +++ b/src/controllers/serenity.js @@ -63,6 +63,7 @@ import { } from '../support/serenity/handlers/tags.js'; import { ensureSubworkspace, decommissionBrandWorkspace } from '../support/serenity/workspace-lifecycle.js'; import { isSerenityActiveForOrg } from '../support/serenity/serenity-active.js'; +import { isDynamicAllocationEnabled } from '../support/serenity/dynamic-allocation-active.js'; import { MAX_TOPICS_ON_CREATE } from '../support/serenity/brand-provisioning.js'; import { marketForGeoTargetId } from '../support/serenity/locations.js'; import { brandNeedles, classifyBrandedTag } from '../support/serenity/branded-classifier.js'; @@ -420,6 +421,11 @@ function SerenityController(context, log, env) { return createSerenityTransport({ env: ctx.env || env, imsToken }); } + // Global dynamic-allocation kill-switch for this request (env/Vault boolean, default OFF). Read + // per request off ctx.env, mirroring buildTransport's env resolution. When OFF the metered + // handlers front through a no-op guard (byte-for-byte pre-PR behavior). + const dynamicAllocationEnabled = (ctx) => isDynamicAllocationEnabled(ctx?.env || env); + /** Loads the Brand model instance (for subworkspace-mode write/lifecycle flows). */ async function loadBrand(ctx, brandUuid) { const Brand = ctx?.dataAccess?.Brand; @@ -501,6 +507,10 @@ function SerenityController(context, log, env) { ctx.data || {}, log, classifyPromptType, + { + dynamicAllocation: dynamicAllocationEnabled(ctx), + parentWorkspaceId: auth.parentWorkspaceId ?? '', + }, ) : await handleCreatePrompts( transport, @@ -689,6 +699,9 @@ function SerenityController(context, log, env) { // in depth: this options bag flows into markets-subworkspace.js and // shouldn't carry access to unrelated tables). dataAccess: { BrandSemrushProject: ctx.dataAccess.BrandSemrushProject }, + // Dynamic-allocation kill-switch. The JIT top-up units pool is the org parent passed + // positionally above (auth.parentWorkspaceId) — not duplicated in this options bag. + dynamicAllocation: dynamicAllocationEnabled(ctx), }, ); // Mirror this market as a SpaceCat Site (+ brand_sites link) keyed on the @@ -982,7 +995,16 @@ function SerenityController(context, log, env) { } const transport = buildTransport(ctx, imsToken); const result = auth.mode === 'subworkspace' - ? await handleUpdateModelsSubworkspace(transport, auth.workspaceId, ctx.data || {}, log) + ? await handleUpdateModelsSubworkspace( + transport, + /** @type {string} */ (auth.workspaceId), + ctx.data || {}, + log, + { + dynamicAllocation: dynamicAllocationEnabled(ctx), + parentWorkspaceId: auth.parentWorkspaceId ?? '', + }, + ) : await handleUpdateModels( transport, ctx.dataAccess, @@ -1072,6 +1094,7 @@ function SerenityController(context, log, env) { log, {}, brandPointerReloader(ctx, auth.brandUuid), + { dynamicAllocation: dynamicAllocationEnabled(ctx) }, ); let bareSucceeded = true; if (typeof brand.setStatus === 'function') { @@ -1166,6 +1189,7 @@ function SerenityController(context, log, env) { log, {}, brandPointerReloader(ctx, auth.brandUuid), + { dynamicAllocation: dynamicAllocationEnabled(ctx) }, ); const results = []; for (const m of markets) { @@ -1214,6 +1238,8 @@ function SerenityController(context, log, env) { // Narrowed to the one model the mapping-row helpers touch — see // the single-market create call site for the same rationale. dataAccess: { BrandSemrushProject: ctx.dataAccess.BrandSemrushProject }, + // JIT units pool = the org parent passed positionally above; not duplicated here. + dynamicAllocation: dynamicAllocationEnabled(ctx), }, ); } catch (e) { diff --git a/src/support/serenity/dynamic-allocation-active.js b/src/support/serenity/dynamic-allocation-active.js new file mode 100644 index 0000000000..c13c1bb17c --- /dev/null +++ b/src/support/serenity/dynamic-allocation-active.js @@ -0,0 +1,102 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +// @ts-check + +import { ensureAiHeadroom, requireWorkspaceId } from './resource-manager.js'; +import { withResourceLock } from './resource-lock.js'; + +/** + * The GLOBAL kill-switch for dynamic (just-in-time) Semrush AI resource allocation + * (serenity-docs#22, Rainer's item 3 — supersedes the per-org DB flag of the abandoned #2750). + * + * A single env/Vault boolean, DEFAULT OFF, read once per request off `context.env` on the hot path + * — the same shape this codebase already uses for its other global serenity toggles + * (`SERENITY_ALLOW_WORKSPACE_DELETE`, `SERENITY_ENFORCE_LINKED_SUBWORKSPACE_GUARD`, + * `SERENITY_ALLOW_NON_IMS_AUTH` in rest-transport.js / serenity.js). No PostgREST read, no per-org + * cache: a rollback is one env flip, and when OFF `ensureAiHeadroom` never runs, so the metered + * handlers behave byte-for-byte as they did before this PR. Wired to Vault at + * `dx_mysticat//api-service`. + */ +export const DYNAMIC_ALLOCATION_ENV_FLAG = 'SERENITY_DYNAMIC_ALLOCATION'; + +/** + * Reads the global dynamic-allocation kill-switch. `true` ONLY for the exact string `'true'` + * (env values are strings); anything else — unset, `'false'`, a typo — is OFF. Fail-safe by design. + * @param {object} [env] - the request env (`context.env`). + * @returns {boolean} + */ +export function isDynamicAllocationEnabled(env) { + return env?.[DYNAMIC_ALLOCATION_ENV_FLAG] === 'true'; +} + +/** + * @typedef {object} HeadroomGuard + * @property {boolean} enabled whether JIT top-up is active for this request. + * @property {(need?: import('./resource-manager.js').Dims, + * opts?: { includeDrafted?: boolean }) => Promise<{ toppedUp: boolean }>} ensure + * the choke point every subworkspace metered-write path calls before its metered op. + */ + +/** + * Builds the per-request AI-headroom guard the metered-write handlers front their ops through — the + * single enforcement choke point for JIT top-up. Handlers ALWAYS call `guard.ensure(need, opts)` + * before a metered `createProject` / `publishProject` / model-add publish, regardless of the flag. + * + * - Flag OFF: `ensure` is a genuine no-op — it issues ZERO transport calls and returns immediately + * — so the OFF path is byte-for-byte the pre-PR behavior. + * - Flag ON with the child/master ids missing: FAILS LOUD (throws) rather than silently degrading + * to a no-op. A brand whose org has no parent workspace would otherwise get neither the + * (now-skipped) flat carve nor a JIT top-up — its sub-workspace sits at zero AI resources and the + * very next metered write fails at the Semrush gateway with an opaque error, instead of a clear + * 500 at the moment the misconfiguration is knowable. "Flag ON but silently not metering" is + * exactly the failure mode a kill-switch rollout must not have. + * - Flag ON with both ids present: `ensure` serializes per child (see {@link withResourceLock}) and + * tops up just-in-time via the FAIL-FAST {@link ensureAiHeadroom} (one transfer, no poll; 503 if + * still settling). + * + * @param {any} transport - Serenity transport. + * @param {object} opts + * @param {boolean} opts.enabled - the global kill-switch value for this request. + * @param {string} [opts.subWorkspaceId] - the sub-workspace being written to (`auth.workspaceId`). + * @param {string} [opts.parentWorkspaceId] - the org parent workspace (`auth.parentWorkspaceId`). + * @param {import('./resource-manager.js').Blocks} [opts.ceiling] - per-brand ceiling (optional). + * @param {import('./resource-manager.js').Blocks} [opts.blocks] - grace blocks (optional). + * @param {any} [log] + * @returns {HeadroomGuard} + */ +export function createHeadroomGuard(transport, { + enabled, subWorkspaceId, parentWorkspaceId, ceiling, blocks, +}, log) { + if (!enabled) { + // Disabled path: a genuine no-op, byte-for-byte the pre-PR behavior. + return { + enabled: false, + ensure: async () => ({ toppedUp: false }), + }; + } + // Flag ON: the ids are required from here on — fail loud (throw) rather than silently no-op. + requireWorkspaceId('createHeadroomGuard', { subWorkspaceId, parentWorkspaceId }); + // requireWorkspaceId above throws unless both are non-empty strings; narrow the (JSDoc-optional) + // params for tsc, which cannot narrow a destructured variable through a plain function call. + const childId = /** @type {string} */ (subWorkspaceId); + const parentId = /** @type {string} */ (parentWorkspaceId); + return { + enabled: true, + ensure: (need = {}, { includeDrafted = false } = {}) => withResourceLock( + childId, + () => ensureAiHeadroom(transport, { + subWorkspaceId: childId, parentWorkspaceId: parentId, need, ceiling, blocks, includeDrafted, + }, log), + ), + }; +} diff --git a/src/support/serenity/errors.js b/src/support/serenity/errors.js index 0a9e69b1e3..38559607f5 100644 --- a/src/support/serenity/errors.js +++ b/src/support/serenity/errors.js @@ -35,6 +35,77 @@ export function isUpstreamGone(e) { return e instanceof SerenityTransportError && e.status === 404; } +/** + * The upstream error body as a lowercased string, for message-based classification. The gateway + * returns either a JSON `{ message }` object or a bare text/html string (the disguised-405 case); + * this normalises both so a predicate can match on substrings. Callers guard `instanceof + * SerenityTransportError` (short-circuit) before calling, so `e` is always a transport error here. + * @param {SerenityTransportError} e + * @returns {string} + */ +function bodyText(e) { + const { body } = e; + if (typeof body === 'string') { + return body.toLowerCase(); + } + // Only trust the upstream body's `message`; never fall back to `e.message` (the transport URL), + // which would let a classifier match on the request URL rather than the upstream content. + const msg = body && typeof body === 'object' ? /** @type {{message?: unknown}} */ (body).message : undefined; + return typeof msg === 'string' ? msg.toLowerCase() : ''; +} + +/** + * Terminal parent-pool exhaustion: a `422` whose body says the subscription is out of units. The + * dynamic allocator maps this to `orgPoolExhausted` (409). MUST match on the message, not the bare + * status — the same route also emits a transient `422 "workspace not ready"` (see + * {@link isWorkspaceNotReady}) that is retried, not surfaced. + * @param {unknown} e + * @returns {boolean} + */ +export function isPoolExhausted(e) { + return e instanceof SerenityTransportError + && e.status === 422 + && bodyText(e).includes('insufficient available units'); +} + +/** + * Transient async-lock `422 "workspace not ready"` — fired when a transfer/delete races a still + * settling workspace, and (Gate 0) can persist even after `getWorkspaceStatus` reports `created`. + * Retry with backoff; do NOT surface as pool exhaustion. + * @param {unknown} e + * @returns {boolean} + */ +export function isWorkspaceNotReady(e) { + return e instanceof SerenityTransportError + && e.status === 422 + && bodyText(e).includes('workspace not ready'); +} + +/** + * The disguised metered-quota rejection: a `405` from a metered write/publish whose body signals a + * quota (`used + need > total`). Matches ONLY on an explicit quota signal in the body — NOT any + * `405`, so a legitimate Method-Not-Allowed is not absorbed. The exact disguised-405 body is pinned + * by the PR-4 live-gateway canary; widen the signal here only from that pinned shape. + * @param {unknown} e + * @returns {boolean} + */ +export function isMeteredQuota(e) { + if (!(e instanceof SerenityTransportError) || e.status !== 405) { + return false; + } + const text = bodyText(e); + return text.includes('quota') || text.includes('allocation exhausted'); +} + +/** + * Upstream rate limiting (`429`) — retryable with backoff rather than the generic 502. + * @param {unknown} e + * @returns {boolean} + */ +export function isRateLimited(e) { + return e instanceof SerenityTransportError && e.status === 429; +} + /** * Frozen catalog of error-token strings handlers attach to * `ErrorWithStatusCode.code` so the controller's `mapError` emits them @@ -60,4 +131,10 @@ export const ERROR_CODES = Object.freeze({ // Subworkspace provisioning (serenity dual-mode, subworkspace path). AMBIGUOUS_WORKSPACE: 'ambiguousWorkspace', LINKED_SUBWORKSPACES: 'linkedSubworkspaces', + // Dynamic AI resource allocation (JIT top-up path). + ORG_POOL_EXHAUSTED: 'orgPoolExhausted', + BRAND_AI_LIMIT: 'brandAiLimit', + // Transient: a transfer never cleared the async `workspace not ready` lock — retryable, NOT + // pool exhaustion (distinct from ORG_POOL_EXHAUSTED so the operator/client isn't misled). + WORKSPACE_BUSY: 'workspaceBusy', }); diff --git a/src/support/serenity/handlers/markets-subworkspace.js b/src/support/serenity/handlers/markets-subworkspace.js index ab23babfef..a844c443a9 100644 --- a/src/support/serenity/handlers/markets-subworkspace.js +++ b/src/support/serenity/handlers/markets-subworkspace.js @@ -27,6 +27,7 @@ import { listGlobalModelCatalog, listSliceModels, syncModelsForProject, + countPublishedPrompts, MAX_MODEL_IDS, validateParentIdQuery, } from './markets.js'; @@ -34,6 +35,10 @@ import { listMarkets, resolveProject, mapPublishStatus, projectToSlice, } from '../subworkspace-projects.js'; import { ensureSubworkspace } from '../workspace-lifecycle.js'; +import { createHeadroomGuard } from '../dynamic-allocation-active.js'; +import { + modelChangeUnits, releaseAiSurplus, PROJECT_BLOCK, PROMPT_BLOCK, +} from '../resource-manager.js'; import { DIMENSION, STANDARD_PROMPT_TAG_VALUES } from '../prompt-tags.js'; import { provisionDimensionTree } from '../tag-tree.js'; import { classifyBrandedTag, needlesFromNames } from '../branded-classifier.js'; @@ -330,6 +335,11 @@ async function generateAndAttachPrompts(transport, workspaceId, projectId, { * `brand_to_semrush_projects` mapping row for this project (best-effort, * never fails the create). Omit for a `brand` that is not yet a persisted * row — see `mapping-rows.js` `upsertMappingRow` doc. + * @param {boolean} [options.dynamicAllocation=false] - global kill-switch value. When true, JIT + * top-up fronts the project-create and publish seams (fail-fast) and the flat re-grant in + * `ensureSubworkspace` is skipped. When false, byte-for-byte the pre-this-PR behavior. + * (The units pool for JIT sizing is the positional `parentWorkspaceId` arg — the same id given + * to `ensureSubworkspace` — so it is not duplicated in this options bag.) */ export async function handleCreateMarketSubworkspace( transport, @@ -348,6 +358,7 @@ export async function handleCreateMarketSubworkspace( competitors = [], publishMode = 'require', dataAccess = null, + dynamicAllocation = false, } = {}, ) { const errors = validateCreateBody(body); @@ -373,7 +384,24 @@ export async function handleCreateMarketSubworkspace( // path passes nothing, so we ensure on the spot, sized for one market. const workspaceId = preResolvedWorkspaceId && hasText(preResolvedWorkspaceId) ? preResolvedWorkspaceId - : await ensureSubworkspace(transport, brand, parentWorkspaceId, 1, log, {}, reloadPointer); + : await ensureSubworkspace( + transport, + brand, + parentWorkspaceId, + 1, + log, + {}, + reloadPointer, + { dynamicAllocation }, + ); + + // JIT top-up choke point. The sub-workspace id is only known after ensureSubworkspace resolves + // it; OFF (or a missing parent) yields a no-op guard so the flag-OFF path issues zero reads. + const headroom = createHeadroomGuard( + transport, + { enabled: dynamicAllocation, subWorkspaceId: workspaceId, parentWorkspaceId }, + log, + ); const existing = await resolveProject( transport, @@ -397,6 +425,9 @@ export async function handleCreateMarketSubworkspace( if (!languageId) { return { status: 400, body: { error: 'unknownLanguage', message: `Language '${languageCode}' not found` } }; } + // PROJECT metering seam: ensure one project of headroom before creating a new project (no-op + // when the flag is OFF). Not needed on the adopt-existing-draft branch above (no new project). + await headroom.ensure({ projects: 1 }); const createResp = await transport.createProject( workspaceId, buildCreateProjectBody(body, location, languageId, aliasNames), @@ -527,6 +558,14 @@ export async function handleCreateMarketSubworkspace( }); } + // PROMPT metering seam: before publishing, ensure prompt headroom sized from `used + drafted` + // (includeDrafted) — the just-staged models + generated prompts are drafted synchronously and + // meter at publish, while the post-publish `used` reconciles asynchronously (stale-low). No-op + // when the flag is OFF. Skipped when we won't publish at all. + if (publishMode !== 'skip') { + await headroom.ensure({}, { includeDrafted: true }); + } + // Publish per mode. 'best-effort' swallows a quota 405 (publishing an // empty-units child 405s as a disguised quota rejection, workspace doc §5) and // leaves the project a draft so the brand still succeeds. @@ -620,6 +659,12 @@ export async function handleDeleteMarketSubworkspace( if (dataAccess) { await tombstoneMappingRow(dataAccess, project.id, log); } + // SCOPE NOTE (serenity-docs#22): this market DELETE does NOT yet call `releaseAiSurplus` to hand + // the freed project/prompt units back to the parent pool — release wiring on the delete path is a + // deliberate fast-follow, not built here. The stranded units self-heal (next `ensureAiHeadroom` + // reuse, a later release, or decommission — see the `releaseAiSurplus` scope note). When it + // lands, follow the model-update seam (`handleUpdateModelsSubworkspace`): inline, fail-fast + // (`releaseAiSurplus({ failFast: true })`), best-effort, post-publish — no async queue/worker. return { status: 204 }; } @@ -792,8 +837,25 @@ export async function handleListModelsSubworkspace(transport, workspaceId, query * PUT /serenity/models (subworkspace) — replace the AI-model set for a slice. Resolves * the slice's project from the live listing (404 if absent), then reuses the * shared diff-based sync. Validation mirrors the flat-mode handler exactly. + * + * @param {object} transport + * @param {string} workspaceId + * @param {object} body + * @param {object} log + * @param {object} [options] + * @param {boolean} [options.dynamicAllocation=false] - global kill-switch. When on, size on the + * SIGNED net model delta: top up (`publishedTexts × netDelta`) before the sync's publish on a net + * ADD, and release the freed units after it on a net REMOVAL. Lives at this mode-guarded caller, + * NOT inside the shared `syncModelsForProject`, so flat mode is untouched (byte-for-byte). + * @param {string} [options.parentWorkspaceId=''] - org parent/master workspace id (units pool). */ -export async function handleUpdateModelsSubworkspace(transport, workspaceId, body, log) { +export async function handleUpdateModelsSubworkspace( + transport, + workspaceId, + body, + log, + { dynamicAllocation = false, parentWorkspaceId = '' } = {}, +) { const geoTargetId = normalizeGeoTargetId(Number(body?.geoTargetId)); const languageCode = normalizeLanguageCode(body?.languageCode); if (geoTargetId === null || languageCode === null) { @@ -814,12 +876,66 @@ export async function handleUpdateModelsSubworkspace(transport, workspaceId, bod if (!project) { throw new ErrorWithStatusCode('Market not found for this brand', 404); } - return syncModelsForProject( + const projectId = String(project.id); + + // PROMPT re-meter seam. PUT /models is REPLACE semantics — one request may add AND remove models + // — so size on the SIGNED NET model delta (`finalModelCount − currentModelCount`), not adds. + // published consumption is `publishedTexts × finalModelCount`, so a swap (net 0) or a net removal + // consumes nothing extra and must NOT top up (gross-add sizing over-grants and can spuriously + // 409 ORG_POOL_EXHAUSTED). A net ADD tops up before the sync's publish; a net REMOVAL frees units + // that are handed back after it. No-op when OFF (guarded on `enabled` → zero extra reads). + const headroom = createHeadroomGuard( + transport, + { enabled: dynamicAllocation, subWorkspaceId: workspaceId, parentWorkspaceId }, + log, + ); + let netDelta = 0; + if (headroom.enabled) { + // Reads the current model set to size the delta; syncModelsForProject below reads it again to + // diff — one duplicate ai_models fetch on the flag-ON path (latency-only, not a correctness + // concern; deduping needs a shared pre-mutation diff seam in syncModelsForProject — PR-4). + // listSliceModels returns one cleaned item per attached model, so its length IS the current + // attached-model count (all we need for the net delta — not the id set). + const current = await listSliceModels(transport, workspaceId, projectId); + const finalModelCount = new Set(modelIds.map(String)).size; + netDelta = finalModelCount - current.items.length; + if (netDelta > 0) { + const publishedTexts = await countPublishedPrompts(transport, workspaceId, projectId, log); + // RESOLVED (serenity-docs#22 §2, write-only metering): the prompt dimension is consumed at + // prompt WRITE, not re-gated at publish, so the sync's publish converting this project's + // DRAFTED prompts to `used` triggers NO prompt-dimension quota check — only the published + // texts re-meter against the added models. Sizing `publishedTexts × netDelta` is therefore + // correct, and this seam deliberately does NOT pass `includeDrafted` (unlike the + // create/bulk-prompt seams — theirs guards read-staleness of a SUBSEQUENT op's `used`, a + // different concern from a publish-time gate). The §8 canary confirms §2 empirically. + await headroom.ensure({ prompts: modelChangeUnits(publishedTexts, netDelta) }); + } + } + + const result = await syncModelsForProject( transport, workspaceId, - String(project.id), + projectId, modelIds, { geoTargetId, languageCode }, log, ); + + // Net model REMOVAL freed published-prompt units. Release them AFTER the sync's publish — by then + // the child's `used` reflects the removal, so releaseAiSurplus reads it and lowers `total` to it + // (ordering §3 requires: publish → read → release). Fail-fast (one transfer, no settle poll) + + // best-effort per the release scope decision; the sub-workspace id is present (headroom.enabled). + if (headroom.enabled && netDelta < 0) { + // Non-zero floor is the PRIMARY guard against the all-zero-transfer no-op: a release target of + // 0 for any dim is silently ignored by the gateway (only workspace delete reclaims to zero). + // Retain at least one block per dim so an idle child never asks releaseAiSurplus for a to-zero + // transfer (which it would refuse anyway — this just keeps the request off that path). + await releaseAiSurplus(transport, { + subWorkspaceId: workspaceId, + floor: { projects: PROJECT_BLOCK, prompts: PROMPT_BLOCK }, + failFast: true, + }, log); + } + + return result; } diff --git a/src/support/serenity/handlers/markets.js b/src/support/serenity/handlers/markets.js index fe0d7df339..0181793c64 100644 --- a/src/support/serenity/handlers/markets.js +++ b/src/support/serenity/handlers/markets.js @@ -953,6 +953,59 @@ export async function listSliceModels(transport, semrushWorkspaceId, projectId) return { items }; } +/** + * Counts the project's currently-PUBLISHED prompts (the live layer), by paginating + * `listPromptsByTags` with an empty tag filter — the same live-layer walk `listTagsForProject` + * uses, with the same page ceiling. Used by the dynamic allocator to size the prompt re-meter of a + * model-set change: attaching Δ models to a project re-meters every published text + * (`publishedTexts × Δmodels`, plan §12 / resource-manager `modelChangeUnits`). Bounded by + * `PROMPT_COUNT_PAGE_LIMIT` pages; on a truncated walk it returns the counted-so-far (a floor), + * which can only UNDER-state the need — the transfer 422 remains the authoritative backstop. + * + * @param {any} transport - Serenity transport. + * @param {string} semrushWorkspaceId + * @param {string} projectId + * @param {any} [log] + * @returns {Promise} number of published prompts on the project. + */ +export async function countPublishedPrompts(transport, semrushWorkspaceId, projectId, log) { + const LIMIT = 200; + const PROMPT_COUNT_PAGE_LIMIT = 50; + let count = 0; + let page = 1; + while (page <= PROMPT_COUNT_PAGE_LIMIT) { + let resp; + try { + // eslint-disable-next-line no-await-in-loop + resp = await transport.listPromptsByTags(semrushWorkspaceId, projectId, { + tag_ids: [], page, limit: LIMIT, + }); + } catch (e) { + // An upstream failure MID-WALK is truncation, same as hitting the page ceiling below: return + // the counted-so-far (a floor) instead of propagating the rejection and failing the WHOLE + // metered write over a partial-page read error. The transfer 422 remains the authoritative + // backstop if this under-states the real need. + log?.warn?.('countPublishedPrompts: upstream failure mid-walk; returning counted-so-far', { + semrushWorkspaceId, projectId, page, error: e?.message, + }); + return count; + } + const items = Array.isArray(resp?.items) ? resp.items : []; + count += items.length; + if (items.length < LIMIT) { + break; + } + if (page === PROMPT_COUNT_PAGE_LIMIT) { + log?.warn?.('countPublishedPrompts: page ceiling hit; published-prompt count may be under-stated', { + semrushWorkspaceId, projectId, pages: PROMPT_COUNT_PAGE_LIMIT, + }); + break; + } + page += 1; + } + return count; +} + export async function handleListModels( transport, dataAccess, diff --git a/src/support/serenity/handlers/prompts-subworkspace.js b/src/support/serenity/handlers/prompts-subworkspace.js index 4b2c348459..16a0feccf7 100644 --- a/src/support/serenity/handlers/prompts-subworkspace.js +++ b/src/support/serenity/handlers/prompts-subworkspace.js @@ -34,6 +34,7 @@ import { } from './prompts.js'; import { resolveProject, buildSliceProjectMap, sliceKey } from '../subworkspace-projects.js'; import { redactUpstreamMessage } from '../rest-transport.js'; +import { createHeadroomGuard } from '../dynamic-allocation-active.js'; /** * Subworkspace-mode prompt handlers (serenity dual-mode, subworkspace path). Behaviourally @@ -125,6 +126,7 @@ export async function handleCreatePromptsSubworkspace( body, log, classifyPromptType, + { dynamicAllocation = false, parentWorkspaceId = '' } = {}, ) { const inputs = Array.isArray(body?.prompts) ? body.prompts : []; if (inputs.length === 0) { @@ -206,6 +208,20 @@ export async function handleCreatePromptsSubworkspace( invalidateTagCacheForProject(workspaceId, pid); } + // PROMPT metering seam: the just-created prompts are drafted synchronously across the affected + // projects of THIS child; size headroom from `used + drafted` (includeDrafted, staleness-immune) + // before the publish. One workspace-level top-up covers all affected projects (the allocation is + // per sub-workspace, not per project). No-op when the flag is OFF; skipped when nothing was + // created so the OFF path and the empty path issue zero headroom reads. + if (affectedProjectIds.length > 0) { + const headroom = createHeadroomGuard( + transport, + { enabled: dynamicAllocation, subWorkspaceId: workspaceId, parentWorkspaceId }, + log, + ); + await headroom.ensure({}, { includeDrafted: true }); + } + const publishErrors = await publishAffected(transport, workspaceId, affectedProjectIds, log); // publishAffected returns { projectId, message } records whose message is // ALREADY redacted (redactUpstreamMessage) — pubErr is a record, not a raw error. diff --git a/src/support/serenity/resource-lock.js b/src/support/serenity/resource-lock.js new file mode 100644 index 0000000000..d4142ad522 --- /dev/null +++ b/src/support/serenity/resource-lock.js @@ -0,0 +1,107 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +// @ts-check + +/** + * Per-key async serialization for same-child resource mutations (serenity-docs#22, item 5). + * + * The dynamic allocator's transfer is ABSOLUTE (it SETS `total`, not increments it). Two operations + * that read the same child's `total` concurrently and then each write an absolute value clobber one + * another — the later write wins with a value computed from a stale read, UNDER-provisioning the + * child (e.g. a batch create-prompts fanning out with `mapLimit` across projects of the SAME child, + * or two concurrent create-market calls). Serializing `ensureAiHeadroom` per child turns the + * read-then-absolute-set into a critical section, so each op reads what the previous one wrote. + * + * SCOPE: this lock is IN-PROCESS only — it serializes contention WITHIN one warm Lambda container + * (where the intra-request `mapLimit` fan-out and same-container concurrent requests race). + * Cross-container serialization (two Lambda instances topping up one child at once) needs a + * distributed lock or an async worker and is deferred to the PR-4 rollout hardening — the + * fail-fast single-attempt transfer already shrinks that cross-instance window (fewer, faster + * inline transfers, no multi-second settle hold). + * + * Queue depth is self-bounding: a key's chain is only as deep as the same-child ops in flight in + * this container (the batch `mapLimit` fan-out is itself capped, e.g. `BULK_CREATE_CONCURRENCY`), + * and the chain entry is evicted the moment it drains — so no explicit depth cap is needed. + * + * SAFETY VALVE (MysticatBot 2026-07-06 / 2026-07-09, Rainer 2026-07-13): the predecessor wait has + * no timeout of its own. If a predecessor's `task` hangs (e.g. a transport call against a hung TCP + * connection with no fetch-level timeout), every subsequent same-child op would otherwise queue + * behind it FOREVER, until the warm container recycles — a single stuck write starves the whole + * child. `LOCK_TIMEOUT_MS` bounds the wait: after it elapses, a waiter stops waiting on the + * predecessor and runs anyway, trading a bounded, rare risk of racing the eventual absolute-set + * from a hung predecessor for guaranteed forward progress. The predecessor's own `task` is NOT + * cancelled by this — it keeps running and its result is still delivered to whoever originally + * awaited it; only the QUEUE stops waiting on it. + */ + +/** @type {Map>} tail of the in-flight chain per key. */ +const chains = new Map(); + +/** Default safety-valve wait (see the module doc above). Exported so a caller can reason about / + * override it; `withResourceLock`'s own `timeoutMs` param is the per-call override point (tests use + * a tiny value so the timeout path doesn't need a real multi-second wait). */ +export const LOCK_TIMEOUT_MS = 10_000; + +/** + * Resolves once `promise` settles OR once `ms` elapses, whichever comes first. NEVER rejects — the + * timeout is a safety valve, not a failure signal, so both branches resolve. + * @param {Promise} promise + * @param {number} ms + * @returns {Promise} + */ +function settledOrTimedOut(promise, ms) { + return new Promise((resolve) => { + const timer = setTimeout(resolve, ms); + const onSettled = () => { + clearTimeout(timer); + resolve(); + }; + promise.then(onSettled, onSettled); + }); +} + +/** + * Runs `task` only after any previously-queued task for the same `key` has settled — OR after + * `timeoutMs` elapses, whichever comes first (see the safety-valve note above) — serializing + * same-key work into a chain. A rejected predecessor does NOT poison the queue (the next task runs + * regardless), and the caller still receives `task`'s own result/rejection. + * + * @template T + * @param {string} key - the serialization key (the child workspace id). + * @param {() => Promise} task - the critical section to run under the lock. + * @param {number} [timeoutMs] - safety-valve wait cap (default {@link LOCK_TIMEOUT_MS}). + * @returns {Promise} resolves/rejects with `task`'s outcome. + */ +export function withResourceLock(key, task, timeoutMs = LOCK_TIMEOUT_MS) { + const prev = chains.get(key) ?? Promise.resolve(); + // Wait for the predecessor to settle, but never longer than timeoutMs (the safety valve). + const run = settledOrTimedOut(prev, timeoutMs).then(() => task()); + // The chain tail swallows settlement so (a) the next waiter starts regardless of this outcome and + // (b) we never leave an unhandled rejection dangling on the stored tail. + const tail = run.then(() => {}, () => {}); + chains.set(key, tail); + // Evict once this is the last task in the chain so the Map can't grow unbounded over a warm + // container's lifetime. If another task queued behind us, `chains.get(key)` is a newer tail, so + // we leave it in place. + tail.then(() => { + if (chains.get(key) === tail) { + chains.delete(key); + } + }); + return run; +} + +/** Test-only: drop all in-flight chains. Production code never calls this. */ +export function clearResourceLocks() { + chains.clear(); +} diff --git a/src/support/serenity/resource-manager.js b/src/support/serenity/resource-manager.js new file mode 100644 index 0000000000..df19185222 --- /dev/null +++ b/src/support/serenity/resource-manager.js @@ -0,0 +1,440 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +// @ts-check + +/** + * Dynamic (just-in-time) Semrush AI resource allocation — the pure allocator. + * + * Replaces the up-front flat carve: before a metered op, read the sub-workspace's real headroom and + * transfer only the delta from the parent; hand surplus back on delete. Semrush model, + * live-verified 2026-07-02 ("Gate 0"): a transfer is ABSOLUTE (sets `total`) + idempotent; a carve + * decrements the MASTER's `total`; `free = total − used`; over-carving the master → terminal + * `422 "insufficient available units in subscription"`; a transfer briefly flips the child off + * `created` and may `422 "workspace not ready"` transiently even past `status:created`. + * + * Two pure, transport-injected entry points sit between the handlers and the transport: + * - {@link ensureAiHeadroom} BEFORE a metered op — reads child headroom, tops up in grace-sized + * blocks only when needed (hot path = read + compare, no transfer). + * - {@link releaseAiSurplus} AFTER a delete/removal (async/reconciler) — hands whole freed blocks + * back, never below the child's floor; best-effort (never throws). + * + * `used` reconciles asynchronously after publish (post-publish reads are stale-low), so the caller + * fronting the PROMPT dimension should size `need` at the publish seam from `used + drafted` + * (drafted is set synchronously at draft time) — see the handler-fronting plan; this module exposes + * both figures via {@link readAiTotals}. + */ + +import { ErrorWithStatusCode } from '../utils.js'; +import { SerenityTransportError } from './rest-transport.js'; +import { pollUntilCreated } from './workspace-lifecycle.js'; +import { + ERROR_CODES, isPoolExhausted, isWorkspaceNotReady, +} from './errors.js'; + +/** @typedef {{ used: number, drafted: number, total: number }} AiDim */ +/** @typedef {{ projects: AiDim, prompts: AiDim }} AiTotals */ +/** @typedef {{ projects?: number, prompts?: number }} Dims a per-dimension unit count */ +/** @typedef {{ projects: number, prompts: number }} Blocks */ +/** + * @typedef {object} PollOpts + * @property {number} attempts + * @property {number} intervalMs + * @property {(ms: number) => Promise} sleep + */ + +/** Grace-sized top-up blocks: projects one-at-a-time (checked at publish), prompts in bulk. */ +export const PROJECT_BLOCK = 1; +export const PROMPT_BLOCK = 100; +/** @type {Blocks} */ +export const DEFAULT_BLOCKS = Object.freeze({ projects: PROJECT_BLOCK, prompts: PROMPT_BLOCK }); + +/** The AI dimensions this allocator moves. */ +const DIMS = Object.freeze(/** @type {const} */ (['projects', 'prompts'])); + +/** Real-clock sleep; overridable via the injected `poll.sleep` in tests. */ +const realSleep = (ms) => new Promise((resolve) => { + setTimeout(resolve, ms); +}); + +/** + * Budget-sized settle poll — deliberately tighter than the 30-attempt provisioning poll + * (`workspace-lifecycle.js`) so a transfer-settle wait cannot alone exceed the request budget. + * @type {PollOpts} + */ +export const DEFAULT_POLL = Object.freeze({ attempts: 12, intervalMs: 1000, sleep: realSleep }); + +/** Bounded RETRIES for the transient `422 "workspace not ready"` — 3 retries after the initial + * attempt (4 transfer attempts total). */ +export const NOT_READY_RETRIES = 3; + +/** + * Rounds a unit count up to the next whole block (never below the count; 0 → 0). + * @param {number} n + * @param {number} block + * @returns {number} + */ +export function roundUpToBlock(n, block) { + if (n <= 0) { + return 0; + } + return Math.ceil(n / block) * block; +} + +// ---- need calculators (domain intent — a transport decorator can't compute these) -------------- + +/** + * Prompt units re-metered when a project's model set changes by `deltaModels` — every one of its + * `publishedTexts` published texts re-meters by the model delta (`texts × Δmodels`, live-verified). + * The model-update seam sizes on the SIGNED net delta (`finalModelCount − currentModelCount`): a + * positive value is a top-up need, and this clamps a swap (net 0) or a net removal (net < 0) to 0 + * so no top-up fires — the freed units are handed back by `releaseAiSurplus` after the publish + * instead. (The create/bulk-prompt seams size at their PUBLISH seam from `used + drafted` via + * `ensureAiHeadroom({ includeDrafted })`, which is staleness-immune, so they need no calculator.) + * @param {number} publishedTexts @param {number} deltaModels @returns {number} + */ +export const modelChangeUnits = (publishedTexts, deltaModels) => Math.max(0, publishedTexts) + * Math.max(0, deltaModels); + +/** + * Fail-loud guard for the workspace ids the allocator needs. A missing/blank id means a caller + * didn't thread the resolved auth ids into the fronting seam — a wiring bug, not a client error, so + * this throws a plain Error (→ 500, surfaced in monitoring) rather than a mapped 4xx. Exported so + * {@link createHeadroomGuard} (dynamic-allocation-active.js) can enforce it at the guard's own + * construction point — the only place the flag-ON, ids-missing case is actually reachable from the + * request path (this module's own `ensureAiHeadroom`/`releaseAiSurplus` callers are only ever + * invoked once the guard has already decided to proceed). + * @param {string} fn calling function name (for the message) + * @param {Record} ids id name → value + * @returns {void} + */ +export function requireWorkspaceId(fn, ids) { + for (const [name, value] of Object.entries(ids)) { + if (typeof value !== 'string' || value.trim() === '') { + throw new Error(`resource-manager: ${fn} requires a non-empty ${name}`); + } + } +} + +// ---- reads -------------------------------------------------------------------------------------- + +/** + * STRICT accessor over a `NewWorkspaceResources` body — pulls the nested + * `product_resources.ai.resources.{projects,prompts}` triples. Fails LOUD on a missing/renamed key + * rather than defaulting to 0 (a silent `undefined→0` would cause pool drain or spurious 405s). + * @param {any} resources the `getWorkspaceResources` response + * @returns {AiTotals} + */ +export function readAiTotals(resources) { + const ai = resources?.product_resources?.ai?.resources; + if (!ai || typeof ai !== 'object') { + throw new Error('resource-manager: product_resources.ai.resources missing from workspace resources'); + } + /** @param {'projects'|'prompts'} name @returns {AiDim} */ + const dim = (name) => { + const d = ai[name]; + if (!d || typeof d.used !== 'number' || typeof d.total !== 'number') { + throw new Error(`resource-manager: ai.resources.${name}.{used,total} missing/non-numeric`); + } + return { used: d.used, drafted: typeof d.drafted === 'number' ? d.drafted : 0, total: d.total }; + }; + return { projects: dim('projects'), prompts: dim('prompts') }; +} + +// ---- typed errors ------------------------------------------------------------------------------- +// +// Client-facing messages are GENERIC and carry NO internal ids or pool numbers — the controller's +// mapError echoes `.message` to the client, so anything embedded here (child/master workspace ids, +// free/needed counts) would leak internal Semrush topology + capacity. The distinguishing signal is +// `.code` (part of the public contract); the detailed context (ids, numbers) is logged at the throw +// site instead. + +/** @returns {ErrorWithStatusCode} */ +function orgPoolExhausted() { + const e = new ErrorWithStatusCode('Organization AI resource pool is exhausted', 409); + e.code = ERROR_CODES.ORG_POOL_EXHAUSTED; + return e; +} + +/** @returns {ErrorWithStatusCode} */ +function brandAiLimit() { + const e = new ErrorWithStatusCode('Brand AI resource allocation limit reached', 409); + e.code = ERROR_CODES.BRAND_AI_LIMIT; + return e; +} + +/** + * A transient failure — the transfer hit the async `workspace not ready` lock. `503` (retryable), + * NOT `orgPoolExhausted`: this is lock contention, not a full pool. + * @returns {ErrorWithStatusCode} + */ +function workspaceBusy() { + const e = new ErrorWithStatusCode('Sub-workspace is provisioning, retry', 503); + e.code = ERROR_CODES.WORKSPACE_BUSY; + return e; +} + +// ---- transfer + settle -------------------------------------------------------------------------- + +/** + * Absolute transfer of `{ projects, prompts }` totals onto `workspaceId`, then settle-poll. Retries + * the transient `422 "workspace not ready"` with backoff; maps terminal pool exhaustion to + * `orgPoolExhausted`. Idempotent, so retry after a poll timeout is safe (the poll's 504 propagates; + * the controller maps it to a retryable `503`). + * @param {any} transport + * @param {string} workspaceId + * @param {{ projects: number, prompts: number }} totals + * @param {PollOpts} poll + * @param {any} [log] + * @returns {Promise} + */ +async function transferAndSettle(transport, workspaceId, totals, poll, log) { + for (let attempt = 0; attempt <= NOT_READY_RETRIES; attempt += 1) { + try { + // eslint-disable-next-line no-await-in-loop + await transport.transferWorkspaceResources(workspaceId, { ai: totals }); + // eslint-disable-next-line no-await-in-loop + await pollUntilCreated(transport, workspaceId, poll); + return; + } catch (e) { + if (isPoolExhausted(e)) { + log?.warn?.('SERENITY_ALLOC org pool exhausted on transfer', { workspaceId }); + throw orgPoolExhausted(); + } + if (!isWorkspaceNotReady(e)) { + throw e; // any other error (incl. poll timeout 504) propagates for the caller to map + } + if (attempt === NOT_READY_RETRIES) { + break; // exhausted retries on the transient lock + } + log?.info?.(`SERENITY_ALLOC workspace-not-ready, retrying transfer (attempt ${attempt + 1})`, { workspaceId }); + // eslint-disable-next-line no-await-in-loop + await poll.sleep(poll.intervalMs * (attempt + 1)); + } + } + // Lock contention, not a full pool — surface a retryable 503, not orgPoolExhausted. + log?.warn?.('SERENITY_ALLOC transfer never cleared workspace-not-ready', { workspaceId }); + throw workspaceBusy(); +} + +/** + * FAIL-FAST hot-path transfer (serenity-docs#22, Rainer's hard constraint). A synchronous metered + * write cannot afford the {@link transferAndSettle} settle-poll + not-ready retry loop: a Semrush + * `/resources` transfer now gates every Adobe write, and stacking a 12s poll + 1+2+3s backoff blows + * the ~15s Fastly/Lambda edge budget → a 504 with a half-applied absolute transfer stranded. So on + * the request path we do exactly ONE transfer attempt and NO blocking settle loop: the transfer is + * absolute + idempotent, so if the child is still settling (`422 "workspace not ready"`) we return + * a retryable `503` (`workspaceBusy`) IMMEDIATELY and let the client/UI retry — the next attempt + * finds it settled. Terminal pool exhaustion still maps to `orgPoolExhausted` (409); anything else + * propagates for the controller to map. (The multi-second settle is a hot-path defect, not a tuning + * knob — shrinking the poll doesn't fix it, removing it from the request does.) + * @param {any} transport + * @param {string} workspaceId + * @param {{ projects: number, prompts: number }} totals + * @param {any} [log] + * @returns {Promise} + */ +async function transferOnce(transport, workspaceId, totals, log) { + try { + await transport.transferWorkspaceResources(workspaceId, { ai: totals }); + } catch (e) { + if (isPoolExhausted(e)) { + log?.warn?.('SERENITY_ALLOC org pool exhausted on transfer', { workspaceId }); + throw orgPoolExhausted(); + } + if (isWorkspaceNotReady(e)) { + // Still settling — do NOT poll. Fail fast with a retryable 503; the transfer is idempotent. + log?.info?.('SERENITY_ALLOC workspace not ready on transfer — returning 503', { workspaceId }); + throw workspaceBusy(); + } + throw e; + } +} + +// ---- entry points ------------------------------------------------------------------------------- + +/** + * Ensure `subWorkspaceId` has headroom for `need` before a metered op. Hot path (already covered) + * does a single read and returns without a transfer. Otherwise tops up in whole blocks, gating on + * the per-brand ceiling and the org pool (read from the parent workspace's own `/resources`). + * @param {any} transport + * @param {object} opts + * @param {string} opts.subWorkspaceId the sub-workspace being written to + * @param {string} opts.parentWorkspaceId the parent/master workspace (units source; the org pool) + * @param {Dims} opts.need per-dimension units the imminent op will consume + * @param {Partial} [opts.ceiling] per-brand max `total` per dim (default: no ceiling) + * @param {Blocks} [opts.blocks] grace blocks (default {@link DEFAULT_BLOCKS}) + * @param {boolean} [opts.includeDrafted] size the PROMPT dimension from `used + drafted + need` + * instead of `used + need`. Set at the PUBLISH seam: `drafted` is written synchronously at draft + * time and converts to `used` at publish, but the post-publish `used` reconciles asynchronously + * (stale-low). Sizing from `used + drafted` is staleness-immune, so a just-drafted batch has the + * quota it needs the instant it publishes (plan §21). Prompts only — a project has + * no draft-then-publish metering seam. + * @param {any} [log] + * @returns {Promise<{ toppedUp: boolean, newTotal: { projects: number, prompts: number } }>} + */ +export async function ensureAiHeadroom(transport, { + subWorkspaceId, parentWorkspaceId, need, + ceiling = {}, blocks = DEFAULT_BLOCKS, includeDrafted = false, +}, log) { + // Fail LOUD on a missing sub-/parent-workspace id: an empty id is a fronting/wiring bug (the + // caller failed to thread auth.workspaceId / auth.parentWorkspaceId), and silently reading `''` + // would either drain the wrong pool or throw an opaque transport error. Surface it as a 500. + requireWorkspaceId('ensureAiHeadroom', { subWorkspaceId, parentWorkspaceId }); + + const child = readAiTotals(await transport.getWorkspaceResources(subWorkspaceId)); + + /** @type {{ projects: number, prompts: number }} */ + const newTotal = { projects: child.projects.total, prompts: child.prompts.total }; + let toppedUp = false; + for (const dim of DIMS) { + const draftedAdd = includeDrafted && dim === 'prompts' ? child[dim].drafted : 0; + const required = child[dim].used + draftedAdd + (Number(need?.[dim]) || 0); + if (required > child[dim].total) { + const target = roundUpToBlock(required, blocks[dim]); + const cap = ceiling[dim]; + if (typeof cap === 'number' && target > cap) { + log?.warn?.('SERENITY_ALLOC brand ceiling reached', { + subWorkspaceId, dim, target, cap, + }); + throw brandAiLimit(); + } + newTotal[dim] = target; + toppedUp = true; + } + } + + if (!toppedUp) { + return { toppedUp: false, newTotal }; // hot path — no transfer, no poll + } + + // Advisory pool gauge (NOT a gate): read the MASTER's OWN /resources — NOT /parent/resources, + // which returns child-relative numbers (Gate 0). This read is a non-atomic advisory: under + // concurrent top-ups it can race ahead of the authoritative transfer and report the pool low when + // it isn't. So we do NOT throw on it — a low reading is logged (a greppable pool-free signal for + // the observability follow-up) and we PROCEED to the transfer, whose `422 insufficient units` is + // the single authoritative exhaustion signal (mapped to orgPoolExhausted in transferOnce). + const master = readAiTotals(await transport.getWorkspaceResources(parentWorkspaceId)); + for (const dim of DIMS) { + const delta = newTotal[dim] - child[dim].total; + const free = master[dim].total - master[dim].used; + if (delta > 0 && free < delta) { + log?.warn?.('SERENITY_ALLOC advisory pool-free low (proceeding; transfer 422 is authoritative)', { + subWorkspaceId, dim, free, delta, + }); + } + } + + // FAIL-FAST: one transfer attempt, no settle poll (serenity-docs#22). A still-settling child + // returns a retryable 503 immediately rather than blocking the request on a poll. + log?.info?.('SERENITY_ALLOC top-up', { subWorkspaceId, newTotal }); + await transferOnce(transport, subWorkspaceId, newTotal, log); + return { toppedUp: true, newTotal }; +} + +/** + * Release whole freed blocks back to the parent AFTER a delete/removal has been re-published (so + * `used` reflects the removal). Never drops the child below `floor`; best-effort — logs and returns + * on any failure. + * + * SCOPE DECISION (serenity-docs#22, Rainer 2026-07-08 — explicit, not an oversight): release runs + * INLINE, fail-fast, best-effort, with NO async/queue/worker/reconciler infra — and none is + * needed. Because a transfer is ABSOLUTE + idempotent, a missed or failed release simply leaves + * surplus units stranded on the child; that self-heals three ways — the child's next + * `ensureAiHeadroom` reuses the headroom (no re-transfer), a later release retry lowers it, or + * decommission reclaims the whole allocation. So do NOT read "reconciler" here as "async infra + * still needed"; the earlier "async/reconciler path" framing is superseded by this inline + * best-effort decision. If/when a caller wires release into the delete / model-remove paths, keep + * this same inline best-effort shape — do not introduce a queue or worker for it. + * @param {any} transport + * @param {object} opts + * @param {string} opts.subWorkspaceId + * @param {Partial} [opts.floor] minimum `total` per dim to retain (default 0) + * @param {Blocks} [opts.blocks] + * @param {PollOpts} [opts.poll] + * @param {boolean} [opts.failFast] when true, do the release transfer with ONE attempt and NO + * settle poll (via the hot-path {@link transferOnce}) instead of the poll-based settle loop. Set + * this for a SYNCHRONOUS request-path caller (e.g. the model-update seam releasing units the + * removal just freed): honours the fail-fast hot-path rule, and is safe because release is + * best-effort + the transfer is absolute/idempotent and has no dependent op after it in-request. + * @param {any} [log] + * @returns {Promise<{ + * released: boolean, + * reason?: 'nothing-to-release' | 'requires-decommission' | 'error', + * target?: { projects: number, prompts: number }, + * }>} `released:false` carries a `reason`: `nothing-to-release` (already at/below target), + * `requires-decommission` (surplus exists but its target floors to 0 — unreclaimable via + * transfer; see below), or `error` (a swallowed best-effort transport failure). + */ +export async function releaseAiSurplus(transport, { + subWorkspaceId, floor = {}, blocks = DEFAULT_BLOCKS, poll = DEFAULT_POLL, failFast = false, +}, log) { + try { + // Fail LOUD on a missing/blank id, same as ensureAiHeadroom: a blank id is a caller wiring bug, + // not a client error. Deliberately INSIDE the try — the surrounding catch only re-throws + // unexpected errors (this is one) and swallows expected/best-effort ones, so a wiring bug still + // propagates as a clear 500 rather than the opaque transport error a blank id would otherwise + // produce, while still going through the same instanceof re-throw gate below. + requireWorkspaceId('releaseAiSurplus', { subWorkspaceId }); + const child = readAiTotals(await transport.getWorkspaceResources(subWorkspaceId)); + /** @type {{ projects: number, prompts: number }} */ + const target = { projects: child.projects.total, prompts: child.prompts.total }; + let lowered = false; + let requiresDecommission = false; + for (const dim of DIMS) { + const t = Math.max(Number(floor[dim]) || 0, roundUpToBlock(child[dim].used, blocks[dim])); + if (t < child[dim].total) { + if (t > 0) { + // Only ever lower a dim to a NON-ZERO target. A transfer that sets a dim to 0 is a + // SILENT NO-OP (live-verified 2026-07, release-probe.mjs): the gateway ignores an + // all-zero (and, by extension, a to-zero) transfer. `target` deliberately keeps BOTH + // dims (an un-lowered dim retains its current total), so the emitted payload is always + // the live-verified both-dims shape — never a partial object (which the strict + // transferAndSettle/transferOnce type also forbids) and never a newly-introduced 0. + target[dim] = t; + lowered = true; + } else { + // Surplus exists but the target floors to 0. It CANNOT be reclaimed via transfer (the + // no-op above); full reclamation is via workspace delete/decommission. Leave this dim at + // its current total and flag the caller — do NOT emit a 0 and falsely report success + // (the quiet twin of the original all-zero-no-op bug). + requiresDecommission = true; + } + } + } + if (!lowered) { + if (requiresDecommission) { + log?.warn?.('SERENITY_ALLOC releaseAiSurplus: surplus cannot be reclaimed via transfer ' + + `(target floors to 0) — needs decommission for ${subWorkspaceId}`); + return { released: false, reason: 'requires-decommission' }; + } + return { released: false, reason: 'nothing-to-release' }; + } + if (failFast) { + await transferOnce(transport, subWorkspaceId, target, log); + } else { + await transferAndSettle(transport, subWorkspaceId, target, poll, log); + } + return { released: true, target }; + } catch (e) { + // Best-effort for EXPECTED failures (transport / pool / busy) — a reconciler converges strays. + // Let unexpected bugs (TypeError, malformed-response Error) propagate so they surface in + // monitoring rather than disappearing into a warn log. + if (!(e instanceof SerenityTransportError) && !(e instanceof ErrorWithStatusCode)) { + throw e; + } + log?.warn?.('SERENITY_ALLOC releaseAiSurplus best-effort failure', { + subWorkspaceId, error: e?.message, + }); + return { released: false, reason: 'error' }; + } +} diff --git a/src/support/serenity/rest-transport.js b/src/support/serenity/rest-transport.js index f6d1b54d8b..6890190596 100644 --- a/src/support/serenity/rest-transport.js +++ b/src/support/serenity/rest-transport.js @@ -677,6 +677,21 @@ export function createSerenityTransport({ env, imsToken }) { )); }, + /** + * GET /v1/workspaces/{ws}/resources — the workspace's own AI resources + * (`NewWorkspaceResources`: `product_resources.ai.resources.{projects,prompts,weekly_prompts}` + * as `{ used, drafted, total }`). Read by the dynamic-allocation allocator before a metered op + * (child headroom) and against the MASTER workspace for the org pool (`free = total − used`). + * NOTE: use this on the master id for the pool — `/parent/resources` returns the workspace's + * OWN allocation, not the master pool (live-verified 2026-07-02). + */ + async getWorkspaceResources(workspaceId) { + return unwrap('GET', await users.GET( + '/v1/workspaces/{id}/resources', + { params: { path: { id: workspaceId } } }, + )); + }, + /** * GET /v1/workspaces/{parent}/family — list the parent's sub-workspaces * (and nested sub-workspaces). Used for ambiguous-create recovery: on a diff --git a/src/support/serenity/workspace-lifecycle.js b/src/support/serenity/workspace-lifecycle.js index 434d59bbab..37f95236ac 100644 --- a/src/support/serenity/workspace-lifecycle.js +++ b/src/support/serenity/workspace-lifecycle.js @@ -96,7 +96,7 @@ function subworkspaceTitle(brand) { return hasText(name) ? `${name} [${suffix}]` : `brand-${suffix}`; } -async function pollUntilCreated(transport, workspaceId, { attempts, intervalMs, sleep }) { +export async function pollUntilCreated(transport, workspaceId, { attempts, intervalMs, sleep }) { for (let i = 0; i < attempts; i += 1) { // eslint-disable-next-line no-await-in-loop const status = await transport.getWorkspaceStatus(workspaceId); @@ -259,6 +259,9 @@ async function adoptFromFamily(transport, parentWorkspaceId, title, log) { * re-reads the brand's CURRENT semrush_sub_workspace_id from the data layer. * When supplied, the create path uses it as a last-update concurrency guard * (see below) so a parallel activation cannot orphan a resourced workspace. + * @param {object} [options] - feature-flag toggles for the dual-mode carve. + * @param {boolean} [options.dynamicAllocation] - when true (LLMO/dynamic-allocation ON), skip + * the flat re-grant on an existing sub-workspace; JIT top-up owns sizing. Default false. * @returns {Promise} the subworkspace id. */ export async function ensureSubworkspace( @@ -269,7 +272,9 @@ export async function ensureSubworkspace( log, timing = {}, reloadPointer = null, + options = {}, ) { + const { dynamicAllocation = false } = options; const poll = { attempts: timing.attempts ?? DEFAULT_POLL_ATTEMPTS, intervalMs: timing.intervalMs ?? DEFAULT_POLL_INTERVAL_MS, @@ -290,9 +295,25 @@ export async function ensureSubworkspace( // and a subsequent op 422s "workspace not ready" (verified live // 2026-06-15). So settle before AND after the transfer so the caller can // immediately create/publish projects against it. + // This pre-poll runs regardless of mode — the workspace must be `created` before we return it + // (dynamic allocation only skips the flat re-grant below, not the readiness settle). await pollUntilCreated(transport, existing, poll); - await transport.transferWorkspaceResources(existing, resourceAllocation(marketCount)); - await pollUntilCreated(transport, existing, poll); + // Dynamic allocation (flag ON): SKIP the flat re-grant. The pre-sized + // `resourceAllocation(marketCount)` carve is exactly the up-front over/under-allocation JIT + // replaces — the metered handlers top up just-in-time (ensureAiHeadroom) and release surplus, + // so re-flattening the total here would both undo a JIT top-up and, on an ON→OFF rollback of an + // already-grown child, risk setting `total` below `used`. Flag OFF unchanged (byte-for-byte). + // + // SCOPE DECISION (serenity-docs#22, Rainer 2026-07-08 — explicit, NOT a deferral): there is NO + // rightsizing/backfill sweep for children already carved under the OLD flat allocation, and + // none is planned. It was evaluated and rejected as unnecessary: decommission already releases + // a child's FULL allocation to the parent pool, and any over-provisioned survivor self-heals + // on its next delete/model-remove release or on decommission (the carve only over-reserves — it + // never breaks the child). So do not read the absence of a migration sweep as missing work. + if (!dynamicAllocation) { + await transport.transferWorkspaceResources(existing, resourceAllocation(marketCount)); + await pollUntilCreated(transport, existing, poll); + } return existing; } diff --git a/test/controllers/serenity.test.js b/test/controllers/serenity.test.js index b5cc84848d..e56ec27c3a 100644 --- a/test/controllers/serenity.test.js +++ b/test/controllers/serenity.test.js @@ -1261,7 +1261,12 @@ describe('SerenityController', () => { brandUrlSources: { urls: [], socialAccounts: [], earnedContent: [] }, competitors: [], dataAccess: { BrandSemrushProject: ctx.dataAccess.BrandSemrushProject }, + // Dynamic-allocation kill-switch defaults OFF (env unset) — the guard is a no-op. + dynamicAllocation: false, }); + // The org parent (JIT units pool) is threaded POSITIONALLY (arg index 2), not in the + // options bag — the same id given to ensureSubworkspace. + expect(handlers.handleCreateMarketSubworkspace.firstCall.args[2]).to.equal(WORKSPACE); }); it('createMarket forwards the brand URL sources so the project carries them', async () => { @@ -1857,10 +1862,14 @@ describe('SerenityController', () => { brandUrlSources: { urls: [], socialAccounts: [], earnedContent: [] }, competitors: [], dataAccess: { BrandSemrushProject: ctx.dataAccess.BrandSemrushProject }, + dynamicAllocation: false, }; const { firstCall, secondCall } = handlers.handleCreateMarketSubworkspace; expect(firstCall.args[7]).to.deep.equal(expectedOpts); expect(secondCall.args[7]).to.deep.equal(expectedOpts); + // Org parent (JIT units pool) threaded positionally (arg index 2), not in the options bag. + expect(firstCall.args[2]).to.equal(WORKSPACE); + expect(secondCall.args[2]).to.equal(WORKSPACE); }); it('activate reads the brand URL sources once and applies them to every market', async () => { diff --git a/test/it/postgres/seed-data/organizations.js b/test/it/postgres/seed-data/organizations.js index abee75c1c7..8de2349f98 100644 --- a/test/it/postgres/seed-data/organizations.js +++ b/test/it/postgres/seed-data/organizations.js @@ -10,6 +10,8 @@ * governing permissions and limitations under the License. */ +import { SERENITY_ORG_PARENT_WS_ID } from '../../shared/seed-ids.js'; + /** * Immutable baseline organizations for IT tests. * @@ -24,6 +26,12 @@ export const organizations = [ id: '11111111-1111-4111-b111-111111111111', name: 'Test Org Accessible', ims_org_id: 'AAAAAAAABBBBBBBBCCCCCCCC@AdobeOrg', + // Org-level Semrush parent workspace: gives subworkspace brands under ORG_1 (e.g. BRAND_1) a + // non-null parentWorkspaceId, which the dynamic-allocation JIT guard requires to engage and + // reads as the advisory units pool on a top-up. Import the shared constant (not a literal) so + // a mock-seed change updates both sides in lock-step — see its own doc in seed-ids.js for why + // it deliberately aliases SERENITY_MOCK_PROJECT_ID rather than being a fresh, unique id. + semrush_workspace_id: SERENITY_ORG_PARENT_WS_ID, config: { handlers: {}, slack: { channel: 'C0FAKE0ORG1', workspace: 'WORKSPACE_TEST' }, diff --git a/test/it/postgres/serenity.test.js b/test/it/postgres/serenity.test.js index 9d046f59d7..da12b76a44 100644 --- a/test/it/postgres/serenity.test.js +++ b/test/it/postgres/serenity.test.js @@ -12,7 +12,10 @@ import { ctx } from './harness.js'; import { resetPostgres } from './seed.js'; -import { resetSemrushMocks } from './setup.js'; +import { resetSemrushMocks, setUmMockQuota, dumpUmMock } from './setup.js'; import serenityTests from '../shared/tests/serenity.js'; -serenityTests(() => ctx.httpClient, resetPostgres, resetSemrushMocks); +serenityTests(() => ctx.httpClient, resetPostgres, resetSemrushMocks, { + setUmMockQuota, + dumpUmMock, +}); diff --git a/test/it/postgres/setup.js b/test/it/postgres/setup.js index 50a2eaa252..fbefcc4d11 100644 --- a/test/it/postgres/setup.js +++ b/test/it/postgres/setup.js @@ -152,6 +152,40 @@ export async function resetSemrushMocks() { ); } +/** + * Sets a workspace's finite AI resources on the User Manager mock via its `POST /__quota` control + * route (makes it metered). Each dim is a bare `total` or `{ used, drafted, total }`. Used by the + * dynamic-allocation flag-ON IT to put BRAND_1's sub-workspace into a metered state so the JIT + * guard reads real `/resources` over the wire. TLS verification is already off process-wide. + * + * @param {string} workspaceId - the workspace to meter + * @param {{ projects?: number|object, prompts?: number|object }} dims - per-dimension resources + * @returns {Promise} + */ +export async function setUmMockQuota(workspaceId, dims) { + const res = await fetch(`${UM_MOCK_BASE}/__quota`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ workspaceId, ...dims }), + }); + if (!res.ok) { + throw new Error(`UM mock __quota failed (${res.status}) for ${workspaceId}`); + } +} + +/** + * Reads the User Manager mock's full store snapshot (`GET /__dump`) — to assert mock-side state + * (e.g. a workspace's resource `total` did/didn't change) after a flag-ON request. + * @returns {Promise} + */ +export async function dumpUmMock() { + const res = await fetch(`${UM_MOCK_BASE}/__dump`); + if (!res.ok) { + throw new Error(`UM mock __dump failed (${res.status})`); + } + return res.json(); +} + /** * Creates the MinIO bucket used by IT tests if it does not already exist. * MinIO is S3-compatible so `NoSuchBucket` errors are replaced by `HeadBucket 404`. diff --git a/test/it/shared/seed-ids.js b/test/it/shared/seed-ids.js index baf8a103be..d13413cee4 100644 --- a/test/it/shared/seed-ids.js +++ b/test/it/shared/seed-ids.js @@ -66,6 +66,24 @@ export const BP_PROMPT_1_INTENT = 'informational'; export const SERENITY_MOCK_WORKSPACE_ID = 'a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d'; export const SERENITY_MOCK_PROJECT_ID = 'b2c3d4e5-f6a7-4b8c-9d0e-1f2a3b4c5d6e'; +// Org-level parent workspace for ORG_1, so a subworkspace brand resolves a non-null +// `parentWorkspaceId` — the condition the dynamic-allocation JIT guard needs to engage, and the +// workspace whose `/resources` the allocator reads as the advisory pool gauge when a top-up fires. +// +// DELIBERATELY ALIASES SERENITY_MOCK_PROJECT_ID (not a coincidence, not a new id) — the UM mock's +// `GET .../resources` / `createSubworkspace` handlers both 403 on any id that isn't a +// pre-registered `workspace` entity, and the loaded boot seed (`parent-with-child`) has exactly +// TWO: the one already claimed by SERENITY_MOCK_WORKSPACE_ID (BRAND_1's own sub-workspace — can't +// double as its own org parent) and this one. A genuinely third, distinct workspace would need +// either reseeding the mock's whole store (`__seed` replaces the RESET BASELINE store-wide — too +// broad a blast radius to risk for this) or the mock's internal seed-construction factories, which +// aren't available here: the `mock/*` subpath is an in-monorepo-only export (`files: ["src"]` in +// the package's `package.json`), so api-service, an external npm consumer, can't reach it. +// Referencing the SAME export (not a copy-pasted literal) keeps this a single source of truth +// instead of a silent, unexplained coincidence — its role as a PE-mock project id elsewhere is +// irrelevant to its use here as a UM workspace id metered via `__quota`. +export const SERENITY_ORG_PARENT_WS_ID = SERENITY_MOCK_PROJECT_ID; + // ── FACS state-layer managers (hybrid-model §8.3) ── // The brandManager persona holds state-layer `llmo/can_manage_users` on // MANAGED_BRAND_ID only (seeded in facs-access-mappings.js). It has an EMPTY diff --git a/test/it/shared/tests/serenity.js b/test/it/shared/tests/serenity.js index 59035b4ec3..ea50793ef8 100644 --- a/test/it/shared/tests/serenity.js +++ b/test/it/shared/tests/serenity.js @@ -11,7 +11,9 @@ */ import { expect } from 'chai'; -import { ORG_1_ID, BRAND_1_ID } from '../seed-ids.js'; +import { + ORG_1_ID, BRAND_1_ID, SERENITY_MOCK_WORKSPACE_ID, SERENITY_ORG_PARENT_WS_ID, +} from '../seed-ids.js'; /** * End-to-end tests for the /serenity/* surface (LLMO-5190), driven against the @@ -53,7 +55,12 @@ import { ORG_1_ID, BRAND_1_ID } from '../seed-ids.js'; * `publish_status` -> `live`. Pinned by the bumped client deps, so it runs * unconditionally. */ -export default function serenityTests(getHttpClient, resetData, resetMocks = async () => {}) { +export default function serenityTests( + getHttpClient, + resetData, + resetMocks = async () => {}, + mockControls = {}, +) { // Seed the baseline org/brand rows the catalog + brand-resolution tests read. // (The route-gate cases fire before any DB access, but the org-level reads // need ORG_1 present.) Mirrors every other postgres factory. @@ -398,11 +405,11 @@ export default function serenityTests(getHttpClient, resetData, resetMocks = asy await createUsMarket(); const parent = await createTag('Footwear'); const child = await createTag('Sneakers', parent.body.id); - const childId = child.body.id; + const childTagId = child.body.id; // Rename-only: parentId omitted — the proxy must re-send the child's current parent itself // (gate 5) so the child stays nested, not promoted to root. - const renamed = await getHttpClient().admin.patch(`${base}/tags/${childId}`, { + const renamed = await getHttpClient().admin.patch(`${base}/tags/${childTagId}`, { name: 'Boots', geoTargetId: US_GEO, languageCode: 'en', }); expect(renamed.status).to.equal(200); @@ -411,7 +418,7 @@ export default function serenityTests(getHttpClient, resetData, resetMocks = asy // An explicit null parent is refused: the root level is reserved for the four // dimension roots, so promoting a tag would leave it with no dimension. - const promoted = await getHttpClient().admin.patch(`${base}/tags/${childId}`, { + const promoted = await getHttpClient().admin.patch(`${base}/tags/${childTagId}`, { name: 'Boots', parentId: null, geoTargetId: US_GEO, languageCode: 'en', }); expect(promoted.status).to.equal(400); @@ -682,4 +689,76 @@ export default function serenityTests(getHttpClient, resetData, resetMocks = asy expect(list.body.items.filter((p) => p.text === text)).to.have.lengthOf(1); }); }); + + // Dynamic-allocation kill-switch ON — drives the JIT top-up FRONTING end-to-end against the live + // metered User Manager mock (Rainer's review item #4). ORG_1 carries a parent workspace + // (SERENITY_ORG_PARENT_WS_ID) so BRAND_1's sub-workspace resolves a non-null parent id and + // the guard engages; the flag is a global env kill-switch read per request, toggled here around + // the block. The `__quota` / `__dump` mock control routes are injected by the postgres harness. + describe('Serenity API — dynamic allocation ON (metered JIT via the live UM mock)', () => { + const { setUmMockQuota, dumpUmMock } = mockControls; + const base = `/v2/orgs/${ORG_1_ID}/brands/${BRAND_1_ID}/serenity`; + const US_GEO = 2840; + const CHILD = SERENITY_MOCK_WORKSPACE_ID; // BRAND_1's sub-workspace (the metered child) + const PARENT = SERENITY_ORG_PARENT_WS_ID; // ORG_1's parent workspace (advisory units pool) + + const childTotal = (dump, dim) => { + const rec = (dump.workspace_resources || []).find((r) => r.id === CHILD); + // A missing record (shape mismatch / wiring regression) must fail with an ACTIONABLE message + // here, not surface downstream as the opaque "expected undefined to be above 0". + expect(rec, `expected a workspace_resources record for CHILD (${CHILD}) in the mock dump`) + .to.exist; + return rec.ai?.[dim]?.total; + }; + + // Skip cleanly if a wiring didn't inject the mock control routes (only the postgres harness has + // the live containers); this keeps the shared factory usable by any future non-metered wiring. + before(function skipWithoutMockControls() { + if (typeof setUmMockQuota !== 'function' || typeof dumpUmMock !== 'function') { + this.skip(); + } + }); + + beforeEach(async () => { + await resetData(); + await resetMocks(); + process.env.SERENITY_DYNAMIC_ALLOCATION = 'true'; + // Parent (units pool) amply provisioned; child seeded at ZERO total so a metered write must + // top it up. Both metered so the allocator's strict /resources reads resolve. + await setUmMockQuota(PARENT, { projects: 100, prompts: 100000 }); + await setUmMockQuota(CHILD, { + projects: { used: 0, drafted: 0, total: 0 }, + prompts: { used: 0, drafted: 0, total: 0 }, + }); + }); + + afterEach(() => { + delete process.env.SERENITY_DYNAMIC_ALLOCATION; + }); + + it('tops up the sub-workspace via a live /resources transfer when a metered write needs headroom', async () => { + // create-market fronts PROJECT headroom before createProject: from a seeded 0 total the guard + // tops the child up to a whole block (>=1) with a REAL /resources transfer to the mock, then + // creates + publishes the project. + const created = await getHttpClient().admin.post(`${base}/markets`, { + market: 'US', languageCode: 'en', brandDomain: 'example.com', brandNames: ['Test Brand'], + }); + expect(created.status).to.equal(201); + + // Positive proof the flag-ON JIT engaged end-to-end over the wire: the child's PROJECT total + // grew from the seeded 0 via a live transfer. A flag-OFF run never fronts/transfers, so it + // would still read 0 — asserting the top-up AND that the kill-switch env toggle took effect + // for the request. (The prompt dimension's `texts × models` sizing is covered by the + // resource-manager unit tests; here the project carve is the decisive over-the-wire signal.) + const dump = await dumpUmMock(); + expect(childTotal(dump, 'projects'), 'projects topped up from 0 via a live transfer') + .to.be.greaterThan(0); + + // Smoke: the prompt write path also runs cleanly under the flag (fronts, then publishes). + const post = await getHttpClient().admin.post(`${base}/prompts`, { + prompts: [{ text: 'best trail running shoes?', geoTargetId: US_GEO, languageCode: 'en' }], + }); + expect(post.status).to.equal(200); + }); + }); } diff --git a/test/support/serenity/dynamic-allocation-active.test.js b/test/support/serenity/dynamic-allocation-active.test.js new file mode 100644 index 0000000000..59a56ef1c8 --- /dev/null +++ b/test/support/serenity/dynamic-allocation-active.test.js @@ -0,0 +1,147 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { expect, use } from 'chai'; +import sinonChai from 'sinon-chai'; +import sinon from 'sinon'; +import { + isDynamicAllocationEnabled, + createHeadroomGuard, + DYNAMIC_ALLOCATION_ENV_FLAG, +} from '../../../src/support/serenity/dynamic-allocation-active.js'; +import { clearResourceLocks } from '../../../src/support/serenity/resource-lock.js'; + +use(sinonChai); + +const CHILD = 'child-ws'; +const MASTER = 'master-ws'; +const log = { info: () => {}, warn: () => {}, error: () => {} }; + +const dimObj = (used, drafted, total) => ({ used, drafted, total }); +const resources = (projects, prompts) => ({ + product_resources: { ai: { resources: { projects, prompts } } }, +}); + +function makeTransport({ child, master } = {}) { + const ample = resources(dimObj(0, 0, 100), dimObj(0, 0, 800)); + const getWorkspaceResources = sinon.stub(); + getWorkspaceResources.withArgs(CHILD).resolves(child ?? ample); + getWorkspaceResources.withArgs(MASTER).resolves(master ?? ample); + return { + getWorkspaceResources, + transferWorkspaceResources: sinon.stub().resolves(), + getWorkspaceStatus: sinon.stub().resolves({ status: 'created' }), + }; +} + +describe('dynamic-allocation-active — isDynamicAllocationEnabled', () => { + it('is true ONLY for the exact string "true"', () => { + expect(isDynamicAllocationEnabled({ [DYNAMIC_ALLOCATION_ENV_FLAG]: 'true' })).to.equal(true); + }); + + it('is false for unset, "false", "TRUE", "1", boolean true, or no env (fail-safe OFF)', () => { + expect(isDynamicAllocationEnabled({})).to.equal(false); + expect(isDynamicAllocationEnabled({ [DYNAMIC_ALLOCATION_ENV_FLAG]: 'false' })).to.equal(false); + expect(isDynamicAllocationEnabled({ [DYNAMIC_ALLOCATION_ENV_FLAG]: 'TRUE' })).to.equal(false); + expect(isDynamicAllocationEnabled({ [DYNAMIC_ALLOCATION_ENV_FLAG]: '1' })).to.equal(false); + expect(isDynamicAllocationEnabled({ [DYNAMIC_ALLOCATION_ENV_FLAG]: true })).to.equal(false); + expect(isDynamicAllocationEnabled(undefined)).to.equal(false); + }); +}); + +describe('dynamic-allocation-active — createHeadroomGuard', () => { + afterEach(() => clearResourceLocks()); + + it('OFF: a genuine no-op — enabled=false and ZERO transport calls', async () => { + const t = makeTransport(); + const guard = createHeadroomGuard( + t, + { enabled: false, subWorkspaceId: CHILD, parentWorkspaceId: MASTER }, + log, + ); + expect(guard.enabled).to.equal(false); + const r = await guard.ensure({ prompts: 100 }, { includeDrafted: true }); + expect(r).to.deep.equal({ toppedUp: false }); + expect(t.getWorkspaceResources).to.not.have.been.called; + expect(t.transferWorkspaceResources).to.not.have.been.called; + }); + + it('ON with ids: routes ensure() through ensureAiHeadroom (reads child, tops up short dim)', async () => { + const t = makeTransport({ + child: resources(dimObj(0, 0, 0), dimObj(0, 0, 0)), + master: resources(dimObj(0, 0, 100), dimObj(0, 0, 800)), + }); + const guard = createHeadroomGuard( + t, + { enabled: true, subWorkspaceId: CHILD, parentWorkspaceId: MASTER }, + log, + ); + expect(guard.enabled).to.equal(true); + const r = await guard.ensure({ projects: 1 }); + expect(r.toppedUp).to.equal(true); + expect(t.getWorkspaceResources).to.have.been.calledWith(CHILD); + expect(t.transferWorkspaceResources).to.have.been.calledOnce; + }); + + it('ON but missing parentWorkspaceId: FAILS LOUD at construction (never silently no-ops)', () => { + // A silent no-op here would mean "flag ON but not actually metering" — the exact failure mode a + // kill-switch rollout must not have (Rainer's review). Throws BEFORE any transport call. + const t = makeTransport(); + expect(() => createHeadroomGuard(t, { enabled: true, subWorkspaceId: CHILD, parentWorkspaceId: '' }, log)).to.throw(/requires a non-empty parentWorkspaceId/); + expect(t.getWorkspaceResources).to.not.have.been.called; + }); + + it('ON but missing subWorkspaceId: FAILS LOUD at construction', () => { + const t = makeTransport(); + expect(() => createHeadroomGuard(t, { enabled: true, subWorkspaceId: '', parentWorkspaceId: MASTER }, log)).to.throw(/requires a non-empty subWorkspaceId/); + expect(t.getWorkspaceResources).to.not.have.been.called; + }); + + it('OFF with missing ids: still a genuine no-op (the flag gate is checked first)', async () => { + // Disabled must never throw regardless of id presence — OFF stays byte-for-byte a no-op even + // when the caller hasn't resolved a parent workspace at all (today's common case pre-rollout). + const t = makeTransport(); + const guard = createHeadroomGuard(t, { enabled: false, subWorkspaceId: '', parentWorkspaceId: '' }, log); + expect(guard.enabled).to.equal(false); + const r = await guard.ensure({ projects: 1 }); + expect(r).to.deep.equal({ toppedUp: false }); + expect(t.getWorkspaceResources).to.not.have.been.called; + }); + + it('serializes concurrent ensure() calls against the same child — the 2nd reads the 1st\'s write', async () => { + // STATEFUL stub: the child's `total` reflects the last absolute transfer. Two concurrent + // top-ups each needing +1 project. WITH the lock, op1 tops projects 0→1 and op2 then reads + // total=1 (used=0) → already covered → NO second transfer. WITHOUT the lock both read the stale + // total=0 and both transfer (a lost-update clobber). Asserting exactly ONE transfer proves the + // critical section actually serializes the read-then-absolute-set. + const childState = { projects: dimObj(0, 0, 0), prompts: dimObj(0, 0, 0) }; + const t = makeTransport(); + t.getWorkspaceResources = sinon.stub().callsFake(async (id) => (id === CHILD + ? resources(childState.projects, childState.prompts) + : resources(dimObj(0, 0, 100), dimObj(0, 0, 800)))); + t.transferWorkspaceResources = sinon.stub().callsFake(async (id, payload) => { + if (id === CHILD) { + childState.projects = dimObj(0, 0, payload.ai.projects); + childState.prompts = dimObj(0, 0, payload.ai.prompts); + } + return null; + }); + const guard = createHeadroomGuard( + t, + { enabled: true, subWorkspaceId: CHILD, parentWorkspaceId: MASTER }, + log, + ); + await Promise.all([guard.ensure({ projects: 1 }), guard.ensure({ projects: 1 })]); + expect(t.transferWorkspaceResources).to.have.callCount(1); + expect(childState.projects.total).to.equal(1); + }); +}); diff --git a/test/support/serenity/dynamic-allocation-fronting.test.js b/test/support/serenity/dynamic-allocation-fronting.test.js new file mode 100644 index 0000000000..fd82eba75f --- /dev/null +++ b/test/support/serenity/dynamic-allocation-fronting.test.js @@ -0,0 +1,383 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/** + * Handler-fronting + enforcement choke-point tests for dynamic AI resource allocation. + * + * Proves that the three subworkspace metered-write handlers front their metered ops through the + * JIT top-up guard when the global kill-switch is ON, and are byte-for-byte no-ops when it is OFF. + * The `choke point` describe is the regression guard the plan calls for: a future metered handler + * added without fronting fails these assertions. + */ + +import { use, expect } from 'chai'; +import chaiAsPromised from 'chai-as-promised'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; + +import { + handleCreateMarketSubworkspace, + handleUpdateModelsSubworkspace, +} from '../../../src/support/serenity/handlers/markets-subworkspace.js'; +import { handleCreatePromptsSubworkspace } from '../../../src/support/serenity/handlers/prompts-subworkspace.js'; +import { clearTagCache } from '../../../src/support/serenity/handlers/markets.js'; +import { clearResourceLocks } from '../../../src/support/serenity/resource-lock.js'; +import { makeProvisioningTransportStubs } from './fixtures/tag-tree.js'; + +use(chaiAsPromised); +use(sinonChai); + +const BRAND = 'brand-1'; +const WS = 'subworkspace-ws-1'; +const PARENT = 'parent-ws'; +const MASTER = 'parent-ws'; +const log = { info: () => {}, error: () => {}, warn: () => {} }; + +const dimObj = (used, drafted, total) => ({ used, drafted, total }); +const resources = (projects, prompts) => ({ + product_resources: { ai: { resources: { projects, prompts } } }, +}); +// Ample child: covered on both dims (no top-up needed). Ample master pool. +const COVERED_CHILD = resources(dimObj(0, 0, 50), dimObj(0, 0, 5000)); +const AMPLE_MASTER = resources(dimObj(0, 0, 100), dimObj(0, 0, 8000)); + +function proj({ id = 'p-us-en', geo = 2840, lang = 'en' } = {}) { + return { + id, + publish_status: 'live', + updated_at: '2026-06-02T00:00:00Z', + settings: { ai: { location: { id: geo }, language: { name: lang } } }, + }; +} + +function makeTransport(overrides = {}) { + const getWorkspaceResources = sinon.stub(); + getWorkspaceResources.withArgs(WS).resolves(COVERED_CHILD); + getWorkspaceResources.withArgs(MASTER).resolves(AMPLE_MASTER); + // create-market provisions the dimension-root tag tree (provisionDimensionTree) before its + // metered ops; this test suite cares about JIT fronting, not tag-tree logic, so start from an + // empty project and let provisioning actually run against a stub that folds writes back in. + const { listProjectTags, createProjectTags } = makeProvisioningTransportStubs(); + return { + listProjects: sinon.stub().resolves({ items: [] }), + getInitStatus: sinon.stub().resolves({ initialized: true }), + createProject: sinon.stub().resolves({ id: 'p-us-en' }), + publishProject: sinon.stub().resolves(null), + deleteProject: sinon.stub().resolves(null), + listLanguages: sinon.stub().resolves({ items: [{ id: 'lang-en', name: 'English' }] }), + transferWorkspaceResources: sinon.stub().resolves(null), + getWorkspaceStatus: sinon.stub().resolves({ status: 'created' }), + getWorkspaceResources, + listPromptsByTags: sinon.stub().resolves({ items: [] }), + createPromptsByIds: sinon.stub().resolves({ items: [{ id: 'prompt-1' }] }), + listAiModels: sinon.stub().resolves({ items: [] }), + listGlobalAiModels: sinon.stub().resolves({ items: [] }), + addAiModel: sinon.stub().resolves(null), + deleteAiModelsByIds: sinon.stub().resolves(null), + listProjectTags, + createProjectTags, + listBenchmarks: sinon.stub().resolves({ aio_benchmarks: [{ id: 'bench-1', main_brand: true }] }), + resolveUrl: sinon.stub().callsFake( + (url) => Promise.resolve({ domain: url, primary_url: url, is_valid: true }), + ), + createBrandUrls: sinon.stub().resolves({ ids: [], existing_count: 0 }), + createBenchmarks: sinon.stub().resolves({ ids: ['bm-new'], existing_count: 0 }), + ...overrides, + }; +} + +function makeBrand() { + let ws = WS; + return { + getId: () => BRAND, + getName: () => 'Acme', + getSemrushSubWorkspaceId: () => ws, + setSemrushSubWorkspaceId: (v) => { ws = v; }, + save: sinon.stub().resolves(), + }; +} + +const createBody = { + market: 'us', languageCode: 'en', brandDomain: 'example.com', brandNames: ['B'], brandDisplayName: 'B', +}; + +describe('dynamic-allocation fronting — create-market', () => { + afterEach(() => { + sinon.restore(); + clearTagCache(); + clearResourceLocks(); + }); + + it('ON: fronts the project seam before createProject and the publish seam before publishProject', async () => { + const t = makeTransport(); + await handleCreateMarketSubworkspace(t, makeBrand(), PARENT, createBody, log, null, null, { + dynamicAllocation: true, parentWorkspaceId: MASTER, publishMode: 'require', + }); + // Child headroom read happened (guard is live), and BEFORE the metered ops it fronts. + expect(t.getWorkspaceResources).to.have.been.calledWith(WS); + expect(t.getWorkspaceResources.calledBefore(t.createProject)).to.equal(true); + expect(t.getWorkspaceResources.calledBefore(t.publishProject)).to.equal(true); + }); + + it('ON + covered child: NO transfer at all — proves the flat re-grant carve is skipped', async () => { + const t = makeTransport(); + await handleCreateMarketSubworkspace(t, makeBrand(), PARENT, createBody, log, null, null, { + dynamicAllocation: true, parentWorkspaceId: MASTER, publishMode: 'require', + }); + // With the flag ON, ensureSubworkspace skips the flat resourceAllocation re-grant AND the + // covered child needs no JIT top-up → zero transfers. + expect(t.transferWorkspaceResources).to.not.have.been.called; + }); + + it('OFF: byte-for-byte — the flat re-grant transfer still runs and NO headroom read happens', async () => { + const t = makeTransport(); + await handleCreateMarketSubworkspace(t, makeBrand(), PARENT, createBody, log, null, null, { + dynamicAllocation: false, parentWorkspaceId: MASTER, publishMode: 'require', + }); + // Flag OFF: the pre-PR flat re-grant transfer runs, and the guard is a genuine no-op. + expect(t.transferWorkspaceResources).to.have.been.called; + expect(t.getWorkspaceResources).to.not.have.been.called; + }); + + it('ON + short child: tops up the project dimension before createProject', async () => { + const getWorkspaceResources = sinon.stub(); + getWorkspaceResources.withArgs(WS).resolves(resources(dimObj(0, 0, 0), dimObj(0, 0, 5000))); + getWorkspaceResources.withArgs(MASTER).resolves(AMPLE_MASTER); + const t = makeTransport({ getWorkspaceResources }); + await handleCreateMarketSubworkspace(t, makeBrand(), PARENT, createBody, log, null, null, { + dynamicAllocation: true, parentWorkspaceId: MASTER, publishMode: 'require', + }); + // A projects top-up (0 → 1 block) fired, and before the project was created. + expect(t.transferWorkspaceResources).to.have.been.called; + expect(t.transferWorkspaceResources.calledBefore(t.createProject)).to.equal(true); + }); +}); + +describe('dynamic-allocation fronting — create-prompts', () => { + afterEach(() => { + sinon.restore(); + clearResourceLocks(); + }); + + it('ON: fronts the publish seam with a child headroom read before publishProject', async () => { + const t = makeTransport({ listProjects: sinon.stub().resolves({ items: [proj()] }) }); + await handleCreatePromptsSubworkspace( + t, + WS, + { + prompts: [{ + text: 'q', geoTargetId: 2840, languageCode: 'en', tagIds: ['tag-1'], + }], + }, + log, + undefined, // classifyPromptType (tag-dimension path — not under test here) + { dynamicAllocation: true, parentWorkspaceId: MASTER }, + ); + expect(t.getWorkspaceResources).to.have.been.calledWith(WS); + expect(t.getWorkspaceResources.calledBefore(t.publishProject)).to.equal(true); + }); + + it('OFF: byte-for-byte — no headroom read', async () => { + const t = makeTransport({ listProjects: sinon.stub().resolves({ items: [proj()] }) }); + await handleCreatePromptsSubworkspace( + t, + WS, + { + prompts: [{ + text: 'q', geoTargetId: 2840, languageCode: 'en', tagIds: ['tag-1'], + }], + }, + log, + undefined, // classifyPromptType + { dynamicAllocation: false, parentWorkspaceId: MASTER }, + ); + expect(t.getWorkspaceResources).to.not.have.been.called; + }); +}); + +describe('dynamic-allocation fronting — update-models', () => { + afterEach(() => { + sinon.restore(); + clearResourceLocks(); + }); + + it('ON: sizes the prompt re-meter from publishedTexts × Δmodels and fronts before the sync', async () => { + // Current model m1 attached; request adds m2 (Δ=1). 2 published prompts → need.prompts = 2. + const listAiModels = sinon.stub().resolves({ items: [{ id: 'a1', model: { id: 'm1', key: 'k1' } }] }); + const listPromptsByTags = sinon.stub().resolves({ items: [{ id: 'q1' }, { id: 'q2' }] }); + // Child short on prompts so the 2-unit need forces a visible top-up. + const getWorkspaceResources = sinon.stub(); + getWorkspaceResources.withArgs(WS).resolves(resources(dimObj(0, 0, 50), dimObj(0, 0, 0))); + getWorkspaceResources.withArgs(MASTER).resolves(AMPLE_MASTER); + const t = makeTransport({ + listProjects: sinon.stub().resolves({ items: [proj()] }), + listAiModels, + listPromptsByTags, + getWorkspaceResources, + }); + await handleUpdateModelsSubworkspace( + t, + WS, + { geoTargetId: 2840, languageCode: 'en', modelIds: ['m1', 'm2'] }, + log, + { dynamicAllocation: true, parentWorkspaceId: MASTER }, + ); + expect(t.getWorkspaceResources).to.have.been.calledWith(WS); + // Top-up ran before the model add/publish (headroom precedes the metered change). + expect(t.getWorkspaceResources.calledBefore(t.addAiModel)).to.equal(true); + expect(t.transferWorkspaceResources).to.have.been.called; + }); + + it('ON + swap (drop 1, add 1 → net delta 0): ZERO resource transfers (no over-grant)', async () => { + // Current [m1]; request [m2] — a swap. finalCount 1 − currentCount 1 = net 0. Gross-add sizing + // would top up publishedTexts × 1; net-delta sizing must transfer nothing. + const listAiModels = sinon.stub().resolves({ items: [{ id: 'a1', model: { id: 'm1', key: 'k1' } }] }); + const listPromptsByTags = sinon.stub().resolves({ items: [{ id: 'q1' }, { id: 'q2' }] }); + const t = makeTransport({ + listProjects: sinon.stub().resolves({ items: [proj()] }), + listAiModels, + listPromptsByTags, + }); + await handleUpdateModelsSubworkspace( + t, + WS, + { geoTargetId: 2840, languageCode: 'en', modelIds: ['m2'] }, + log, + { dynamicAllocation: true, parentWorkspaceId: MASTER }, + ); + // No top-up (net 0) and no release (net not < 0) → zero resource transfers. + expect(t.transferWorkspaceResources).to.not.have.been.called; + expect(t.getWorkspaceResources).to.not.have.been.called; + }); + + it('ON + pure removal (net delta < 0): NO top-up, ONE release AFTER the publish', async () => { + // Current [m1, m2]; request [m1] — net −1. No top-up; after the sync publishes, release the + // freed units (child prompts used 100 / total 500 → release lowers total to 100). + const listAiModels = sinon.stub().resolves({ + items: [{ id: 'a1', model: { id: 'm1', key: 'k1' } }, { id: 'a2', model: { id: 'm2', key: 'k2' } }], + }); + const getWorkspaceResources = sinon.stub(); + getWorkspaceResources.withArgs(WS).resolves(resources(dimObj(1, 0, 5), dimObj(100, 0, 500))); + getWorkspaceResources.withArgs(MASTER).resolves(AMPLE_MASTER); + const t = makeTransport({ + listProjects: sinon.stub().resolves({ items: [proj()] }), + listAiModels, + getWorkspaceResources, + }); + await handleUpdateModelsSubworkspace( + t, + WS, + { geoTargetId: 2840, languageCode: 'en', modelIds: ['m1'] }, + log, + { dynamicAllocation: true, parentWorkspaceId: MASTER }, + ); + // Exactly one transfer (the release), and it ran AFTER the publish (publish → read → release). + expect(t.transferWorkspaceResources).to.have.been.calledOnce; + expect(t.publishProject.calledBefore(t.transferWorkspaceResources)).to.equal(true); + expect(t.getWorkspaceResources.calledAfter(t.publishProject)).to.equal(true); + }); + + it('ON but no models added (net-zero Δ): no headroom read', async () => { + const listAiModels = sinon.stub().resolves({ items: [{ id: 'a1', model: { id: 'm1', key: 'k1' } }] }); + const t = makeTransport({ + listProjects: sinon.stub().resolves({ items: [proj()] }), + listAiModels, + }); + await handleUpdateModelsSubworkspace( + t, + WS, + { geoTargetId: 2840, languageCode: 'en', modelIds: ['m1'] }, + log, + { dynamicAllocation: true, parentWorkspaceId: MASTER }, + ); + expect(t.getWorkspaceResources).to.not.have.been.called; + }); + + it('OFF: byte-for-byte — no model list read for metering, no headroom read', async () => { + const t = makeTransport({ listProjects: sinon.stub().resolves({ items: [proj()] }) }); + await handleUpdateModelsSubworkspace( + t, + WS, + { geoTargetId: 2840, languageCode: 'en', modelIds: ['m1', 'm2'] }, + log, + { dynamicAllocation: false, parentWorkspaceId: MASTER }, + ); + expect(t.getWorkspaceResources).to.not.have.been.called; + }); +}); + +describe('dynamic-allocation — enforcement choke point', () => { + afterEach(() => { + sinon.restore(); + clearTagCache(); + clearResourceLocks(); + }); + + // Every subworkspace metered-write path MUST front through the headroom guard. If a new metered + // handler is added without fronting, add it here and it will fail until it reads child headroom. + const meteredPaths = [ + { + name: 'handleCreateMarketSubworkspace', + // Empty listing → a fresh project is created (the project + publish seams both front). + makeT: () => makeTransport({ listProjects: sinon.stub().resolves({ items: [] }) }), + run: (t) => handleCreateMarketSubworkspace( + t, + makeBrand(), + PARENT, + createBody, + log, + null, + null, + { dynamicAllocation: true, parentWorkspaceId: MASTER, publishMode: 'require' }, + ), + }, + { + name: 'handleCreatePromptsSubworkspace', + makeT: () => makeTransport({ listProjects: sinon.stub().resolves({ items: [proj()] }) }), + run: (t) => handleCreatePromptsSubworkspace( + t, + WS, + { + prompts: [{ + text: 'q', geoTargetId: 2840, languageCode: 'en', tagIds: ['tag-1'], + }], + }, + log, + undefined, // classifyPromptType + { dynamicAllocation: true, parentWorkspaceId: MASTER }, + ), + }, + { + name: 'handleUpdateModelsSubworkspace', + makeT: () => makeTransport({ + listProjects: sinon.stub().resolves({ items: [proj()] }), + listAiModels: sinon.stub().resolves({ items: [] }), + }), + run: (t) => handleUpdateModelsSubworkspace( + t, + WS, + { geoTargetId: 2840, languageCode: 'en', modelIds: ['m1', 'm2'] }, + log, + { dynamicAllocation: true, parentWorkspaceId: MASTER }, + ), + }, + ]; + + meteredPaths.forEach(({ name, makeT, run }) => { + it(`${name} is fronted — reads child headroom when dynamic allocation is ON`, async () => { + const t = makeT(); + await run(t); + expect(t.getWorkspaceResources, `${name} must front its metered op through the headroom guard`) + .to.have.been.calledWith(WS); + }); + }); +}); diff --git a/test/support/serenity/errors.test.js b/test/support/serenity/errors.test.js index 49b594b761..6f40f5f95d 100644 --- a/test/support/serenity/errors.test.js +++ b/test/support/serenity/errors.test.js @@ -15,6 +15,10 @@ import { expect } from 'chai'; import { isUpstreamGone, ERROR_CODES, + isPoolExhausted, + isWorkspaceNotReady, + isMeteredQuota, + isRateLimited, } from '../../../src/support/serenity/errors.js'; import { SerenityTransportError } from '../../../src/support/serenity/rest-transport.js'; @@ -35,7 +39,41 @@ describe('serenity error classification', () => { expect(ERROR_CODES.MARKET_NOT_FOUND).to.equal('marketNotFound'); expect(ERROR_CODES.AMBIGUOUS_WORKSPACE).to.equal('ambiguousWorkspace'); expect(ERROR_CODES.LINKED_SUBWORKSPACES).to.equal('linkedSubworkspaces'); + expect(ERROR_CODES.ORG_POOL_EXHAUSTED).to.equal('orgPoolExhausted'); + expect(ERROR_CODES.BRAND_AI_LIMIT).to.equal('brandAiLimit'); expect(Object.isFrozen(ERROR_CODES)).to.be.true; }); }); + + describe('dynamic-allocation classifiers (body/message, not status alone)', () => { + const err = (status, body) => new SerenityTransportError(status, 'upstream', body); + + it('isPoolExhausted: only a 422 whose message says insufficient units', () => { + expect(isPoolExhausted(err(422, { message: 'insufficient available units in subscription' }))).to.be.true; + expect(isPoolExhausted(err(422, { message: 'workspace not ready' }))).to.be.false; // transient, not exhaustion + expect(isPoolExhausted(err(422, {}))).to.be.false; // object without a message → no match + expect(isPoolExhausted(err(500, { message: 'insufficient available units' }))).to.be.false; + expect(isPoolExhausted(new Error('x'))).to.be.false; + }); + + it('isWorkspaceNotReady: only the transient 422 lock', () => { + expect(isWorkspaceNotReady(err(422, { message: 'workspace not ready' }))).to.be.true; + expect(isWorkspaceNotReady(err(422, { message: 'insufficient available units' }))).to.be.false; + }); + + it('isMeteredQuota: only a 405 with an explicit quota signal — never a bare Method-Not-Allowed', () => { + expect(isMeteredQuota(err(405, { message: 'Quota exceeded' }))).to.be.true; + expect(isMeteredQuota(err(405, 'quota exceeded'))).to.be.true; // string body + expect(isMeteredQuota(err(405, 'prompt allocation exhausted'))).to.be.true; + expect(isMeteredQuota(err(405, { message: 'Method Not Allowed' }))).to.be.false; // legitimate 405 + expect(isMeteredQuota(err(405, 'method not allowed'))).to.be.false; + expect(isMeteredQuota(err(405, null))).to.be.false; // no body signal → not quota + expect(isMeteredQuota(err(404, null))).to.be.false; + }); + + it('isRateLimited: a 429', () => { + expect(isRateLimited(err(429, null))).to.be.true; + expect(isRateLimited(err(503, null))).to.be.false; + }); + }); }); diff --git a/test/support/serenity/handlers/markets.test.js b/test/support/serenity/handlers/markets.test.js index db90ffe6cc..34aff823b4 100644 --- a/test/support/serenity/handlers/markets.test.js +++ b/test/support/serenity/handlers/markets.test.js @@ -28,6 +28,7 @@ import { clearLanguageCache, clearTagCache, listProjectTagTree, + countPublishedPrompts, } from '../../../../src/support/serenity/handlers/markets.js'; import { SerenityTransportError } from '../../../../src/support/serenity/rest-transport.js'; import { ErrorWithStatusCode } from '../../../../src/support/utils.js'; @@ -1659,6 +1660,49 @@ describe('handlers/markets.js — handleUpdateModels', () => { }); }); +describe('countPublishedPrompts', () => { + afterEach(() => sinon.restore()); + + it('sums prompt counts across pages until a short page ends the walk', async () => { + const listPromptsByTags = sinon.stub(); + listPromptsByTags.onCall(0).resolves({ items: new Array(200).fill({ id: 'q' }) }); + listPromptsByTags.onCall(1).resolves({ items: new Array(30).fill({ id: 'q' }) }); + const count = await countPublishedPrompts({ listPromptsByTags }, WORKSPACE, 'p1'); + expect(count).to.equal(230); + expect(listPromptsByTags).to.have.callCount(2); + }); + + it('stops at the page ceiling on a still-full last page and warns (count is a floor)', async () => { + // Every page returns a full 200 → the walk hits the 50-page ceiling and stops, warning that the + // count may be under-stated (it is used as a floor for the metering need). + const warn = sinon.spy(); + const listPromptsByTags = sinon.stub().resolves({ items: new Array(200).fill({ id: 'q' }) }); + const count = await countPublishedPrompts({ listPromptsByTags }, WORKSPACE, 'p1', { warn }); + expect(count).to.equal(200 * 50); + expect(listPromptsByTags).to.have.callCount(50); + expect(warn).to.have.been.calledWithMatch('countPublishedPrompts: page ceiling hit'); + }); + + it('returns 0 for an empty project', async () => { + const listPromptsByTags = sinon.stub().resolves({ items: [] }); + expect(await countPublishedPrompts({ listPromptsByTags }, WORKSPACE, 'p1')).to.equal(0); + }); + + it('an upstream failure MID-WALK returns the counted-so-far instead of rejecting', async () => { + // Page 1 succeeds (200 items); page 2 throws. Must NOT propagate — countPublishedPrompts backs + // resource-manager's modelChangeUnits sizing, and a rejection here would fail the whole metered + // write over what should only under-state the need (the transfer 422 is the real backstop). + const warn = sinon.spy(); + const listPromptsByTags = sinon.stub(); + listPromptsByTags.onCall(0).resolves({ items: new Array(200).fill({ id: 'q' }) }); + listPromptsByTags.onCall(1).rejects(new Error('upstream 500')); + const count = await countPublishedPrompts({ listPromptsByTags }, WORKSPACE, 'p1', { warn }); + expect(count).to.equal(200); + expect(listPromptsByTags).to.have.callCount(2); + expect(warn).to.have.been.calledWithMatch('countPublishedPrompts: upstream failure mid-walk'); + }); +}); + describe('listLanguageCatalog', () => { it('returns the Semrush language catalog, name-sorted, dropping nameless rows', async () => { const transport = { diff --git a/test/support/serenity/resource-lock.test.js b/test/support/serenity/resource-lock.test.js new file mode 100644 index 0000000000..51d21a7992 --- /dev/null +++ b/test/support/serenity/resource-lock.test.js @@ -0,0 +1,129 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { expect } from 'chai'; +import { withResourceLock, clearResourceLocks } from '../../../src/support/serenity/resource-lock.js'; + +const deferred = () => { + let resolve; + let reject; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +}; + +// The safety-valve wait wraps the predecessor in an extra layer of Promise machinery, adding a few +// microtask ticks before a queued task starts (see resource-lock.js). A single `await +// Promise.resolve()` is no longer enough to observe "has the predecessor's task started" — flush +// the whole microtask queue via setImmediate (fires after ALL pending microtasks, well before any +// realistic lock timeout) instead of counting exact ticks. +const flushMicrotasks = () => new Promise((resolve) => { + setImmediate(resolve); +}); + +describe('resource-lock — withResourceLock', () => { + afterEach(() => clearResourceLocks()); + + it('serializes same-key tasks: the second does not start until the first settles', async () => { + const order = []; + const first = deferred(); + const p1 = withResourceLock('child-a', async () => { + order.push('start-1'); + await first.promise; + order.push('end-1'); + return 1; + }); + const p2 = withResourceLock('child-a', async () => { + order.push('start-2'); + return 2; + }); + // p2 must NOT have started while p1 is in flight. + await flushMicrotasks(); + expect(order).to.deep.equal(['start-1']); + first.resolve(); + const [r1, r2] = await Promise.all([p1, p2]); + expect(r1).to.equal(1); + expect(r2).to.equal(2); + expect(order).to.deep.equal(['start-1', 'end-1', 'start-2']); + }); + + it('runs different keys concurrently', async () => { + const order = []; + const a = deferred(); + const p1 = withResourceLock('child-a', async () => { + order.push('start-a'); + await a.promise; + order.push('end-a'); + }); + const p2 = withResourceLock('child-b', async () => { + order.push('start-b'); + }); + await p2; + // b started and finished while a is still blocked → keys are independent. + expect(order).to.deep.equal(['start-a', 'start-b']); + a.resolve(); + await p1; + expect(order).to.deep.equal(['start-a', 'start-b', 'end-a']); + }); + + it('a rejected predecessor does not poison the queue, and the caller still sees its rejection', async () => { + const p1 = withResourceLock('child-a', async () => { + throw new Error('boom'); + }); + const p2 = withResourceLock('child-a', async () => 'ok'); + const e = await p1.catch((x) => x); + expect(e).to.be.instanceOf(Error); + expect(e.message).to.equal('boom'); + expect(await p2).to.equal('ok'); + }); + + it('evicts the chain once idle so the map does not grow unbounded', async () => { + await withResourceLock('child-a', async () => 'done'); + // A fresh call after the chain drained still works (new chain, no stale tail). + expect(await withResourceLock('child-a', async () => 'again')).to.equal('again'); + }); + + it('safety valve: a HUNG predecessor does not block the next task forever — it runs after timeoutMs', async () => { + // A predecessor that never settles (the pathological case the safety valve targets). A tiny + // timeoutMs keeps the test fast; the never-settling promise is never awaited to completion. + const hungTask = () => new Promise(() => {}); + const p1 = withResourceLock('child-a', hungTask, 5); + const p2 = withResourceLock('child-a', async () => 'ran-anyway', 5); + // Without the safety valve this would hang the test (mocha's own timeout would fail it); the + // valve lets p2 proceed once the 5ms wait elapses, even though p1 never settles. + expect(await p2).to.equal('ran-anyway'); + // p1 itself is still pending (its own task never resolves) — the valve only unblocks the QUEUE, + // it does not force-settle the predecessor's own promise. + const raced = await Promise.race([ + p1.then(() => 'p1-settled', () => 'p1-settled'), + new Promise((resolve) => { setTimeout(() => resolve('still-pending'), 10); }), + ]); + expect(raced).to.equal('still-pending'); + }); + + it('safety valve: a FAST predecessor still serializes normally (no timeout hit)', async () => { + // The valve must not change behavior in the common case — a predecessor well under timeoutMs + // is waited for exactly as before; the second task still sees the first one's effect. + const order = []; + const p1 = withResourceLock('child-a', async () => { + order.push('start-1'); + order.push('end-1'); + }, 5000); + const p2 = withResourceLock('child-a', async () => { + order.push('start-2'); + }, 5000); + await Promise.all([p1, p2]); + expect(order).to.deep.equal(['start-1', 'end-1', 'start-2']); + }); +}); diff --git a/test/support/serenity/resource-manager.test.js b/test/support/serenity/resource-manager.test.js new file mode 100644 index 0000000000..5c5768522c --- /dev/null +++ b/test/support/serenity/resource-manager.test.js @@ -0,0 +1,403 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { expect, use } from 'chai'; +import chaiAsPromised from 'chai-as-promised'; +import sinonChai from 'sinon-chai'; +import sinon from 'sinon'; +import { SerenityTransportError } from '../../../src/support/serenity/rest-transport.js'; +import { ErrorWithStatusCode } from '../../../src/support/utils.js'; +import { + roundUpToBlock, modelChangeUnits, readAiTotals, + ensureAiHeadroom, releaseAiSurplus, DEFAULT_BLOCKS, DEFAULT_POLL, +} from '../../../src/support/serenity/resource-manager.js'; + +use(chaiAsPromised); +use(sinonChai); + +const CHILD = 'child-ws'; +const MASTER = 'master-ws'; +const log = { info: () => {}, warn: () => {}, error: () => {} }; +// Fast poll: created immediately, no real sleep. +const poll = { attempts: 3, intervalMs: 0, sleep: () => Promise.resolve() }; + +const dim = (used, drafted, total) => ({ used, drafted, total }); +const resources = (projects, prompts) => ({ + product_resources: { ai: { resources: { projects, prompts } } }, +}); +/** Expected transfer payload. */ +const ai = (projects, prompts) => ({ ai: { projects, prompts } }); +const notReady = () => new SerenityTransportError(422, 'x', { message: 'workspace not ready' }); +const poolFull = () => new SerenityTransportError(422, 'x', { message: 'insufficient available units in subscription' }); + +/** Transport stub: per-workspace resources + transfer + status(created). */ +function makeTransport({ child, master, transfer } = {}) { + const getWorkspaceResources = sinon.stub(); + getWorkspaceResources.withArgs(CHILD).resolves(child); + getWorkspaceResources.withArgs(MASTER).resolves(master); + return { + getWorkspaceResources, + transferWorkspaceResources: transfer || sinon.stub().resolves(), + getWorkspaceStatus: sinon.stub().resolves({ status: 'created' }), + }; +} + +describe('resource-manager — pure helpers', () => { + it('roundUpToBlock: 0/negative → 0, else next whole block', () => { + expect(roundUpToBlock(0, 100)).to.equal(0); + expect(roundUpToBlock(-5, 100)).to.equal(0); + expect(roundUpToBlock(1, 100)).to.equal(100); + expect(roundUpToBlock(100, 100)).to.equal(100); + expect(roundUpToBlock(101, 100)).to.equal(200); + expect(roundUpToBlock(3, 1)).to.equal(3); + }); + + it('modelChangeUnits: publishedTexts × Δmodels, clamps a swap/removal (Δ ≤ 0) to 0', () => { + expect(modelChangeUnits(10, 1)).to.equal(10); // net add of 1 model over 10 texts + expect(modelChangeUnits(200, 2)).to.equal(400); + expect(modelChangeUnits(10, 0)).to.equal(0); // swap (net 0) → no top-up + expect(modelChangeUnits(10, -1)).to.equal(0); // net removal → no top-up (release handles it) + expect(modelChangeUnits(-3, 2)).to.equal(0); // negative texts clamped + }); + + it('readAiTotals: strict nested accessor, drafted defaults to 0', () => { + expect(readAiTotals(resources(dim(1, 0, 5), dim(2, 3, 50)))).to.deep.equal({ + projects: dim(1, 0, 5), prompts: dim(2, 3, 50), + }); + // drafted missing → 0 + expect(readAiTotals(resources({ used: 1, total: 5 }, { used: 0, total: 10 })).projects.drafted) + .to.equal(0); + }); + + it('readAiTotals: fails loud on missing ai block or dimension', () => { + expect(() => readAiTotals({})).to.throw(/product_resources.ai.resources missing/); + expect(() => readAiTotals(resources(undefined, dim(0, 0, 10)))).to.throw(/ai.resources.projects/); + expect(() => readAiTotals(resources({ used: 'x', total: 5 }, dim(0, 0, 10)))).to.throw(/projects/); + }); +}); + +describe('resource-manager — ensureAiHeadroom', () => { + it('fails LOUD on a missing/blank subWorkspaceId or parentWorkspaceId — no transport call', async () => { + const t = makeTransport({ child: resources(dim(1, 0, 5), dim(50, 0, 800)) }); + await expect(ensureAiHeadroom(t, { + subWorkspaceId: '', parentWorkspaceId: MASTER, need: { prompts: 100 }, + }, log)).to.be.rejectedWith(/requires a non-empty subWorkspaceId/); + await expect(ensureAiHeadroom(t, { + subWorkspaceId: CHILD, parentWorkspaceId: ' ', need: { prompts: 100 }, + }, log)).to.be.rejectedWith(/requires a non-empty parentWorkspaceId/); + expect(t.getWorkspaceResources).to.not.have.been.called; + }); + + it('hot path: already covered → no transfer, no poll', async () => { + const t = makeTransport({ child: resources(dim(1, 0, 5), dim(50, 0, 800)) }); + const r = await ensureAiHeadroom(t, { + subWorkspaceId: CHILD, parentWorkspaceId: MASTER, need: { projects: 1, prompts: 100 }, + }, log); + expect(r.toppedUp).to.equal(false); + expect(t.transferWorkspaceResources).to.not.have.been.called; + expect(t.getWorkspaceResources).to.have.been.calledOnceWith(CHILD); // master never read + }); + + it('need = 0 (or absent) is a true no-op — zero transport writes', async () => { + const t = makeTransport({ child: resources(dim(2, 0, 2), dim(800, 0, 800)) }); + const r = await ensureAiHeadroom(t, { + subWorkspaceId: CHILD, parentWorkspaceId: MASTER, need: {}, + }, log); + expect(r.toppedUp).to.equal(false); + expect(t.transferWorkspaceResources).to.not.have.been.called; + }); + + it('tops up only the short dimension to the next block; leaves the covered dim at its total', async () => { + const t = makeTransport({ + // projects covered; prompts short (need 20 → 70 > 60) + child: resources(dim(2, 0, 2), dim(50, 0, 60)), + master: resources(dim(0, 0, 100), dim(0, 0, 800)), + }); + const r = await ensureAiHeadroom(t, { + subWorkspaceId: CHILD, parentWorkspaceId: MASTER, need: { projects: 0, prompts: 20 }, poll, + }, log); + expect(r).to.deep.equal({ toppedUp: true, newTotal: { projects: 2, prompts: 100 } }); + expect(t.transferWorkspaceResources).to.have.been.calledOnceWith(CHILD, ai(2, 100)); + }); + + it('tops up the projects dimension one block at a time', async () => { + const t = makeTransport({ + child: resources(dim(2, 0, 2), dim(0, 0, 100)), + master: resources(dim(2, 0, 13), dim(0, 0, 800)), + }); + const r = await ensureAiHeadroom(t, { + subWorkspaceId: CHILD, parentWorkspaceId: MASTER, need: { projects: 1 }, poll, + }, log); + expect(r.newTotal.projects).to.equal(3); // roundUpToBlock(3, 1) + expect(t.transferWorkspaceResources).to.have.been.calledWith(CHILD, ai(3, 100)); + }); + + it('throws brandAiLimit (409) when the top-up would exceed the per-brand ceiling', async () => { + const t = makeTransport({ child: resources(dim(2, 0, 2), dim(0, 0, 100)) }); + const p = ensureAiHeadroom(t, { + subWorkspaceId: CHILD, + parentWorkspaceId: MASTER, + need: { projects: 3 }, + ceiling: { projects: 3 }, + poll, + }, log); + await expect(p).to.be.rejectedWith(ErrorWithStatusCode); + const e = await p.catch((x) => x); + expect(e.status).to.equal(409); + expect(e.code).to.equal('brandAiLimit'); + expect(t.transferWorkspaceResources).to.not.have.been.called; + }); + + it('advisory pool-free low: does NOT throw (non-atomic read) — warns and PROCEEDS to the transfer', async () => { + // The precheck is advisory only; a low reading races concurrent top-ups. It must not throw a + // spurious 409 — it warns and proceeds, letting the transfer 422 be the authoritative signal. + const warn = sinon.spy(); + const t = makeTransport({ + child: resources(dim(2, 0, 2), dim(750, 0, 750)), // prompts need 100 → 850 target, delta 100 + master: resources(dim(2, 0, 13), dim(760, 0, 800)), // prompts free = 40 < 100 + }); + const r = await ensureAiHeadroom(t, { + subWorkspaceId: CHILD, parentWorkspaceId: MASTER, need: { prompts: 100 }, + }, { ...log, warn }); + expect(r.toppedUp).to.equal(true); + expect(t.transferWorkspaceResources).to.have.been.calledOnce; // proceeded despite the low gauge + expect(warn).to.have.been.calledWithMatch('SERENITY_ALLOC advisory pool-free low (proceeding; transfer 422 is authoritative)'); + }); + + it('maps a terminal 422 "insufficient units" on the transfer to orgPoolExhausted', async () => { + const t = makeTransport({ + child: resources(dim(2, 0, 2), dim(0, 0, 0)), + master: resources(dim(0, 0, 100), dim(0, 0, 800)), + transfer: sinon.stub().rejects(poolFull()), + }); + const e = await ensureAiHeadroom(t, { + subWorkspaceId: CHILD, parentWorkspaceId: MASTER, need: { prompts: 10 }, poll, + }, log).catch((x) => x); + expect(e.code).to.equal('orgPoolExhausted'); + }); + + it('FAIL-FAST: a transient "workspace not ready" 422 → immediate 503 workspaceBusy, ONE transfer, NO poll', async () => { + const transfer = sinon.stub().rejects(notReady()); + const t = makeTransport({ + child: resources(dim(2, 0, 2), dim(0, 0, 0)), + master: resources(dim(0, 0, 100), dim(0, 0, 800)), + transfer, + }); + const e = await ensureAiHeadroom(t, { + subWorkspaceId: CHILD, parentWorkspaceId: MASTER, need: { prompts: 10 }, + }, log).catch((x) => x); + expect(e.status).to.equal(503); + expect(e.code).to.equal('workspaceBusy'); + expect(e.message).to.match(/provisioning, retry/); + // Hard constraint (serenity-docs#22): NO retry loop, NO settle poll on the request path. + expect(transfer).to.have.callCount(1); + expect(t.getWorkspaceStatus).to.not.have.been.called; + }); + + it('FAIL-FAST: a successful top-up does exactly ONE transfer and never polls getWorkspaceStatus', async () => { + const t = makeTransport({ + child: resources(dim(2, 0, 2), dim(0, 0, 0)), + master: resources(dim(0, 0, 100), dim(0, 0, 800)), + }); + const r = await ensureAiHeadroom(t, { + subWorkspaceId: CHILD, parentWorkspaceId: MASTER, need: { prompts: 10 }, + }, log); + expect(r.toppedUp).to.equal(true); + expect(t.transferWorkspaceResources).to.have.callCount(1); + expect(t.getWorkspaceStatus).to.not.have.been.called; + }); + + it('publish-seam sizing (includeDrafted): sizes prompts from used + drafted, staleness-immune', async () => { + // used=50 (stale-low), drafted=120 just staged. Without includeDrafted required=50 (covered by + // total 100 → no top-up, then publish 405s). With includeDrafted required=50+120=170 → top up + // to the next PROMPT_BLOCK (200). + const t = makeTransport({ + child: resources(dim(1, 0, 5), dim(50, 120, 100)), + master: resources(dim(0, 0, 100), dim(0, 0, 800)), + }); + const r = await ensureAiHeadroom(t, { + subWorkspaceId: CHILD, parentWorkspaceId: MASTER, need: {}, includeDrafted: true, + }, log); + expect(r.toppedUp).to.equal(true); + expect(t.transferWorkspaceResources).to.have.been.calledWith(CHILD, ai(5, 200)); + }); + + it('publish-seam sizing (includeDrafted) is a hot-path no-op when used + drafted already fits', async () => { + const t = makeTransport({ + child: resources(dim(1, 0, 5), dim(50, 120, 200)), + master: resources(dim(0, 0, 100), dim(0, 0, 800)), + }); + const r = await ensureAiHeadroom(t, { + subWorkspaceId: CHILD, parentWorkspaceId: MASTER, need: {}, includeDrafted: true, + }, log); + expect(r.toppedUp).to.equal(false); + expect(t.transferWorkspaceResources).to.not.have.been.called; + }); + + it('propagates a non-quota transfer error (e.g. 500) unchanged', async () => { + const t = makeTransport({ + child: resources(dim(2, 0, 2), dim(0, 0, 0)), + master: resources(dim(0, 0, 100), dim(0, 0, 800)), + transfer: sinon.stub().rejects(new SerenityTransportError(500, 'boom')), + }); + const e = await ensureAiHeadroom(t, { + subWorkspaceId: CHILD, parentWorkspaceId: MASTER, need: { prompts: 10 }, poll, + }, log).catch((x) => x); + expect(e).to.be.instanceOf(SerenityTransportError); + expect(e.status).to.equal(500); + }); +}); + +describe('resource-manager — releaseAiSurplus', () => { + it('no-op when nothing frees a whole block', async () => { + const t = makeTransport({ child: resources(dim(2, 0, 2), dim(90, 0, 100)) }); + const r = await releaseAiSurplus(t, { subWorkspaceId: CHILD, poll }, log); + expect(r).to.deep.equal({ released: false, reason: 'nothing-to-release' }); + expect(t.transferWorkspaceResources).to.not.have.been.called; + }); + + it('idle child + zero floor → requires-decommission, NO all-zero transfer (regression)', async () => { + // A fully-idle child (used 0) with the default zero floor would floor BOTH dims to 0. A + // transfer to {0,0} is a live-verified silent no-op, so releaseAiSurplus must NOT emit it and + // must NOT falsely report released:true — it flags requires-decommission and warns instead. + const warn = sinon.spy(); + const t = makeTransport({ child: resources(dim(0, 0, 1), dim(0, 0, 100)) }); + const r = await releaseAiSurplus(t, { subWorkspaceId: CHILD, poll }, { ...log, warn }); + expect(r).to.deep.equal({ released: false, reason: 'requires-decommission' }); + expect(t.transferWorkspaceResources).to.not.have.been.called; + expect(warn).to.have.been.calledWithMatch('needs decommission'); + }); + + it('lowers one dim to non-zero while a zero-floored dim stays at its current total', async () => { + // prompts used 150 → roundUp = 200 < 500 (lowers to a NON-ZERO 200); projects used 0, floor 0 + // → target floors to 0 (unreclaimable via transfer) so it stays at its current total (1). The + // emitted payload keeps BOTH dims non-zero — never a partial object, never a newly-added 0. + const t = makeTransport({ child: resources(dim(0, 0, 1), dim(150, 0, 500)) }); + const r = await releaseAiSurplus(t, { subWorkspaceId: CHILD, failFast: true }, log); + expect(r.released).to.equal(true); + expect(t.transferWorkspaceResources).to.have.been.calledOnceWith(CHILD, ai(1, 200)); + }); + + it('lowers totals to rounded used, never below the floor', async () => { + const t = makeTransport({ child: resources(dim(1, 0, 5), dim(50, 0, 400)) }); + const r = await releaseAiSurplus(t, { + subWorkspaceId: CHILD, floor: { projects: 1, prompts: 0 }, poll, + }, log); + // projects → max(1, roundUp(1))=1 < 5; prompts → max(0, roundUp(50)=100)=100 < 400 + expect(r.released).to.equal(true); + expect(t.transferWorkspaceResources).to.have.been.calledOnceWith(CHILD, ai(1, 100)); + }); + + it('respects a floor above rounded used', async () => { + const t = makeTransport({ child: resources(dim(0, 0, 5), dim(0, 0, 500)) }); + const r = await releaseAiSurplus(t, { + subWorkspaceId: CHILD, floor: { projects: 2, prompts: 200 }, poll, + }, log); + expect(t.transferWorkspaceResources).to.have.been.calledOnceWith(CHILD, ai(2, 200)); + expect(r.target).to.deep.equal({ projects: 2, prompts: 200 }); + }); + + it('is best-effort — swallows an EXPECTED transport failure and returns released:false', async () => { + const warn = sinon.spy(); + const t = makeTransport({ child: resources(dim(0, 0, 5), dim(0, 0, 500)) }); + t.getWorkspaceResources.withArgs(CHILD).rejects(new SerenityTransportError(503, 'transport boom')); + const r = await releaseAiSurplus(t, { subWorkspaceId: CHILD, poll }, { ...log, warn }); + expect(r).to.deep.equal({ released: false, reason: 'error' }); + expect(warn).to.have.been.called; + }); + + it('propagates an UNEXPECTED error (e.g. a bug) rather than hiding it', async () => { + const t = makeTransport({ child: resources(dim(0, 0, 5), dim(0, 0, 500)) }); + t.getWorkspaceResources.withArgs(CHILD).rejects(new TypeError('undefined is not a function')); + await expect(releaseAiSurplus(t, { subWorkspaceId: CHILD, poll }, log)) + .to.be.rejectedWith(TypeError); + }); + + it('fails LOUD on a missing/blank subWorkspaceId — propagates, never swallowed as best-effort', async () => { + // A blank id is a caller wiring bug, not an expected/best-effort failure (transport/pool/busy) + // — it must surface as a clear 500, not the opaque transport error a blank id would otherwise + // produce, and NOT be reported as released:false the way a real best-effort failure would be. + const t = makeTransport({ child: resources(dim(0, 0, 5), dim(0, 0, 500)) }); + await expect(releaseAiSurplus(t, { subWorkspaceId: '', poll }, log)) + .to.be.rejectedWith(/requires a non-empty subWorkspaceId/); + expect(t.getWorkspaceResources).to.not.have.been.called; + }); + + it('release path (async/reconciler) retries the transient "workspace not ready" 422 then settles', async () => { + // releaseAiSurplus keeps the settle-poll + not-ready retry loop — it is the async/reconciler + // path, NOT the synchronous hot path (only ensureAiHeadroom fails fast). + const transfer = sinon.stub(); + transfer.onCall(0).rejects(notReady()); + transfer.onCall(1).resolves(); + const t = makeTransport({ child: resources(dim(1, 0, 5), dim(50, 0, 400)), transfer }); + const r = await releaseAiSurplus(t, { subWorkspaceId: CHILD, poll }, log); + expect(r.released).to.equal(true); + expect(transfer).to.have.callCount(2); + }); + + it('release path surfaces workspaceBusy (503) as an EXPECTED best-effort failure when not-ready never clears', async () => { + const transfer = sinon.stub().rejects(notReady()); + const t = makeTransport({ child: resources(dim(1, 0, 5), dim(50, 0, 400)), transfer }); + const r = await releaseAiSurplus(t, { subWorkspaceId: CHILD, poll }, log); + // Best-effort: the ErrorWithStatusCode(503) is swallowed and reported as released:false. + expect(r).to.deep.equal({ released: false, reason: 'error' }); + expect(transfer).to.have.callCount(4); // 1 initial + NOT_READY_RETRIES(3) + }); + + it('release path maps a terminal pool-exhausted 422 to a swallowed best-effort failure', async () => { + // The release transfer can 422 "insufficient units" if the parent can't (yet) reabsorb the + // surplus; transferAndSettle maps it to orgPoolExhausted, which releaseAiSurplus swallows + // (best-effort — a reconciler converges it later). + const warn = sinon.spy(); + const transfer = sinon.stub().rejects(poolFull()); + const t = makeTransport({ child: resources(dim(1, 0, 5), dim(50, 0, 400)), transfer }); + const r = await releaseAiSurplus(t, { subWorkspaceId: CHILD, poll }, { ...log, warn }); + expect(r).to.deep.equal({ released: false, reason: 'error' }); + expect(warn).to.have.been.calledWithMatch('SERENITY_ALLOC org pool exhausted on transfer'); + }); + + it('release path maps a poll timeout to a swallowed best-effort failure', async () => { + const t = makeTransport({ child: resources(dim(1, 0, 5), dim(50, 0, 400)) }); + t.getWorkspaceStatus = sinon.stub().resolves({ status: 'not ready' }); + const r = await releaseAiSurplus(t, { + subWorkspaceId: CHILD, poll: { attempts: 2, intervalMs: 0, sleep: () => Promise.resolve() }, + }, log); + expect(r).to.deep.equal({ released: false, reason: 'error' }); + }); + + it('failFast: lowers total with ONE transfer and NO settle poll (synchronous request-path release)', async () => { + // projects at floor (used 5 / total 5, no surplus); prompts used dropped to 50 after a removal + // republish, total 400 → release lowers prompts to roundUp(50)=100, projects untouched. + const t = makeTransport({ child: resources(dim(5, 0, 5), dim(50, 0, 400)) }); + const r = await releaseAiSurplus(t, { subWorkspaceId: CHILD, failFast: true }, log); + expect(r.released).to.equal(true); + expect(t.transferWorkspaceResources).to.have.been.calledOnceWith(CHILD, ai(5, 100)); + expect(t.getWorkspaceStatus).to.not.have.been.called; // no settle poll on the fail-fast path + }); + + it('failFast: a transient "workspace not ready" is swallowed best-effort (503 not thrown)', async () => { + const transfer = sinon.stub().rejects(notReady()); + const t = makeTransport({ child: resources(dim(1, 0, 5), dim(50, 0, 400)), transfer }); + const r = await releaseAiSurplus(t, { subWorkspaceId: CHILD, failFast: true }, log); + expect(r).to.deep.equal({ released: false, reason: 'error' }); + expect(transfer).to.have.callCount(1); // one attempt, no retry loop + }); + + it('exports the default blocks', () => { + expect(DEFAULT_BLOCKS).to.deep.equal({ projects: 1, prompts: 100 }); + }); + + it('DEFAULT_POLL.sleep resolves against the real timer', async () => { + await DEFAULT_POLL.sleep(0); // exercises the real-clock sleep default + expect(DEFAULT_POLL.attempts).to.be.a('number'); + }); +}); diff --git a/test/support/serenity/rest-transport.test.js b/test/support/serenity/rest-transport.test.js index 0afba84733..526b457d61 100644 --- a/test/support/serenity/rest-transport.test.js +++ b/test/support/serenity/rest-transport.test.js @@ -983,6 +983,24 @@ describe('Semrush REST transport', () => { }); }); + describe('getWorkspaceResources', () => { + it('GETs /v1/workspaces/{ws}/resources', async () => { + fetchStub.resolves(fetchOk({ + product_resources: { ai: { resources: { projects: { used: 0, drafted: 0, total: 13 } } } }, + })); + const transport = createSerenityTransport({ env: TEST_ENV, imsToken: IMS }); + + const result = await transport.getWorkspaceResources(WORKSPACE_ID); + + const call = await callOf(fetchStub); + expect(call.method).to.equal('GET'); + expect(call.url).to.equal( + `https://adobe-hackathon.semrush.com/enterprise/users/api/v1/workspaces/${WORKSPACE_ID}/resources`, + ); + expect(result.product_resources.ai.resources.projects.total).to.equal(13); + }); + }); + describe('listWorkspaceFamily', () => { it('GETs /v1/workspaces/{parent}/family', async () => { fetchStub.resolves(fetchOk({ items: [{ id: 'subworkspace-ws-1', title: 'Adobe Express' }] })); diff --git a/test/support/serenity/workspace-lifecycle.test.js b/test/support/serenity/workspace-lifecycle.test.js index 0c80aaca53..f8b9c0198b 100644 --- a/test/support/serenity/workspace-lifecycle.test.js +++ b/test/support/serenity/workspace-lifecycle.test.js @@ -91,6 +91,48 @@ describe('workspace-lifecycle', () => { expect(brand.save).to.not.have.been.called; }); + it('dynamic-allocation ON: SKIPS the flat re-grant on an existing workspace (JIT owns sizing)', async () => { + const transport = makeTransport(); + const brand = makeBrand({ workspaceId: SUB_WS }); + + const result = await ensureSubworkspace( + transport, + brand, + PARENT_WS, + 2, + log, + NOOP_TIMING, + null, + { dynamicAllocation: true }, + ); + + expect(result).to.equal(SUB_WS); + // No flat re-grant transfer — the metered handlers top up just-in-time instead. + expect(transport.transferWorkspaceResources).to.not.have.been.called; + expect(transport.createSubworkspace).to.not.have.been.called; + // The readiness settle still runs (workspace must be `created` before return). + expect(transport.getWorkspaceStatus).to.have.been.calledOnceWith(SUB_WS); + }); + + it('dynamic-allocation OFF (default): re-grant is byte-for-byte unchanged', async () => { + const transport = makeTransport(); + const brand = makeBrand({ workspaceId: SUB_WS }); + + await ensureSubworkspace( + transport, + brand, + PARENT_WS, + 2, + log, + NOOP_TIMING, + null, + { dynamicAllocation: false }, + ); + + expect(transport.transferWorkspaceResources) + .to.have.been.calledOnceWithExactly(SUB_WS, resourceAllocation(2)); + }); + it('creates, polls until created, then persists the column', async () => { const transport = makeTransport(); transport.getWorkspaceStatus