diff --git a/.github/scripts/github-fixture-fetch.test.mjs b/.github/scripts/github-fixture-fetch.test.mjs new file mode 100644 index 0000000000..869afd9fad --- /dev/null +++ b/.github/scripts/github-fixture-fetch.test.mjs @@ -0,0 +1,155 @@ +import { test } from 'node:test'; +import assert from 'node:assert'; +import { createServer } from 'node:http'; +import { + discardBody, + githubHeaders, + peerRevision, + readGitHubText, +} from '../../integrationTests/resources/nested-relationship-docs/github-fixture-fetch.mjs'; + +async function withServer(handler, run) { + const server = createServer(handler); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + await run('http://127.0.0.1:' + server.address().port + '/fixture'); + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + } +} + +test('credentials are attached only to the exact GitHub API origin', () => { + const token = 'test-token-not-a-credential'; + assert.strictEqual( + githubHeaders('https://api.github.com/repos/HarperFast/harper', token).Authorization, + 'Bearer ' + token + ); + for (const url of [ + 'https://raw.githubusercontent.com/HarperFast/harper/main/file', + 'http://api.github.com/file', + 'https://api.github.com.example/file', + 'http://127.0.0.1/file', + ]) + assert.strictEqual(githubHeaders(url, token).Authorization, undefined); + assert.strictEqual(githubHeaders('https://api.github.com/repos/HarperFast/harper').Authorization, undefined); +}); + +test('discarding an already errored response body preserves HTTP failure handling', async () => { + const response = new Response( + new ReadableStream({ + start(controller) { + controller.error(new Error('discarded stream failed')); + }, + }), + { status: 503 } + ); + await assert.rejects(response.body.cancel(), /discarded stream failed/); + await assert.doesNotReject(discardBody(response)); +}); + +test('companion revisions fail closed for closed-unmerged or malformed peers', () => { + const sha = 'a'.repeat(40); + assert.strictEqual(peerRevision({ state: 'open', head: { sha } }), sha); + assert.strictEqual(peerRevision({ state: 'closed', merged_at: '2026-10-19T00:00:00Z' }), 'main'); + assert.throws(() => peerRevision({ state: 'closed', head: { sha } }), /closed without merging/); + assert.throws(() => peerRevision({ state: 'open', head: { sha: 'invalid' } }), /Invalid companion revision/); +}); + +test('transient HTTP failures retry and return the successful body without leaking a token', async () => { + for (const status of [408, 429, 500, 503]) { + let attempts = 0; + await withServer( + (request, response) => { + assert.strictEqual(request.headers.authorization, undefined); + response.writeHead(++attempts < 3 ? status : 200, { 'Retry-After': '0' }); + response.end(attempts < 3 ? 'temporary' : 'fixture bytes'); + }, + async (url) => { + assert.strictEqual(await readGitHubText(url, { token: 'not-for-this-origin' }), 'fixture bytes'); + assert.strictEqual(attempts, 3); + } + ); + } +}); + +test('network disconnects retry within the same request budget', async () => { + let attempts = 0; + await withServer( + (request, response) => { + if (++attempts < 3) request.socket.destroy(); + else response.end('recovered'); + }, + async (url) => { + assert.strictEqual(await readGitHubText(url), 'recovered'); + assert.strictEqual(attempts, 3); + } + ); +}); + +test('permanent HTTP failures do not retry or expose response bodies', async () => { + for (const status of [401, 403, 404]) { + let attempts = 0; + await withServer( + (_request, response) => { + attempts++; + response.writeHead(status, { 'Retry-After': '0' }); + response.end('sensitive-response'); + }, + async (url) => { + await assert.rejects(readGitHubText(url, { token: 'sensitive-token' }), (error) => { + assert(error.message.includes('HTTP ' + status)); + assert(error.message.includes(url)); + assert(!/sensitive|Bearer|Authorization/.test(error.message)); + return true; + }); + assert.strictEqual(attempts, 1); + } + ); + } +}); + +test('transient failures stop after three attempts', async () => { + let attempts = 0; + await withServer( + (_request, response) => { + attempts++; + response.writeHead(503, { 'Retry-After': '0' }); + response.end('unavailable'); + }, + async (url) => { + await assert.rejects(readGitHubText(url), /HTTP 503 after 3 attempt/); + assert.strictEqual(attempts, 3); + } + ); +}); + +test('Retry-After beyond the comparison wait budget fails rather than retrying early', async () => { + let attempts = 0; + await withServer( + (_request, response) => { + attempts++; + response.writeHead(429, { 'Retry-After': '60' }); + response.end(); + }, + async (url) => { + await assert.rejects(readGitHubText(url), /longer Retry-After/); + assert.strictEqual(attempts, 1); + } + ); +}); + +test('the shared comparison signal terminates a stalled request without retries', async () => { + let attempts = 0; + const controller = new AbortController(); + await withServer( + () => { + attempts++; + controller.abort(); + }, + async (url) => { + await assert.rejects(readGitHubText(url, { signal: controller.signal }), /network error or timeout/); + assert.strictEqual(attempts, 1); + } + ); +}); diff --git a/.github/scripts/nested-relationship-fixture-parity.test.mjs b/.github/scripts/nested-relationship-fixture-parity.test.mjs new file mode 100644 index 0000000000..9af67bb531 --- /dev/null +++ b/.github/scripts/nested-relationship-fixture-parity.test.mjs @@ -0,0 +1,13 @@ +import { test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { verifyPeerFixtures } from '../../integrationTests/resources/nested-relationship-docs/github-fixture-fetch.mjs'; + +test('paired documentation fixtures stay synchronized across repositories', { timeout: 35_000 }, async () => { + await verifyPeerFixtures({ + localRoot: fileURLToPath(new URL('../../integrationTests/resources/nested-relationship-docs/', import.meta.url)), + peerRoot: process.env.HARPER_DOCS_FIXTURE_ROOT, + repository: 'HarperFast/documentation', + pull: 736, + peerPath: 'scripts/fixtures/nested-relationship-docs', + }); +}); diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 3398070e17..77bb71277b 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -33,6 +33,23 @@ env: # the `if: failure()` Harper-log upload still runs, while a job timeout CANCELS it and skips the # upload. jobs: + documentation-fixture-parity: + name: Documentation fixture parity + runs-on: ubuntu-latest + timeout-minutes: 3 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false + - name: Test fixture fetch reliability + run: node --test .github/scripts/github-fixture-fetch.test.mjs + - name: Compare companion fixtures + env: + GH_TOKEN: ${{ github.token }} + run: node --test .github/scripts/nested-relationship-fixture-parity.test.mjs + generate-node-version-matrix: name: Generate Node Version Matrix runs-on: ubuntu-latest diff --git a/integrationTests/resources/nested-relationship-docs.test.ts b/integrationTests/resources/nested-relationship-docs.test.ts new file mode 100644 index 0000000000..d08237e292 --- /dev/null +++ b/integrationTests/resources/nested-relationship-docs.test.ts @@ -0,0 +1,132 @@ +/** + * Executes the request/response examples in the nested-relationship documentation companion: + * https://github.com/HarperFast/documentation/pull/736. + * The offline fixtures in that repo are checked against the published Markdown by docs CI. + */ +import { suite, test, before, after } from 'node:test'; +import assert from 'node:assert'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { + setupHarperWithFixture, + teardownHarper, + sendOperation, + DEFAULT_ADMIN_USERNAME, + DEFAULT_ADMIN_PASSWORD, + type ContextWithHarper, +} from '@harperfast/integration-testing'; + +const fixture = resolve(import.meta.dirname, 'nested-relationship-docs'); +const examples = JSON.parse(readFileSync(resolve(fixture, 'examples.json'), 'utf8')); +suite('documented nested relationship requests and responses', { timeout: 120_000 }, (ctx: ContextWithHarper) => { + const auth = 'Basic ' + Buffer.from(DEFAULT_ADMIN_USERNAME + ':' + DEFAULT_ADMIN_PASSWORD).toString('base64'); + async function get(path: string) { + const response = await fetch(ctx.harper.httpURL + path, { + headers: { Authorization: auth }, + signal: AbortSignal.timeout(10_000), + }); + const body = await response.json(); + assert.strictEqual(response.status, 200, JSON.stringify(body)); + return body; + } + async function insert(table: string, records: unknown[]) { + await sendOperation(ctx.harper, { operation: 'insert', schema: 'data', table, records }); + } + before(async () => { + await setupHarperWithFixture(ctx, fixture, { + config: { threads: { count: 1 } }, + env: { HARPER_STORAGE_ENGINE: process.env.HARPER_STORAGE_ENGINE || 'rocksdb' }, + }); + const deadline = Date.now() + 30_000; + let ready = false; + while (Date.now() < deadline) { + try { + await get('/Brand/'); + ready = true; + break; + } catch { + await delay(50); + } + } + assert(ready, 'Brand route did not become ready'); + await insert('Brand', examples.brands); + await insert('Product', examples.products); + }); + after(async () => { + await teardownHarper(ctx); + }); + for (const example of examples.cases) + test(example.key, async () => { + assert.deepStrictEqual(await get(example.url), example.response); + }); + test('the outer limit uses an exclusive end with three eligible parents', async () => { + assert.deepStrictEqual(await get('/Brand/?sort(id)&limit(1,2)&select(id,)'), [{ id: 43 }]); + }); + test('grouped OR preserves both branches before sorting and pagination', async () => { + const example = examples.cases.find((example: { key: string }) => example.key === 'grouped_or_page'); + assert(example, 'grouped OR documentation example is missing'); + const unskipped = await get(example.url.replace('limit(1,2)', 'limit(0,3)')); + assert.deepStrictEqual( + unskipped.map((brand: { products: { id: string }[] }) => brand.products.map((product) => product.id)), + [['p3', 'p1'], ['p4'], ['p6']] + ); + const intersection = await get(example.url.replace('|', '&').replace('limit(1,2)', 'limit(0,3)')); + assert.deepStrictEqual( + intersection.map((brand: { products: unknown[] }) => brand.products), + [[], [], []] + ); + }); + test('both exact programmatic snippets produce their documented responses', async () => { + assert.deepStrictEqual(await get('/DocumentedQueries/'), { + single: examples.single, + programmatic: examples.programmatic, + }); + }); + test('repeating the inherited predicate does not change the response', async () => { + const inherited = examples.cases.find((example: { key: string }) => example.key === 'outer_inherited'); + assert.deepStrictEqual( + await get(inherited.url.replace('products[', 'products[price=ge=number:30&')), + inherited.response + ); + }); + test('the outer page bounds parents independently of child limits', async () => { + const bounded = examples.cases.find((example: { key: string }) => example.key === 'bounded_parent_page'); + assert.deepStrictEqual(await get(bounded.url.replace('limit(0,20)', 'limit(0,1)')), [bounded.response[0]]); + }); + test('nested sorting permits an unindexed child field', async () => { + const rows = await get('/Brand/?select(id,products[sort(name)&limit(1)&select(id,)])'); + assert.deepStrictEqual( + rows.map((row: { products: { id: string }[] }) => row.products), + [[{ id: 'p1' }], [{ id: 'p4' }], [{ id: 'p6' }]] + ); + }); + + test('the unique child sort key breaks equal-price ties', async () => { + try { + await insert('Product', [{ id: 'p0', brandId: 42, name: 'Adapter', price: 35 }]); + const bounded = examples.cases.find((example: { key: string }) => example.key === 'bounded_parent_page'); + const rows = await get(bounded.url); + assert.deepStrictEqual(rows[0].products, [{ id: 'p0', name: 'Adapter', price: 35 }]); + const descending = await get(bounded.url.replace('sort(price,id)', 'sort(price,-id)')); + assert.deepStrictEqual(descending[0].products, [{ id: 'p2', name: 'Keyboard', price: 35 }]); + const window = await get(bounded.url.replace('limit(1)', 'limit(1,2)')); + assert.deepStrictEqual(window[0].products, [{ id: 'p2', name: 'Keyboard', price: 35 }]); + } finally { + await sendOperation(ctx.harper, { operation: 'delete', schema: 'data', table: 'Product', ids: ['p0'] }); + } + }); + test('the unique parent sort key breaks equal-name ties', async () => { + try { + await insert('Brand', [{ id: 45, name: 'Acme' }]); + await insert('Product', [{ id: 'p7', brandId: 45, name: 'Hub', price: 50 }]); + const bounded = examples.cases.find((example: { key: string }) => example.key === 'bounded_parent_page'); + const page = bounded.url.replace('limit(0,20)', 'limit(0,1)'); + assert.strictEqual((await get(page))[0].id, 42); + assert.strictEqual((await get(page.replace('sort(name,id)', 'sort(name,-id)')))[0].id, 45); + } finally { + await sendOperation(ctx.harper, { operation: 'delete', schema: 'data', table: 'Product', ids: ['p7'] }); + await sendOperation(ctx.harper, { operation: 'delete', schema: 'data', table: 'Brand', ids: [45] }); + } + }); +}); diff --git a/integrationTests/resources/nested-relationship-docs/config.yaml b/integrationTests/resources/nested-relationship-docs/config.yaml new file mode 100644 index 0000000000..efffc0833f --- /dev/null +++ b/integrationTests/resources/nested-relationship-docs/config.yaml @@ -0,0 +1,5 @@ +graphqlSchema: + files: '*.graphql' +jsResource: + files: resources.js +rest: true diff --git a/integrationTests/resources/nested-relationship-docs/examples.json b/integrationTests/resources/nested-relationship-docs/examples.json new file mode 100644 index 0000000000..fb9ccd00e9 --- /dev/null +++ b/integrationTests/resources/nested-relationship-docs/examples.json @@ -0,0 +1,255 @@ +{ + "brands": [ + { + "id": 42, + "name": "Acme" + }, + { + "id": 43, + "name": "Budget Basics" + }, + { + "id": 44, + "name": "Studio Gear" + } + ], + "products": [ + { + "id": "p1", + "brandId": 42, + "name": "Cable", + "price": 10 + }, + { + "id": "p2", + "brandId": 42, + "name": "Keyboard", + "price": 35 + }, + { + "id": "p3", + "brandId": 42, + "name": "Dock", + "price": 60 + }, + { + "id": "p4", + "brandId": 43, + "name": "Mouse", + "price": 15 + }, + { + "id": "p5", + "brandId": 44, + "name": "Stand", + "price": 45 + }, + { + "id": "p6", + "brandId": 44, + "name": "Monitor", + "price": 120 + } + ], + "programmatic": [ + { + "id": 42, + "name": "Acme", + "products": [ + { + "id": "p2", + "name": "Keyboard", + "price": 35 + } + ] + }, + { + "id": 44, + "name": "Studio Gear", + "products": [ + { + "id": "p5", + "name": "Stand", + "price": 45 + } + ] + } + ], + "single": { + "id": 42, + "products": [ + { + "id": "p2", + "price": 35 + } + ] + }, + "cases": [ + { + "key": "nested_only", + "url": "/Brand/?select(id,name,products[price=ge=number:30&sort(price)&limit(1)&select(id,name,price)])", + "response": [ + { + "id": 42, + "name": "Acme", + "products": [ + { + "id": "p2", + "name": "Keyboard", + "price": 35 + } + ] + }, + { + "id": 43, + "name": "Budget Basics", + "products": [] + }, + { + "id": 44, + "name": "Studio Gear", + "products": [ + { + "id": "p5", + "name": "Stand", + "price": 45 + } + ] + } + ] + }, + { + "key": "outer_inherited", + "url": "/Brand/?products.price=ge=number:30&select(id,name,products[sort(price)&limit(1)&select(id,name,price)])", + "response": [ + { + "id": 42, + "name": "Acme", + "products": [ + { + "id": "p2", + "name": "Keyboard", + "price": 35 + } + ] + }, + { + "id": 44, + "name": "Studio Gear", + "products": [ + { + "id": "p5", + "name": "Stand", + "price": 45 + } + ] + } + ] + }, + { + "key": "additional_nested", + "url": "/Brand/?products.price=ge=number:30&select(id,name,products[price=ge=number:100&sort(price)&limit(1)&select(id,name,price)])", + "response": [ + { + "id": 42, + "name": "Acme", + "products": [] + }, + { + "id": 44, + "name": "Studio Gear", + "products": [ + { + "id": "p6", + "name": "Monitor", + "price": 120 + } + ] + } + ] + }, + { + "key": "per_parent_offset", + "url": "/Brand/?products.price=ge=number:30&select(id,name,products[sort(price)&limit(1,2)&select(id,name,price)])", + "response": [ + { + "id": 42, + "name": "Acme", + "products": [ + { + "id": "p3", + "name": "Dock", + "price": 60 + } + ] + }, + { + "id": 44, + "name": "Studio Gear", + "products": [ + { + "id": "p6", + "name": "Monitor", + "price": 120 + } + ] + } + ] + }, + { + "key": "bounded_parent_page", + "url": "/Brand/?products.price=ge=number:30&sort(name,id)&limit(0,20)&select(id,name,products[sort(price,id)&limit(1)&select(id,name,price)])", + "response": [ + { + "id": 42, + "name": "Acme", + "products": [ + { + "id": "p2", + "name": "Keyboard", + "price": 35 + } + ] + }, + { + "id": 44, + "name": "Studio Gear", + "products": [ + { + "id": "p5", + "name": "Stand", + "price": 45 + } + ] + } + ] + }, + { + "key": "grouped_or_page", + "url": "/Brand/?sort(name,id)&limit(0,20)&select(id,name,products[[price=lt=number:30|price=ge=number:60]&sort(-price,id)&limit(1,2)&select(id,name,price)])", + "response": [ + { + "id": 42, + "name": "Acme", + "products": [ + { + "id": "p1", + "name": "Cable", + "price": 10 + } + ] + }, + { + "id": 43, + "name": "Budget Basics", + "products": [] + }, + { + "id": 44, + "name": "Studio Gear", + "products": [] + } + ] + } + ] +} diff --git a/integrationTests/resources/nested-relationship-docs/github-fixture-fetch.mjs b/integrationTests/resources/nested-relationship-docs/github-fixture-fetch.mjs new file mode 100644 index 0000000000..81ada7ce9e --- /dev/null +++ b/integrationTests/resources/nested-relationship-docs/github-fixture-fetch.mjs @@ -0,0 +1,105 @@ +import assert from 'node:assert'; +import { readFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; + +export function githubHeaders(url, token) { + if (new URL(url).origin !== 'https://api.github.com') return {}; + return { + Accept: 'application/vnd.github+json', + ...(token ? { Authorization: 'Bearer ' + token } : {}), + }; +} + +export async function discardBody(response) { + try { + await response.body?.cancel(); + } catch { + // An errored discarded body must not replace the HTTP failure or suppress retries. + } +} + +export async function readGitHubText(url, { token, signal, timeout = 15_000 } = {}) { + for (let attempt = 1; attempt <= 3; attempt++) { + let response; + try { + response = await fetch(url, { + headers: githubHeaders(url, token), + signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(timeout)]) : AbortSignal.timeout(timeout), + }); + if (response.status === 200) return await response.text(); + } catch { + response = undefined; + if (signal?.aborted || attempt === 3) + throw new Error( + 'Cannot read fixture URL ' + url + ' after ' + attempt + ' attempt(s): network error or timeout' + ); + } + let wait = 250 * 2 ** (attempt - 1); + if (response) { + const retryable = response.status === 408 || response.status === 429 || response.status >= 500; + await discardBody(response); + if (!retryable || attempt === 3) + throw new Error( + 'Cannot read fixture URL ' + url + ': HTTP ' + response.status + ' after ' + attempt + ' attempt(s)' + ); + const retryAfter = response.headers.get('retry-after'); + if (retryAfter !== null) { + const seconds = Number(retryAfter); + const requested = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(retryAfter) - Date.now(); + if (Number.isFinite(requested)) wait = Math.max(0, requested); + } + if (wait > 2000) + throw new Error( + 'Cannot read fixture URL ' + url + ': HTTP ' + response.status + ' requires a longer Retry-After' + ); + } + try { + await delay(wait, undefined, { signal }); + } catch { + throw new Error('Fixture comparison deadline exceeded while reading ' + url); + } + } +} + +export function peerRevision(pr) { + assert(pr.merged_at || pr.state === 'open', 'Companion PR closed without merging; update the fixture pairing'); + const revision = pr.merged_at ? 'main' : pr.head?.sha; + assert(revision === 'main' || /^[a-f0-9]{40}$/.test(revision), 'Invalid companion revision'); + return revision; +} + +export async function verifyPeerFixtures({ + localRoot, + peerRoot, + repository, + pull, + peerPath, + token = process.env.GH_TOKEN || process.env.GITHUB_TOKEN, +}) { + const signal = AbortSignal.timeout(30_000); + let revision; + if (!peerRoot) { + assert(!process.env.CI || token, 'Fixture parity in CI requires GH_TOKEN or GITHUB_TOKEN'); + const metadata = await readGitHubText('https://api.github.com/repos/' + repository + '/pulls/' + pull, { + token, + signal, + }); + const pr = JSON.parse(metadata); + revision = peerRevision(pr); + } + for (const name of ['examples.json', 'schema.graphql', 'resources.js', 'github-fixture-fetch.mjs']) { + const expected = await readFile(resolve(localRoot, name), 'utf8'); + const actual = peerRoot + ? await readFile(resolve(peerRoot, name), 'utf8') + : await readGitHubText( + 'https://raw.githubusercontent.com/' + repository + '/' + revision + '/' + peerPath + '/' + name, + { signal } + ); + assert.strictEqual( + actual.replace(/\r\n/g, '\n'), + expected.replace(/\r\n/g, '\n'), + name + ' differs from the companion repository; update both fixtures together' + ); + } +} diff --git a/integrationTests/resources/nested-relationship-docs/resources.js b/integrationTests/resources/nested-relationship-docs/resources.js new file mode 100644 index 0000000000..ac146ed697 --- /dev/null +++ b/integrationTests/resources/nested-relationship-docs/resources.js @@ -0,0 +1,32 @@ +export class DocumentedQueries extends Resource { + static loadAsInstance = false; + async get() { + const Brand = tables.Brand; + const brand = await Brand.get({ + id: 42, + select: [ + 'id', + { + name: 'products', + conditions: [{ attribute: 'price', comparator: 'greater_than_equal', value: 30 }], + sort: { attribute: 'price' }, + offset: 0, + limit: 1, + select: ['id', 'price'], + }, + ], + }); + const brands = []; + for await (const brand of Brand.search({ + conditions: [{ attribute: ['products', 'price'], comparator: 'greater_than_equal', value: 30 }], + select: [ + 'id', + 'name', + { name: 'products', sort: { attribute: 'price' }, limit: 1, select: ['id', 'name', 'price'] }, + ], + })) { + brands.push(brand); + } + return { single: brand, programmatic: brands }; + } +} diff --git a/integrationTests/resources/nested-relationship-docs/schema.graphql b/integrationTests/resources/nested-relationship-docs/schema.graphql new file mode 100644 index 0000000000..484e57d8d2 --- /dev/null +++ b/integrationTests/resources/nested-relationship-docs/schema.graphql @@ -0,0 +1,12 @@ +type Brand @table @export { + id: Int @primaryKey + name: String @indexed + products: [Product] @relationship(to: "brandId") +} + +type Product @table @export { + id: ID @primaryKey + brandId: Int @indexed + name: String + price: Int @indexed +} diff --git a/integrationTests/resources/relationship-and-status-contract.test.ts b/integrationTests/resources/relationship-and-status-contract.test.ts index d32af6f07c..b27e785acb 100644 --- a/integrationTests/resources/relationship-and-status-contract.test.ts +++ b/integrationTests/resources/relationship-and-status-contract.test.ts @@ -135,6 +135,39 @@ suite( await teardownHarper(ctx); }); + test('nested relationship predicates and pagination use the native authenticated REST path', async () => { + await clearAll(); + for (const [table, records] of [ + ['Order', [{ id: 'nested-p1' }, { id: 'nested-p2' }]], + [ + 'OrderItem', + [ + { id: 'nested-c1', orderId: 'nested-p1', price: 10 }, + { id: 'nested-c2', orderId: 'nested-p1', price: 30 }, + { id: 'nested-c3', orderId: 'nested-p1', price: 40 }, + { id: 'nested-c4', orderId: 'nested-p2', price: 50 }, + ], + ], + ] as const) { + const seeded = await op({ operation: 'insert', schema: 'data', table, records }); + strictEqual(seeded.status, 200, JSON.stringify(seeded.body)); + } + try { + const selection = 'select(id,items[price=ge=number:30&sort(price)&limit(1)&select(id,price)])'; + const collection = await restGet('/Order/?' + selection); + strictEqual(collection.status, 200); + deepStrictEqual( + collection.body.map((row: any) => row.items.map((item: any) => item.id)), + [['nested-c2'], ['nested-c4']] + ); + const single = await restGet('/Order/nested-p1?' + selection); + strictEqual(single.status, 200); + deepStrictEqual(single.body.items, [{ id: 'nested-c2', price: 30 }]); + } finally { + await clearAll(); + } + }); + // ========================================================================================= // QA-196 — @relationship parent-DELETE cascade semantics // ========================================================================================= diff --git a/resources/DESIGN.md b/resources/DESIGN.md index 84dd64004b..bd3ea55b0d 100644 --- a/resources/DESIGN.md +++ b/resources/DESIGN.md @@ -1,9 +1,5 @@ # resources/ — Navigation Guide -The Resource layer is Harper's universal abstraction: all queryable/mutable things (tables, caches, message topics, custom endpoints) extend `Resource`. Inbound protocols (REST, GraphQL, MQTT, NATS, WebSockets) all converge on this interface. - -**Read this when:** you're touching the read/write path, authorization, subscriptions, or table CRUD semantics. - > **Navigation convention.** This guide references code by **symbol name** (e.g. `_writeUpdate`) and by **section marker** (e.g. `// #section: write-path-internals`). Jump in your editor via go-to-symbol, or `grep` for the section marker. Line numbers drift; symbols and section markers don't. --- @@ -247,7 +243,9 @@ A contract is `{ path, record?, get/post/put/patch/delete: { query?, body?, resp ## Conventions -- Single-record nested selections use `Resource.transformForSelect`; searches use `Table.transformEntryForSelect`. Their contracts for missing targets, `forceNulls`, Dates, callable values, resolver dictionaries, `Resource.transformForSelect`'s `WeakMap` sub-transform cache (keyed by the selector object so no request-supplied name becomes a key), and a selected `__proto__` are pinned by `../unitTests/resources/nestedSelectMissingValues.test.js` and `../integrationTests/resources/relationship-and-status-contract.test.ts`. The `select` branch in `parseBlock` (`search.ts`) unwraps a single-argument `select(x)` only when that argument is not a named sub-select, keeping a single nested selector (`select(rel{a,b})` or `select(rel[select(a,b)])`) wrapped in its outer array and rejecting bracketed filter conditions inside `select(...)` with a 400 `SyntaxViolation` (pinned by `../unitTests/resources/query-parse.test.js` and `../unitTests/resources/restSelectProjectionFixes.test.js`); [harper#1415](https://github.com/HarperFast/harper/issues/1415) remains open for single-attribute `select(attr)` returning 404 when the selected scalar value is null or missing. Still open, outside these helpers: [asynchronous arrays and `forceNulls`](https://github.com/HarperFast/harper/issues/3168) and [a `__proto__` key poisoning a table, and proto-named attributes rejected at insert](https://github.com/HarperFast/harper/issues/1295). +- Nested relationship `conditions`, `sort`, `offset`, and `limit` share `Table._prepareRelationshipSelection` across search and single-record GET. Predicates keep parent inclusion and inherited relationship candidates; only unsorted eligible pagination bounds reads (sorting or unmatched predicates can scan every candidate). Nested predicates qualify only the immediate selected children; descendant narrowing requires its own subselect, while outer-join candidate restrictions remain inherited. Predicate compilation must not mutate projection maps. Candidates and sort keys retain top-level cache-local semantics; predicates are rechecked after source refresh before offset/limit counts a returned child. Cancellation must terminate the raw source before an upstream mapper reads again. Lazy resolver mode is per-call so ordinary property access stays synchronous. Unsorted FK pagination preserves missing slots; sorting drops them. `unitTests/resources/nestedRelationshipConstraints.test.js` pins authorization, iterator cleanup, missing-value behavior, and bounded entry reads. + +- Single-record nested selections use `Resource.transformForSelect`; searches use `Table.transformEntryForSelect`. Their contracts for missing targets, `forceNulls`, Dates, callable values, resolver dictionaries, `Resource.transformForSelect`'s `WeakMap` sub-transform cache (keyed by the selector object so no request-supplied name becomes a key), and a selected `__proto__` are pinned by `../unitTests/resources/nestedSelectMissingValues.test.js` and `../integrationTests/resources/relationship-and-status-contract.test.ts`. The `select` branch in `parseBlock` (`search.ts`) unwraps a single-argument `select(x)` only when that argument is not a named sub-select, keeping a single nested selector (`select(rel{a,b})` or `select(rel[select(a,b)])`) wrapped in its outer array and preserving named bracket queries for table-relationship filtering while bare comparisons in selector lists remain syntax errors (pinned by `../unitTests/resources/query-parse.test.js` and `../unitTests/resources/restSelectProjectionFixes.test.js`); [harper#1415](https://github.com/HarperFast/harper/issues/1415) remains open for single-attribute `select(attr)` returning 404 when the selected scalar value is null or missing. Still open, outside these helpers: [asynchronous arrays and `forceNulls`](https://github.com/HarperFast/harper/issues/3168) and [a `__proto__` key poisoning a table, and proto-named attributes rejected at insert](https://github.com/HarperFast/harper/issues/1295). - A record's `toJSON` is a response projection, not durable state. `recordUpdater` (and the cache-fill and audit paths feeding it) projects records to stored fields via `storedFieldsOnly`, so no durable form carries a resolver-owned (`@computed`/`@relationship`) name; materialization skips such names when promoting a decoded record (`assignStoredFields`), which is what keeps rows written by affected releases readable. See harper#2359 and `integrationTests/resources/cachedComputedAttribute.test.ts`. - **Never** remove `transactional()` from a static method on `Resource` — it owns transaction context lifetime. - New `Resource` subclasses should override **instance** methods (`get`, `put`, ...) for behavior; static methods are the protocol entry points and stay generic. diff --git a/resources/Resource.ts b/resources/Resource.ts index 06fea6bb2d..ce5ec54561 100644 --- a/resources/Resource.ts +++ b/resources/Resource.ts @@ -130,7 +130,10 @@ export class Resource implements ResourceInterface< static get = transactional( function (resource: any, query: RequestTarget, request: Context, _data: any) { const loadAsInstance = resource.constructor.loadAsInstance; - if (loadAsInstance && query?.select && (query as any).parseError) throw (query as any).parseError; + if (loadAsInstance && query?.select) { + if ((query as any).parseError) throw (query as any).parseError; + assertNestedQueryOptions(query.select, resource.constructor); + } const result = resource.get?.(query); // only false mode hands allowRead to the instance search; other modes settled it before dispatch if (loadAsInstance === false) { @@ -1181,16 +1184,57 @@ function selectFromObject(object, propertyResolvers, context) { // TODO: eventually we will do aggregate functions here return (property) => readPropertyFromObject(object, property, propertyResolvers, context); } +export function hasNestedQueryOptions(property) { + return ( + property && + typeof property === 'object' && + ((property.conditions !== undefined && (!Array.isArray(property.conditions) || property.conditions.length > 0)) || + property.operator !== undefined || + property.limit !== undefined || + property.offset !== undefined || + (typeof property.sort === 'object' && property.sort)) + ); +} + +export function assertNestedQueryOptions(select, resource) { + for (const property of Array.isArray(select) ? select : []) { + if (!property || typeof property !== 'object') continue; + const related = resource?.propertyResolvers?.[property.name]?.definition?.tableClass; + if ( + ((property.conditions !== undefined && (!Array.isArray(property.conditions) || property.conditions.length > 0)) || + property.operator !== undefined) && + (!resource?._prepareRelationshipSelection || !related) + ) + throw new ClientError('Nested query constraints require a relationship attribute'); + assertNestedQueryOptions(property.select || (Array.isArray(property) ? property : undefined), related); + } +} + +function resolveSelectedArray(values) { + return values.some((value) => typeof value?.then === 'function') ? Promise.all(values) : values; +} + export function transformForSelect(select, resource) { + assertNestedQueryOptions(select, resource); const propertyResolvers = resource?.propertyResolvers; const context = resource?.getContext?.(); let subTransforms; + let relationshipSelections; + if (resource?._prepareRelationshipSelection) + for (const property of Array.isArray(select) ? select : []) { + if (propertyResolvers?.[property.name]?.definition?.tableClass && hasNestedQueryOptions(property)) { + (relationshipSelections ||= Object.create(null))[property.name] = resource._prepareRelationshipSelection( + property, + context ?? contextStorage.getStore() + ); + } + } if (typeof select === 'string') // if select is a single string then return property value return function transform(object) { if (object == null || (!resource && typeof object !== 'object')) return object; if (typeof object.then === 'function') return object.then(transform); - if (Array.isArray(object)) return object.map(transform); + if (Array.isArray(object)) return resolveSelectedArray(object.map(transform)); return readPropertyFromObject(object, select, propertyResolvers, context); }; else if (typeof select === 'object') { @@ -1199,13 +1243,23 @@ export function transformForSelect(select, resource) { return function transform(object) { if (object == null || (!resource && typeof object !== 'object')) return object; if (typeof object.then === 'function') return object.then(transform); - if (Array.isArray(object)) return object.map(transform); + if (Array.isArray(object)) return resolveSelectedArray(object.map(transform)); const results = []; - const getProperty = handleProperty(selectFromObject(object, propertyResolvers, context)); + let promises; + const getProperty = handleProperty(selectFromObject(object, propertyResolvers, context), object); for (const property of select) { - results.push(getProperty(property)); + const index = results.length; + const value = getProperty(property); + results.push(value); + if (typeof value?.then === 'function') { + (promises ||= []).push( + value.then((value) => { + results[index] = value; + }) + ); + } } - return results; + return promises ? Promise.all(promises).then(() => results) : results; }; const forceNulls = select.forceNulls; const selectsPrototype = select.some((property) => (property.name || property) === '__proto__'); @@ -1213,10 +1267,12 @@ export function transformForSelect(select, resource) { if (object == null || (!resource && typeof object !== 'object')) return object; if (typeof object.then === 'function') return object.then(transform); if (Array.isArray(object)) - return object.map((value) => (value && typeof value === 'object' ? transform(value) : value)); + return resolveSelectedArray( + object.map((value) => (value && typeof value === 'object' ? transform(value) : value)) + ); // finally the case of returning objects const selectedData = {}; - const getProperty = handleProperty(selectFromObject(object, propertyResolvers, context)); + const getProperty = handleProperty(selectFromObject(object, propertyResolvers, context), object); if (selectsPrototype) Object.defineProperty(selectedData, '__proto__', { value: undefined, @@ -1243,13 +1299,16 @@ export function transformForSelect(select, resource) { return selectedData; }; } else throw new Error('Invalid select argument type ' + typeof select); - function handleProperty(getProperty) { + function handleProperty(getProperty, object) { return (property) => { if (typeof property === 'string') { return getProperty(property); } else if (typeof property === 'object') { // TODO: Handle aggregate functions if (property.name) { + const selection = relationshipSelections?.[property.name]; + if (selection) + return selection(typeof object.getRecord === 'function' ? object.getRecord() || object : object); const value = getProperty(property.name); if (!value || typeof value !== 'object' || value instanceof Date) return value; if (!subTransforms) subTransforms = new WeakMap(); diff --git a/resources/ResourceInterface.ts b/resources/ResourceInterface.ts index 7e178a61c0..f908f89f84 100644 --- a/resources/ResourceInterface.ts +++ b/resources/ResourceInterface.ts @@ -257,8 +257,13 @@ export interface Sort { next?: Sort; } export interface SubSelect { + conditions?: Conditions; + operator?: string; + limit?: number; + offset?: number; + sort?: Sort; name: string; - select: (string | SubSelect)[]; + select?: (string | SubSelect)[]; } export type Select = (string | SubSelect)[]; diff --git a/resources/Table.ts b/resources/Table.ts index fdf69a2df0..d45344f05a 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -40,6 +40,7 @@ import { convertToComparableKeys, } from './Resource.ts'; import { settleBeforeDeadline, when, promiseNormalize } from '../utility/when.ts'; +import { assertNestedQueryOptions, hasNestedQueryOptions } from './Resource.ts'; import { DatabaseTransaction, ImmediateTransaction, @@ -142,6 +143,7 @@ import { COERCIBLE_OPERATORS, executeConditions, resolveComparator, + filterByType, fullTextComparatorMode, ALTERNATE_COMPARATOR_NAMES, } from './search.ts'; @@ -1055,6 +1057,23 @@ interface TableResourceClass { */ lock(target?: RequestTargetOrId | RecordLockOptions, options?: RecordLockOptions, context?: any): Promise; operation(operation: any, context: any): any; + /** @internal */ + _prepareRelationshipSelection( + attribute: any, + context: any, + filtered?: any, + ensureLoaded?: boolean + ): (record: any, entry?: any, isSync?: boolean) => any; + /** @internal */ + _prepareQueryConditions( + conditions: any, + operator: any, + target: any, + context: any, + nested?: boolean + ): { conditions: any; includeFullTextHighlights: boolean }; + /** @internal */ + _assertNestedReadAccess(name: any, context: any, target: any): void; /** * This is responsible for ordering and select()ing the attributes/properties from returned entries */ @@ -1199,6 +1218,7 @@ interface TableResourceClass { cleanup(): void; closeMaintenance(deadline?: number): Promise; resumeMaintenance(): void; + /** @internal */ _readTxnForContext(context: any): (LMDBReadTransaction | RocksTransaction) & { openTimer?: number; retryRisk?: number; @@ -1311,6 +1331,20 @@ function cloneConditions(conditions: any[]): any[] { }); } +function copySelection(select) { + if (Array.isArray(select)) { + const copy = Object.assign([], select); + for (let index = 0; index < copy.length; index++) copy[index] = copySelection(copy[index]); + return copy; + } + if (!select || typeof select !== 'object') return select; + const copy = { ...select }; + if (copy.select) copy.select = copySelection(copy.select); + if (copy.sort && typeof copy.sort === 'object') copy.sort = copySelection(copy.sort); + if (copy.next) copy.next = copySelection(copy.next); + return copy; +} + function selectRequestsProperty(select: any, propertyName: string): boolean { if (!select) return false; const selected = Array.isArray(select) ? select : [select]; @@ -3734,6 +3768,7 @@ export function makeTable(options): TableResourceClass { get(target?: any): any { const constructor: any = this.constructor; if (target?.parseError) throw target.parseError; + assertNestedQueryOptions(target?.select, constructor); if (fullTextFieldNames || hasRelationships) assertFullTextSelection(target?.select); if (fullTextFieldNames) { assertRecordField( @@ -3915,13 +3950,7 @@ export function makeTable(options): TableResourceClass { if (cur.length === 0) return []; } } - if ( - !property.select && - !property.sort && - property.limit === undefined && - property.offset === undefined - ) - return [property.name]; + if (!property.select && !hasNestedQueryOptions(property)) return [property.name]; } return [property]; } @@ -6260,6 +6289,7 @@ export function makeTable(options): TableResourceClass { const txn = txnForContext(context); if (!target) throw new Error('No query provided'); if (target.parseError) throw target.parseError; // if there was a parse error, we can throw it now + assertNestedQueryOptions(target.select, this.constructor); if (fullTextFieldNames || hasRelationships) assertFullTextSelection(target.select); const getColumns = () => { const select = target.select; @@ -6409,136 +6439,8 @@ export function makeTable(options): TableResourceClass { conditions = cloneConditions(conditions); let orderAlignedCondition; let syntheticOrderCondition; - let includeFullTextHighlights = false; const filtered = {}; - function prepareConditions(conditions: any[], operator: string) { - // some validation: - switch (operator) { - case 'and': - case undefined: - if (conditions.length < 1) throw new Error('An "and" operator requires at least one condition'); - break; - case 'or': - if (conditions.length < 2) throw new Error('An "or" operator requires at least two conditions'); - break; - default: - throw new Error('Invalid operator ' + operator); - } - for (const condition of conditions) { - if (condition.conditions) { - condition.conditions = prepareConditions(condition.conditions, condition.operator); - continue; - } - // Normalize `not_X` comparator forms passed in via structured queries. - // The REST parser already does this, but programmatic callers may - // pass `not_in`, `not_starts_with`, etc. directly. - if (condition.comparator) { - const resolved = resolveComparator(condition.comparator); - if (resolved.negated) { - condition.comparator = resolved.comparator; - condition.negated = true; - } - } - const attribute_name = condition[0] ?? condition.attribute; - const fullTextMode = - TableResource.fullTextIndexes.length > 0 || - Array.isArray(attribute_name) || - (typeof attribute_name === 'string' && fullTextFieldNames?.has(attribute_name)) - ? fullTextComparatorMode(condition.comparator) - : undefined; - if (!fullTextMode && (fullTextFieldNames || hasRelationships)) assertFullTextRecordField(attribute_name); - const fullTextDefinition = - fullTextMode && typeof attribute_name === 'string' - ? TableResource.fullTextIndexes.find((definition) => definition.name === attribute_name) - : undefined; - if (fullTextMode && Array.isArray(attribute_name)) - throw new ClientError('Full-text predicates must directly name an index on the queried table', 400); - if (fullTextMode && !fullTextDefinition) throwUnknownFullTextIndex(context, target, attribute_name); - if (fullTextDefinition) { - const fields = condition.fields; - if (fields !== undefined) { - if (!Array.isArray(fields) || fields.length === 0 || fields.some((field) => typeof field !== 'string')) - throw new ClientError(`Full-text index '${attribute_name}' requires a non-empty fields list`, 400); - assertFullTextReadAccess(context, target, fullTextDefinition, fields); - } else assertFullTextReadAccess(context, target, fullTextDefinition); - const value = condition[1] ?? condition.value; - if (typeof value !== 'string' || value.length === 0) - throw new ClientError( - `Full-text index '${attribute_name}' requires a full-text comparator and non-empty string value`, - 400 - ); - if (fullTextMode === 'phrase' && !fullTextDefinition.positions) - throw new ClientError(`Full-text index '${attribute_name}' does not store phrase positions`, 400); - if ((fullTextMode === 'prefix' || fullTextMode === 'fuzzy-prefix') && !fullTextDefinition.surfaceTerms) - throw new ClientError(`Full-text index '${attribute_name}' does not store surface terms`, 400); - if (fields !== undefined) { - const sourceNames = new Set(fullTextDefinition.fields.map(({ name }) => name)); - if (new Set(fields).size !== fields.length || fields.some((field) => !sourceNames.has(field))) - throw new ClientError( - `Full-text index '${attribute_name}' contains an unknown or duplicate field`, - 400 - ); - } - condition.includeHighlights = - condition.includeHighlights === true || selectRequestsProperty(target.select, '$highlights'); - if (condition.includeHighlights && !fullTextDefinition.highlighting) - throw new ClientError(`Full-text index '${attribute_name}' does not enable highlighting`, 400); - if (!TableResource.fullTextQueryIndexes[attribute_name]?.customIndex) - throw new IndexRebuildingError(`Full-text index '${attribute_name}' is not ready`); - includeFullTextHighlights ||= condition.includeHighlights; - continue; - } - let attribute = attribute_name == null ? primaryKeyAttribute : findAttribute(attributes, attribute_name); - if (!attribute && Array.isArray(attribute_name) && attribute_name.length > 1) { - // Plain JSON nested path: the leaf may not be declared in the - // schema. Fall back to the root attribute so we can validate - // existence without requiring the inner structure to be typed. - attribute = findAttribute(attributes, attribute_name[0]); - } - if (!attribute) { - if (attribute_name != null && !target.allowConditionsOnDynamicAttributes) - throw handleHDBError(new Error(), `${attribute_name} is not a defined attribute`, 404); - } else if (attribute.type || COERCIBLE_OPERATORS[condition.comparator]) { - // Do auto-coercion or coercion as required by the attribute type. - // Skipped for nested paths into plain JSON — the root attribute's - // type is not the leaf type, so coercion would be wrong. - const isNestedPathRoot = - Array.isArray(attribute_name) && attribute_name.length > 1 && !attribute.relationship; - if (!isNestedPathRoot) { - if (condition[1] === undefined) condition.value = coerceTypedValues(condition.value, attribute); - else condition[1] = coerceTypedValues(condition[1], attribute); - } - } - if (condition.chainedConditions) { - if (condition.operator === 'or') - throw new ClientError('Chaining a condition with |= is not supported, use | with the attribute name'); - if (condition.operator && condition.operator !== 'and') - throw new ClientError(`Unknown chained condition operator ${condition.operator}`); - if (condition.chainedConditions.length !== 1) - throw new ClientError('Chaining more than one condition onto a condition is not supported'); - const chained = condition.chainedConditions[0]; - const conditionComparator = ALTERNATE_COMPARATOR_NAMES[condition.comparator] || condition.comparator; - const chainedComparator = ALTERNATE_COMPARATOR_NAMES[chained.comparator] || chained.comparator; - const chainedIsLower = chainedComparator === 'gt' || chainedComparator === 'ge'; - const lower = chainedIsLower ? chained : condition; - const upper = chainedIsLower ? condition : chained; - const lowerComparator = chainedIsLower ? chainedComparator : conditionComparator; - const upperComparator = chainedIsLower ? conditionComparator : chainedComparator; - if ( - (lowerComparator !== 'gt' && lowerComparator !== 'ge') || - (upperComparator !== 'lt' && upperComparator !== 'le') - ) { - throw new ClientError( - 'Invalid chained condition, only a greater than and a less than condition can be chained together' - ); - } - condition.comparator = (lowerComparator + upperComparator) as any; - condition.value = [lower.value, upper.value]; - } - } - return conditions; - } function orderConditions(conditions: Condition[], operator: string) { if (target.enforceExecutionOrder) return conditions; // don't rearrange conditions for (const condition of conditions) { @@ -6550,14 +6452,10 @@ export function makeTable(options): TableResourceClass { if (conditions.length > 1 && operator !== 'or') return sortBy(conditions, estimateCondition(TableResource)); else return conditions; } - function coerceTypedValues(value: any, attribute: Attribute) { - if (Array.isArray(value)) { - return value.map((value) => coerceType(value, attribute)); - } - return coerceType(value, attribute); - } const operator = target.operator; - if (conditions.length > 0 || operator) conditions = prepareConditions(conditions, operator); + const prepared = TableResource._prepareQueryConditions(conditions, operator, target, context); + conditions = prepared.conditions; + const includeFullTextHighlights = prepared.includeFullTextHighlights; let sort = typeof target.sort === 'object' && target.sort; if ( TableResource.fullTextIndexes.length > 0 && @@ -6816,9 +6714,12 @@ export function makeTable(options): TableResourceClass { } } } finally { - // We own the iteration here (no results.onDone consumer), so release the read - // transaction unconditionally — including when the drain throws — or the snapshot leaks. - if (readTxn) txn.doneReadTxn(readTxnGeneration); + // Count pages have no streaming onDone consumer to release their cursors. + try { + (transformToRecord as any).return?.(); + } finally { + if (readTxn) txn.doneReadTxn(readTxnGeneration); + } } let total: number | null; if (wantExact) { @@ -6857,6 +6758,7 @@ export function makeTable(options): TableResourceClass { if (target.offset || target.limit !== undefined) results = results.slice(offset, end); results.onDone = () => { results.onDone = null; // ensure that it isn't called twice + (transformToRecord as any).return?.(); txn.unregisterReadIterator(results); if (readTxn) txn.doneReadTxn(readTxnGeneration); }; @@ -6871,6 +6773,428 @@ export function makeTable(options): TableResourceClass { throw error; } } + static _prepareRelationshipSelection( + attribute, + context, + filtered?, + ensureLoaded = true + ): (record: any, entry?: any, isSync?: boolean) => any { + if (attribute.conditions !== undefined && !Array.isArray(attribute.conditions)) + throw new ClientError('Nested conditions must be an array'); + const resolver = propertyResolvers[attribute.name]; + const targetTable = resolver.definition.tableClass; + const offset = attribute.offset ?? 0; + const limit = attribute.limit ?? Infinity; + if ( + !Number.isSafeInteger(offset) || + offset < 0 || + (attribute.limit !== undefined && (!Number.isSafeInteger(limit) || limit < 0)) + ) + throw new ClientError('Nested limit and offset must be non-negative safe integers'); + const filterMap = filtered?.[attribute.name]; + const childFiltered = Object.assign(Object.create(null), filterMap); + const rejectUnsupported = (conditions) => { + for (const condition of conditions) { + if (condition.conditions) rejectUnsupported(condition.conditions); + else if (fullTextComparatorMode(condition.comparator)) + throw new ClientError('Full-text predicates are not supported in relationship selections'); + } + }; + rejectUnsupported(attribute.conditions || []); + const conditions = targetTable._prepareQueryConditions( + cloneConditions(attribute.conditions || []), + attribute.operator, + attribute, + context, + true + ).conditions; + for (let sort = typeof attribute.sort === 'object' && attribute.sort; sort; sort = sort.next) + targetTable._assertNestedReadAccess(sort.attribute, context, attribute); + const compile = (conditions, operator) => { + const predicates = conditions.map((condition) => { + if (condition.conditions) return compile(condition.conditions, condition.operator); + const predicate = filterByType( + condition, + targetTable, + context, + null, + (condition.attribute ?? condition[0]) === targetTable.primaryKey + ); + if (!predicate) throw new ClientError('This predicate is not supported in a relationship selection'); + return predicate; + }); + return operator === 'or' + ? (record, entry) => predicates.some((predicate) => predicate(record, entry)) + : (record, entry) => predicates.every((predicate) => predicate(record, entry)); + }; + const predicate = conditions.length ? compile(conditions, attribute.operator) : undefined; + const plural = Boolean(resolver.to || attributes.find((item) => item.name === attribute.name)?.elements); + if (limit === 0) + return (record) => { + if (resolver.from && !resolver.to && record[resolver.from] == null) return record[resolver.from]; + return plural ? [] : undefined; + }; + let transform; + let offsetGuard; + let transformReadTxn; + const active = new Set<() => void>(); + let selectionStopped = false; + const relationshipSelection = (record, entry?, isSync = false) => { + if (selectionStopped) return undefined; + let value; + if (filterMap?.hasMappings) { + const key = resolver.from ? record[resolver.from] : flattenKey(entry?.key ?? record[primaryKey]); + value = filterMap.get(key) || []; + } else if (filterMap) value = filterMap.fromRecord?.(record); + else value = resolver(record, context, entry, true, undefined, undefined, true); + return when(value, (value) => { + if (selectionStopped) return undefined; + if (value == null) return value; + const many = + Array.isArray(value) || + typeof value[Symbol.iterator] === 'function' || + typeof value[Symbol.asyncIterator] === 'function'; + const source = many ? value : [value]; + let candidates = new ExtendedIterable(); + candidates.iterate = (options) => { + const sourceIterator = + options?.async && source[Symbol.asyncIterator] + ? source[Symbol.asyncIterator]() + : source[Symbol.iterator](); + let ended = false; + return { + next: () => (stopped || ended ? { done: true } : sourceIterator.next()), + return: () => { + if (ended) return { done: true }; + return when(sourceIterator.return?.() || { done: true }, (result) => { + ended = true; + return result; + }); + }, + }; + }; + const readTxn = targetTable._readTxnForContext(context); + const readCandidate = (candidate) => + when(candidate, (candidate) => { + if (stopped) return SKIP; + if (candidate == null) return predicate ? SKIP : candidate; + const entry = + typeof candidate === 'object' && candidate.key !== undefined + ? candidate + : targetTable.primaryStore.getEntry(candidate, { transaction: readTxn }); + return when(entry, (entry) => + predicate && (!entry?.value || !predicate(entry.value, entry)) ? SKIP : entry + ); + }); + let examined = 0; + candidates = candidates.map((candidate) => { + if (!isSync && (++examined & (COUNT_YIELD_INTERVAL - 1)) === 0) + return new Promise((resolve) => setImmediate(resolve)).then(() => readCandidate(candidate)); + return readCandidate(candidate); + }); + let select = attribute.select || (Array.isArray(attribute) ? attribute : undefined); + const sort = typeof attribute.sort === 'object' && attribute.sort; + let nestedSort = false; + for (let order = sort; order; order = order.next) nestedSort ||= Array.isArray(order.attribute); + if (nestedSort) select = copySelection(select); + const ordered = targetTable.transformToOrderedSelect( + candidates, + select, + sort, + context, + readTxn, + (entry) => entry + ); + if (!transform || transformReadTxn !== readTxn || nestedSort) { + transform = targetTable.transformEntryForSelect( + select, + context, + readTxn, + childFiltered, + ensureLoaded, + Boolean(predicate), + predicate + ); + offsetGuard = + predicate && offset + ? targetTable.transformEntryForSelect( + undefined, + context, + readTxn, + undefined, + ensureLoaded, + true, + predicate + ) + : undefined; + transformReadTxn = readTxn; + } + const projectEntry = transform; + const checkOffsetEntry = offsetGuard; + const iterator = ordered[isSync ? Symbol.iterator : Symbol.asyncIterator](); + const results = []; + const txn = txnForContext(context); + let stopped = false; + let closed = false; + let closing; + const close = () => { + stopped = true; + if (closed) return; + if (closing) return closing; + const complete = () => { + closed = true; + active.delete(owned.onDone); + txn.unregisterReadIterator(owned); + }; + const failed = (error) => { + closing = undefined; + txn.registerReadIterator(owned); + throw error; + }; + try { + const returned = iterator.return?.(); + if (typeof returned?.then === 'function') return (closing = returned.then(complete, failed)); + complete(); + } catch (error) { + return failed(error); + } + }; + const owned = { + onDone: () => { + (projectEntry as any).return?.(); + (checkOffsetEntry as any)?.return?.(); + const completion = close(); + if (typeof completion?.catch === 'function') + completion.catch((error) => logger.warn?.('Failed to close a nested relationship iterator', error)); + }, + }; + active.add(owned.onDone); + txn.registerReadIterator(owned); + let skipped = 0; + const finish = () => when(close(), () => (many ? results : results[0])); + const fail = (error) => { + const rethrow = () => { + throw error; + }; + let completion; + try { + completion = close(); + } catch { + throw error; + } + if (typeof completion?.then === 'function') return completion.then(rethrow, rethrow); + throw error; + }; + const advance = () => { + if (stopped) return finish(); + try { + return consume(iterator.next()); + } catch (error) { + return fail(error); + } + }; + const consume = (iteration) => { + try { + if (typeof iteration?.then === 'function') return iteration.then(consume, fail); + while (!iteration.done && !stopped) { + if (skipped < offset) { + const checked = checkOffsetEntry + ? checkOffsetEntry.call({ isSync }, iteration.value) + : iteration.value; + if (typeof checked?.then === 'function') + return checked.then((value) => { + if (stopped) return finish(); + if (value !== SKIP) skipped++; + return advance(); + }, fail); + if (checked !== SKIP) skipped++; + } else { + const projected = projectEntry.call({ isSync }, iteration.value); + if (typeof projected?.then === 'function') + return projected.then((value) => { + if (stopped) return finish(); + if (value !== SKIP) results.push(value); + return results.length >= limit ? finish() : advance(); + }, fail); + if (projected !== SKIP) results.push(projected); + if (results.length >= limit) return finish(); + } + iteration = iterator.next(); + if (typeof iteration?.then === 'function') return iteration.then(consume, fail); + } + return finish(); + } catch (error) { + return fail(error); + } + }; + return advance(); + }); + }; + (relationshipSelection as any).return = () => { + selectionStopped = true; + for (const close of [...active]) { + try { + close(); + } catch (error) { + logger.warn?.('Failed to close a nested relationship iterator', error); + } + } + }; + return relationshipSelection; + } + static _assertNestedReadAccess(name, context, target) { + if (!target.checkPermission) return; + const permission = getTablePermissions(context?.user, target); + if (permission?.isSuperUser) return; + const path = Array.isArray(name) ? name : [name ?? primaryKey]; + const readable = permission?.attribute_permissions; + if (!permission?.read || (readable?.length && !attributesAsObject(readable, 'read')[path[0]])) + throw new AccessViolation(context?.user); + if (path.length > 1) + propertyResolvers[path[0]]?.definition?.tableClass?._assertNestedReadAccess(path.slice(1), context, target); + } + static _prepareQueryConditions(conditions, operator, target, context, nested = false) { + let includeFullTextHighlights = false; + const QueryError = nested ? ClientError : Error; + function prepareConditions(conditions: any[], operator: string) { + // some validation: + switch (operator) { + case 'and': + case undefined: + if (conditions.length < 1) throw new QueryError('An "and" operator requires at least one condition'); + break; + case 'or': + if (conditions.length < 2) throw new QueryError('An "or" operator requires at least two conditions'); + break; + default: + throw new QueryError('Invalid operator ' + operator); + } + for (const condition of conditions) { + if (condition.conditions) { + condition.conditions = prepareConditions(condition.conditions, condition.operator); + continue; + } + // Normalize `not_X` comparator forms passed in via structured queries. + // The REST parser already does this, but programmatic callers may + // pass `not_in`, `not_starts_with`, etc. directly. + if (condition.comparator) { + const resolved = resolveComparator(condition.comparator); + if (resolved.negated) { + condition.comparator = resolved.comparator; + condition.negated = true; + } + } + const attribute_name = condition[0] ?? condition.attribute; + if (nested) TableResource._assertNestedReadAccess(attribute_name, context, target); + const fullTextMode = + TableResource.fullTextIndexes.length > 0 || + Array.isArray(attribute_name) || + (typeof attribute_name === 'string' && fullTextFieldNames?.has(attribute_name)) + ? fullTextComparatorMode(condition.comparator) + : undefined; + if (!fullTextMode && (fullTextFieldNames || hasRelationships)) assertFullTextRecordField(attribute_name); + const fullTextDefinition = + fullTextMode && typeof attribute_name === 'string' + ? TableResource.fullTextIndexes.find((definition) => definition.name === attribute_name) + : undefined; + if (fullTextMode && Array.isArray(attribute_name)) + throw new ClientError('Full-text predicates must directly name an index on the queried table', 400); + if (fullTextMode && !fullTextDefinition) throwUnknownFullTextIndex(context, target, attribute_name); + if (fullTextDefinition) { + const fields = condition.fields; + if (fields !== undefined) { + if (!Array.isArray(fields) || fields.length === 0 || fields.some((field) => typeof field !== 'string')) + throw new ClientError(`Full-text index '${attribute_name}' requires a non-empty fields list`, 400); + assertFullTextReadAccess(context, target, fullTextDefinition, fields); + } else assertFullTextReadAccess(context, target, fullTextDefinition); + const value = condition[1] ?? condition.value; + if (typeof value !== 'string' || value.length === 0) + throw new ClientError( + `Full-text index '${attribute_name}' requires a full-text comparator and non-empty string value`, + 400 + ); + if (fullTextMode === 'phrase' && !fullTextDefinition.positions) + throw new ClientError(`Full-text index '${attribute_name}' does not store phrase positions`, 400); + if ((fullTextMode === 'prefix' || fullTextMode === 'fuzzy-prefix') && !fullTextDefinition.surfaceTerms) + throw new ClientError(`Full-text index '${attribute_name}' does not store surface terms`, 400); + if (fields !== undefined) { + const sourceNames = new Set(fullTextDefinition.fields.map(({ name }) => name)); + if (new Set(fields).size !== fields.length || fields.some((field) => !sourceNames.has(field))) + throw new ClientError( + `Full-text index '${attribute_name}' contains an unknown or duplicate field`, + 400 + ); + } + condition.includeHighlights = + condition.includeHighlights === true || selectRequestsProperty(target.select, '$highlights'); + if (condition.includeHighlights && !fullTextDefinition.highlighting) + throw new ClientError(`Full-text index '${attribute_name}' does not enable highlighting`, 400); + if (!TableResource.fullTextQueryIndexes[attribute_name]?.customIndex) + throw new IndexRebuildingError(`Full-text index '${attribute_name}' is not ready`); + includeFullTextHighlights ||= condition.includeHighlights; + continue; + } + let attribute = attribute_name == null ? primaryKeyAttribute : findAttribute(attributes, attribute_name); + if (!attribute && Array.isArray(attribute_name) && attribute_name.length > 1) { + // Plain JSON nested path: the leaf may not be declared in the + // schema. Fall back to the root attribute so we can validate + // existence without requiring the inner structure to be typed. + attribute = findAttribute(attributes, attribute_name[0]); + } + if (!attribute) { + if (attribute_name != null && !target.allowConditionsOnDynamicAttributes) + throw handleHDBError(new Error(), `${attribute_name} is not a defined attribute`, 404); + } else if (attribute.type || COERCIBLE_OPERATORS[condition.comparator]) { + // Do auto-coercion or coercion as required by the attribute type. + // Skipped for nested paths into plain JSON — the root attribute's + // type is not the leaf type, so coercion would be wrong. + const isNestedPathRoot = + Array.isArray(attribute_name) && attribute_name.length > 1 && !attribute.relationship; + if (!isNestedPathRoot) { + if (condition[1] === undefined) condition.value = coerceTypedValues(condition.value, attribute); + else condition[1] = coerceTypedValues(condition[1], attribute); + } + } + if (condition.chainedConditions) { + if (condition.operator === 'or') + throw new ClientError('Chaining a condition with |= is not supported, use | with the attribute name'); + if (condition.operator && condition.operator !== 'and') + throw new ClientError(`Unknown chained condition operator ${condition.operator}`); + if (condition.chainedConditions.length !== 1) + throw new ClientError('Chaining more than one condition onto a condition is not supported'); + const chained = condition.chainedConditions[0]; + const conditionComparator = ALTERNATE_COMPARATOR_NAMES[condition.comparator] || condition.comparator; + const chainedComparator = ALTERNATE_COMPARATOR_NAMES[chained.comparator] || chained.comparator; + const chainedIsLower = chainedComparator === 'gt' || chainedComparator === 'ge'; + const lower = chainedIsLower ? chained : condition; + const upper = chainedIsLower ? condition : chained; + const lowerComparator = chainedIsLower ? chainedComparator : conditionComparator; + const upperComparator = chainedIsLower ? conditionComparator : chainedComparator; + if ( + (lowerComparator !== 'gt' && lowerComparator !== 'ge') || + (upperComparator !== 'lt' && upperComparator !== 'le') + ) { + throw new ClientError( + 'Invalid chained condition, only a greater than and a less than condition can be chained together' + ); + } + condition.comparator = (lowerComparator + upperComparator) as any; + condition.value = [lower.value, upper.value]; + } + } + return conditions; + } + function coerceTypedValues(value: any, attribute: Attribute) { + if (Array.isArray(value)) { + return value.map((value) => coerceType(value, attribute)); + } + return coerceType(value, attribute); + } + return { + conditions: conditions.length > 0 || operator ? prepareConditions(conditions, operator) : conditions, + includeFullTextHighlights, + }; + } /** * This is responsible for ordering and select()ing the attributes/properties from returned entries * @param select @@ -7122,14 +7446,27 @@ export function makeTable(options): TableResourceClass { checkLoaded = true; } let transformCache; + let relationshipSelections; + for (const attribute of Array.isArray(select) ? select : []) { + if (propertyResolvers?.[attribute.name]?.definition?.tableClass && hasNestedQueryOptions(attribute)) { + (relationshipSelections ||= Object.create(null))[attribute.name] = this._prepareRelationshipSelection( + attribute, + context, + filtered, + ensure_loaded + ); + } + } const source = this.source; const resourceClass = this; + let selectionStopped = false; const lazy = select?.length < 4; const syncLoadOptions = { transaction: readTxn, lazy, ensureLoaded: checkLoaded }; // loadLocalRecord sets `async` on the options of an async load, so sync loads need their own const asyncLoadOptions = { transaction: readTxn, lazy, ensureLoaded: checkLoaded, async: true }; // Transform an entry to a record. Note that *this* instance is intended to be the iterator. const transform = function (entry: Entry) { + if (selectionStopped) return SKIP; let record; if (context?.transaction?.stale) context.transaction.stale = false; if (entry != undefined) { @@ -7200,6 +7537,15 @@ export function makeTable(options): TableResourceClass { const resolver = propertyResolvers?.[attribute_name]; let value; if (resolver) { + const selection = relationshipSelections?.[attribute_name]; + if (selection) { + const selected = selection(record, entry, this.isSync); + if (typeof selected?.then === 'function') { + if (!promises) promises = []; + promises.push(selected.then((value) => callback(value, attribute_name))); + } else callback(selected, attribute_name); + return; + } const filterMap = filtered?.[attribute_name]; if (filterMap) { if (filterMap.hasMappings) { @@ -7367,6 +7713,12 @@ export function makeTable(options): TableResourceClass { } return record; }; + (transform as any).return = () => { + selectionStopped = true; + for (const cache of [relationshipSelections, transformCache]) { + if (cache) for (const child of Object.values(cache)) (child as any).return?.(); + } + }; return transform; } @@ -9022,12 +9374,20 @@ export function makeTable(options): TableResourceClass { hasRelationships = true; if (relationship.to) { if (attribute.elements?.definition) { - propertyResolvers[attribute.name] = attribute.resolve = (object, context, entry, returnEntry?) => { + propertyResolvers[attribute.name] = attribute.resolve = ( + object, + context, + entry, + returnEntry?, + _sort?, + _comparing?, + lazy? + ) => { // TODO: Get raw record/entry? const id = object[relationship.from ? relationship.from : primaryKey]; const relatedTable = attribute.elements.definition.tableClass; if (returnEntry) { - return ( + const candidates = ( searchByIndex( { attribute: relationship.to, value: id }, txnForContext(context).getReadTxn(), @@ -9040,7 +9400,8 @@ export function makeTable(options): TableResourceClass { return relatedTable.primaryStore.getEntry(entry, { transaction: txnForContext(context).getReadTxn(), }); - }).asArray; + }); + return lazy ? candidates : candidates.asArray; } return relatedTable.search([{ attribute: relationship.to, value: id }], context).asArray; }; @@ -9060,7 +9421,15 @@ export function makeTable(options): TableResourceClass { } else if (relationship.from) { const definition = attribute.definition || attribute.elements?.definition; if (definition) { - propertyResolvers[attribute.name] = attribute.resolve = (object, context, entry, returnEntry?) => { + propertyResolvers[attribute.name] = attribute.resolve = ( + object, + context, + entry, + returnEntry?, + _sort?, + _comparing?, + lazy? + ) => { const ids = object[relationship.from]; if (ids == null) return ids; const ctx = context ?? contextStorage.getStore(); @@ -9079,7 +9448,7 @@ export function makeTable(options): TableResourceClass { const store = definition.tableClass.primaryStore; const method = returnEntry ? 'getEntry' : 'getSync'; const options = { transaction: txnForContext(context).getReadTxn() }; - const results = normalizedIds.map((id) => { + const results = (lazy ? new ExtendedIterable(normalizedIds) : normalizedIds).map((id) => { const value = store[method](id, options); if (TableResource.loadAsInstance === false) freezeRecord(returnEntry ? value?.value : value); return value; diff --git a/resources/search.ts b/resources/search.ts index 018781d7d1..b1966e7c53 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -2407,9 +2407,6 @@ function parseBlock(query, expectedEnd) { attribute = null; } } else { - if (value && entry.conditions.length) { - recordError('conditions/comparisons are not allowed in a property list'); - } query.push(entry); } if (queryString[lastIndex] === ',') { diff --git a/unitTests/resources/nestedRelationshipConstraints.test.js b/unitTests/resources/nestedRelationshipConstraints.test.js new file mode 100644 index 0000000000..56d4d83e98 --- /dev/null +++ b/unitTests/resources/nestedRelationshipConstraints.test.js @@ -0,0 +1,1097 @@ +const assert = require('node:assert'); +const { createServer } = require('node:http'); +const { setupTestDBPath } = require('../testUtils.js'); +const { table } = require('#src/resources/databases'); +const { RocksDatabase } = require('@harperfast/rocksdb-js'); +const { Resources } = require('#src/resources/Resources'); +const { transaction } = require('#src/resources/transaction'); +const { RequestTarget } = require('#src/resources/RequestTarget'); +const { handleApplication } = require('#src/server/REST'); +const { Request } = require('#src/server/serverHelpers/Request'); +const { getWorkerIndex, setMainIsWorker } = require('#js/server/threads/manageThreads'); +require('#src/server/serverHelpers/serverUtilities'); + +const children = [ + { id: 'c1', parentId: 'p1', state: 'open', score: 10 }, + { id: 'c2', parentId: 'p1', state: 'closed', score: 20 }, + { id: 'c3', parentId: 'p1', state: 'open', score: 30 }, + { id: 'c4', parentId: 'p1', state: 'open', score: 40 }, + { id: 'c5', parentId: 'p2', state: 'closed', score: 5 }, + { id: 'c6', parentId: 'p2', state: 'open', score: 50 }, + { id: 'c7', parentId: 'p2', state: 'closed', score: 60 }, +]; + +const ordinaryUser = { + role: { + permission: { + nested_constraints: { + tables: { + Parent: { read: true }, + Child: { + read: true, + attribute_permissions: [ + { attribute_name: 'id', read: true }, + { attribute_name: 'state', read: true }, + ], + }, + }, + }, + }, + }, +}; + +const ordinaryAllUser = { + role: { permission: { nested_constraints: { tables: { Parent: { read: true }, Child: { read: true } } } } }, +}; + +describe('nested relationship constraints', function () { + this.timeout(15000); + let Parent, Child, server, base, previousMainIsWorker, resources; + let projected; + before(async function () { + setupTestDBPath(); + previousMainIsWorker = getWorkerIndex() === 0; + setMainIsWorker(true); + Child = table({ + database: 'nested_constraints', + table: 'Child', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'parentId', type: 'String', indexed: true }, + { name: 'state', type: 'String', indexed: true }, + { name: 'score', type: 'Int', indexed: true }, + { name: 'observed', computed: true }, + { + name: 'siblings', + relationship: { to: 'parentId', from: 'parentId' }, + elements: { type: 'Child', definition: {} }, + }, + ], + }); + Child.attributes.find((attribute) => attribute.name === 'siblings').elements.definition.tableClass = Child; + Child.updatedAttributes(); + Child.setComputedAttribute('observed', (record) => { + projected.push(record.id); + return record.score; + }); + Parent = table({ + database: 'nested_constraints', + table: 'Parent', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'childId', type: 'ID' }, + { name: 'metadata', type: 'Any' }, + { name: 'childIds', elements: { type: 'ID' } }, + { + name: 'children', + relationship: { to: 'parentId' }, + elements: { type: 'Child', definition: { tableClass: Child } }, + }, + { name: 'child', relationship: { from: 'childId' }, type: 'Child', definition: { tableClass: Child } }, + { + name: 'references', + relationship: { from: 'childIds' }, + elements: { type: 'Child', definition: { tableClass: Child } }, + }, + ], + }); + await Parent.put('p1', { + id: 'p1', + childId: 'c1', + childIds: ['c1', 'missing', 'c2', 'c3', 'c4'], + metadata: { label: 'kept', hidden: 'omit' }, + }); + await Parent.put('p2', { id: 'p2', childId: 'missing' }); + for (const child of children) await Child.put(child.id, child); + resources = new Resources(); + resources.set('Parent', Parent); + resources.set('Child', Child); + let rest; + handleApplication({ + resources, + options: { getAll: () => ({ webSocket: false }) }, + routeFor: () => ({ urlPath: '/nested-constraints' }), + server: { + http: (handler) => { + rest = handler; + }, + ws: () => [], + }, + }); + server = createServer(async (input, output) => { + try { + const request = new Request(input, output); + request.user = + input.headers['x-fixture-role'] === 'all' + ? ordinaryAllUser + : input.headers['x-fixture-role'] + ? ordinaryUser + : { role: { permission: { super_user: true } } }; + const response = await rest(request, () => ({ status: 404, headers: new Map(), body: 'not found' })); + output.writeHead(response.status, Object.fromEntries(response.headers)); + if (typeof response.body === 'string' || Buffer.isBuffer(response.body)) output.end(response.body); + else { + for await (const chunk of response.body || []) output.write(chunk); + output.end(); + } + } catch (error) { + output.writeHead(error.statusCode || 500); + output.end(error.message); + } + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + base = `http://127.0.0.1:${server.address().port}`; + }); + after(async function () { + if (server) await new Promise((resolve) => server.close(resolve)); + if (previousMainIsWorker !== undefined) setMainIsWorker(previousMainIsWorker); + }); + beforeEach(function () { + projected = []; + }); + async function get(path, headers) { + const response = await fetch(base + path, { headers }); + const text = await response.text(); + assert.strictEqual(response.status, 200, text); + return JSON.parse(text); + } + async function query(selection, extra = {}) { + const rows = []; + for await (const row of Parent.search( + { select: ['id', selection], ...extra }, + extra.checkPermission ? { user: ordinaryUser } : undefined + )) + rows.push(row); + return rows; + } + const ids = (rows, name = 'children') => rows.map((row) => row[name].map((child) => child?.id)); + for (const count of ['exact', 'estimated']) + it(`releases counted nested selections (${count})`, async function () { + const context = {}; + const links = []; + await transaction(context, async (txn) => { + await Parent.get('p1', context); + for (let link = txn; link; link = link.next) links.push([link, link.readTxnsUsed]); + const rows = await Parent.search( + { + count, + limit: 1, + sort: { attribute: 'id' }, + select: ['id', { name: 'children', limit: 1, select: ['id', 'observed'] }], + }, + context + ); + assert.deepStrictEqual( + rows.map((row) => ({ ...row })), + [{ id: 'p1', children: [{ id: 'c1', observed: 10 }] }] + ); + if (count === 'exact') assert.strictEqual(rows.recordCount, 2); + else assert(rows.recordCount >= 2); + assert.strictEqual(rows.recordCountExact, count === 'exact'); + for (const [link, references] of links) { + assert.strictEqual(link.closeOwnedReadIterators(), 0); + assert.strictEqual(link.readTxnsUsed, references); + } + }); + for (const [link] of links) assert.strictEqual(link.readTxnsUsed, 0); + }); + it('releases a counted nested selection when projection fails', async function () { + const context = {}; + Child.setComputedAttribute('observed', () => { + throw new Error('counted projection failure'); + }); + try { + await transaction(context, async (txn) => { + await Parent.get('p1', context); + const links = []; + for (let link = txn; link; link = link.next) links.push([link, link.readTxnsUsed]); + await assert.rejects( + () => + Parent.search( + { count: 'exact', limit: 1, select: ['id', { name: 'children', limit: 1, select: ['observed'] }] }, + context + ), + /counted projection failure/ + ); + for (const [link, references] of links) { + assert.strictEqual(link.closeOwnedReadIterators(), 0); + assert.strictEqual(link.readTxnsUsed, references); + } + }); + } finally { + Child.setComputedAttribute('observed', (record) => { + projected.push(record.id); + return record.score; + }); + } + }); + it('retains unfiltered relationship contents', async function () { + assert.deepStrictEqual(ids(await get('/Parent/?select(id,children{id,state,score})')), [ + ['c1', 'c2', 'c3', 'c4'], + ['c5', 'c6', 'c7'], + ]); + }); + for (const single of [false, true]) { + const path = single ? '/Parent/p1?' : '/Parent/?'; + const rows = (result) => (single ? [result] : result); + it(`applies nested limit per parent over HTTP (${single ? 'record' : 'collection'})`, async function () { + assert.deepStrictEqual( + ids(rows(await get(path + 'select(id,children[limit(1)&select(id,state,score)])'))), + single ? [['c1']] : [['c1'], ['c5']] + ); + }); + it(`applies nested predicates over HTTP (${single ? 'record' : 'collection'})`, async function () { + assert.deepStrictEqual( + ids(rows(await get(path + 'select(id,children[state=open&select(id,state,score)])'))), + single ? [['c1', 'c3', 'c4']] : [['c1', 'c3', 'c4'], ['c6']] + ); + }); + it(`filters before limiting over HTTP (${single ? 'record' : 'collection'})`, async function () { + assert.deepStrictEqual( + ids( + rows(await get(path + 'select(id,children[state=open&score=ge=number:30&limit(1)&select(id,state,score)])')) + ), + single ? [['c3']] : [['c3'], ['c6']] + ); + }); + } + it('retains outer relationship filter candidates', async function () { + assert.deepStrictEqual( + ids( + await get('/Parent/?children.state=open&children.score=ge=number:30&select(id,children[limit(1)&select(id,)])') + ), + [['c3'], ['c6']] + ); + }); + it('retains the outer relationship predicate control', async function () { + assert.deepStrictEqual( + ids(await get('/Parent/?children.state=open&children.score=ge=number:30&select(id,children{id})')), + [['c3', 'c4'], ['c6']] + ); + }); + it('retains the child endpoint control', async function () { + assert.deepStrictEqual( + await get('/Child/?parentId=p1&state=open&score=ge=number:30&sort(score)&limit(1)&select(id,)'), + [{ id: 'c3' }] + ); + }); + it('retains the parent limit control', async function () { + const rows = await get('/Parent/?limit(1)&select(id,children{id})'); + assert.strictEqual(rows.length, 1); + assert.strictEqual(rows[0].children.length, 4); + }); + it('sorts eligible children before offset and limit', async function () { + assert.deepStrictEqual( + ids( + await query({ + name: 'children', + conditions: [{ attribute: 'state', value: 'open' }], + sort: { attribute: 'score', descending: true }, + offset: 1, + limit: 1, + select: ['id'], + }) + ), + [['c3'], []] + ); + }); + it('preserves parents when no child matches', async function () { + assert.deepStrictEqual( + ids( + await query({ + name: 'children', + conditions: [{ attribute: 'score', comparator: 'gt', value: 100 }], + limit: 1, + select: ['id'], + }) + ), + [[], []] + ); + }); + it('coerces typed values and compiles boolean groups and negation', async function () { + const selection = { + name: 'children', + conditions: [ + { + operator: 'or', + conditions: [ + { attribute: 'score', value: '30' }, + { attribute: 'score', value: '50' }, + ], + }, + { attribute: 'state', comparator: 'not_in', value: ['closed'] }, + ], + select: ['id'], + }; + assert.deepStrictEqual(ids(await query(selection)), [['c3'], ['c6']]); + assert.strictEqual(selection.conditions[0].conditions[0].value, '30'); + assert.strictEqual(selection.conditions[1].comparator, 'not_in'); + }); + it('does not project the unused tail of a limited relationship', async function () { + const rows = await query({ name: 'children', limit: 1, select: ['id', 'observed'] }); + assert.deepStrictEqual(ids(rows), [['c1'], ['c5']]); + assert.deepStrictEqual(projected, ['c1', 'c5']); + }); + it('returns empty arrays for zero limits without projecting children', async function () { + assert.deepStrictEqual(ids(await query({ name: 'children', limit: 0, select: ['id', 'observed'] })), [[], []]); + assert.deepStrictEqual(projected, []); + }); + for (const field of ['limit', 'offset']) + for (const value of [-1, 1.5, NaN, Infinity]) { + it(`rejects invalid nested ${field} ${value}`, async function () { + await assert.rejects( + () => query({ name: 'children', [field]: value, select: ['id'] }), + (error) => error.statusCode === 400 + ); + }); + } + for (const operator of ['and', 'or', 'invalid']) + it(`rejects a nested ${operator} operator without conditions as a client error`, async function () { + await assert.rejects( + () => query({ name: 'children', operator, select: ['id'] }), + (error) => error.statusCode === 400 + ); + }); + it('filters singular relationships without removing the parent', async function () { + const rows = await query({ name: 'child', conditions: [{ attribute: 'state', value: 'closed' }], select: ['id'] }); + assert.deepStrictEqual( + rows.map((row) => row.id), + ['p1', 'p2'] + ); + assert(rows.every((row) => row.child === undefined)); + }); + it('filters FK arrays and excludes missing entries when conditions apply', async function () { + const rows = await query({ + name: 'references', + conditions: [{ attribute: 'state', value: 'open' }], + limit: 2, + select: ['id'], + }); + assert.deepStrictEqual(rows[0].references, [{ id: 'c1' }, { id: 'c3' }]); + assert.strictEqual(rows[1].references, undefined); + }); + it('supports structured single-record get', async function () { + const row = await Parent.get({ + id: 'p1', + select: [ + 'id', + { name: 'children', conditions: [{ attribute: 'state', value: 'open' }], limit: 1, select: ['id'] }, + ], + }); + assert.deepStrictEqual(row, { id: 'p1', children: [{ id: 'c1' }] }); + }); + it('supports a parsed offset and sort', async function () { + const rows = []; + for await (const row of Parent.search( + new RequestTarget('?select(id,children[sort(-score)&limit(1,2)&select(id,)])') + )) + rows.push(row); + assert.deepStrictEqual(ids(rows), [['c3'], ['c6']]); + }); + it('preserves nested limits without a select through ordinary-role authorization', async function () { + const rows = await get('/Parent/?select(id,children[limit(1)])', { 'x-fixture-role': 'all' }); + assert.deepStrictEqual(ids(rows), [['c1'], ['c5']]); + assert.strictEqual(rows[0].children[0].score, 10); + }); + it('denies predicates on a child attribute the caller cannot read', async function () { + await assert.rejects( + () => + query( + { name: 'children', conditions: [{ attribute: 'score', comparator: 'gt', value: 20 }], select: ['id'] }, + { checkPermission: true } + ), + (error) => error.statusCode === 403 + ); + }); + it('honors readable predicates after ordinary-role authorization', async function () { + const rows = await get('/Parent/?select(id,children[state=open&limit(1)&select(id,)])', { + 'x-fixture-role': 'ordinary', + }); + assert.deepStrictEqual(ids(rows), [['c1'], ['c6']]); + }); + it('honors readable predicates with boolean permission checks in structured search', async function () { + const rows = await query( + { name: 'children', conditions: [{ attribute: 'state', value: 'open' }], limit: 1, select: ['id'] }, + { checkPermission: true } + ); + assert.deepStrictEqual(ids(rows), [['c1'], ['c6']]); + }); + it('honors readable predicates with boolean permission checks in structured get', async function () { + const row = await Parent.get( + { + id: 'p1', + checkPermission: true, + select: [ + 'id', + { name: 'children', conditions: [{ attribute: 'state', value: 'open' }], limit: 1, select: ['id'] }, + ], + }, + { user: ordinaryUser } + ); + assert.deepStrictEqual(ids([row]), [['c1']]); + }); + it('inherits deeper outer join candidates while paging child relationships', async function () { + const rows = await get( + '/Parent/?children.siblings.state=open&select(id,children[limit(1)&select(id,siblings[limit(1)&select(id,)])])' + ); + assert.deepStrictEqual(rows, [ + { id: 'p1', children: [{ id: 'c1', siblings: [{ id: 'c1' }] }] }, + { id: 'p2', children: [{ id: 'c5', siblings: [{ id: 'c6' }] }] }, + ]); + }); + it('preserves FK holes when only pagination is requested', async function () { + const rows = await query({ name: 'references', limit: 2, select: ['id'] }); + assert.deepStrictEqual(rows[0].references, [{ id: 'c1' }, undefined]); + }); + it('supports asynchronous child projection', async function () { + Child.setComputedAttribute('observed', (record) => Promise.resolve(record.score)); + try { + const rows = await query({ name: 'children', limit: 1, select: ['id', 'observed'] }); + assert.deepStrictEqual(rows[0].children, [{ id: 'c1', observed: 10 }]); + } finally { + Child.setComputedAttribute('observed', (record) => { + projected.push(record.id); + return record.score; + }); + } + }); + it('bounds child entry reads before materialization in a synchronous window', function () { + if (!(Child.primaryStore instanceof RocksDatabase)) return this.skip(); + const before = Child.primaryStore.readCount; + const rows = Array.from( + new Parent(undefined, {}).search({ select: ['id', { name: 'children', limit: 1, select: ['id'] }] }) + ); + const after = Child.primaryStore.readCount; + assert.deepStrictEqual(ids(rows), [['c1'], ['c5']]); + assert.strictEqual(after - before, 2, 'one child entry read per parent, including cache hits'); + const unlimited = Array.from( + new Parent(undefined, {}).search({ select: ['id', { name: 'children', select: ['id'] }] }) + ); + const unbounded = Child.primaryStore.readCount; + assert.strictEqual(unlimited.flatMap((row) => row.children).length, children.length); + assert.strictEqual(unbounded - after, children.length, 'positive control reads the full relationship'); + }); + it('applies constraints independently at two relationship depths', async function () { + const rows = await query({ + name: 'children', + limit: 1, + select: [ + 'id', + { + name: 'siblings', + conditions: [{ attribute: 'state', value: 'open' }], + sort: { attribute: 'score', descending: true }, + limit: 1, + select: ['id'], + }, + ], + }); + assert.deepStrictEqual(rows[0].children, [{ id: 'c1', siblings: [{ id: 'c4' }] }]); + assert.deepStrictEqual(rows[1].children, [{ id: 'c5', siblings: [{ id: 'c6' }] }]); + }); + it('keeps repeated programmatic selections reusable', async function () { + const selection = { + name: 'children', + conditions: [{ attribute: 'score', comparator: 'ge', value: '30' }], + limit: 1, + select: ['id'], + }; + assert.deepStrictEqual(ids(await query(selection)), [['c3'], ['c6']]); + assert.deepStrictEqual(ids(await query(selection)), [['c3'], ['c6']]); + assert.strictEqual(selection.conditions[0].value, '30'); + }); + it('validates pagination even when no parent matches', async function () { + await assert.rejects( + () => + query({ name: 'children', limit: -1, select: ['id'] }, { conditions: [{ attribute: 'id', value: 'absent' }] }), + (error) => error.statusCode === 400 + ); + }); + it('rejects unsupported full-text predicates explicitly', async function () { + await assert.rejects( + () => + query({ + name: 'children', + conditions: [{ attribute: 'state', comparator: 'matches', value: 'open' }], + select: ['id'], + }), + (error) => error.statusCode === 400 + ); + }); + it('closes owned nested iterators when a transaction abandons an async projection', async function () { + if (!(Child.primaryStore instanceof RocksDatabase)) return this.skip(); + let entered, release; + const started = new Promise((resolve) => { + entered = resolve; + }); + const pending = new Promise((resolve) => { + release = resolve; + }); + Child.setComputedAttribute('observed', (record) => { + entered(); + return pending.then(() => record.score); + }); + try { + await transaction({}, async (txn) => { + const rows = Parent.search({ select: ['id', { name: 'children', limit: 2, select: ['id', 'observed'] }] }); + const iterator = rows[Symbol.asyncIterator](); + const first = iterator.next(); + await started; + assert(txn.closeOwnedReadIterators() >= 2, 'owns both parent and nested child iterators'); + assert.strictEqual(txn.closeOwnedReadIterators(), 0); + const readsAtClose = Child.primaryStore.readCount; + release(); + const result = await first; + assert.deepStrictEqual(result.value.children, []); + assert.strictEqual(Child.primaryStore.readCount, readsAtClose, 'no reads after cancellation'); + await iterator.return(); + }); + } finally { + release(); + Child.setComputedAttribute('observed', (record) => { + projected.push(record.id); + return record.score; + }); + } + }); + it('keeps comparison reads of FK arrays synchronous and array-valued', function () { + const values = Parent.propertyResolvers.references( + { childIds: ['c1', 'c2'] }, + {}, + undefined, + false, + undefined, + true + ); + assert(Array.isArray(values)); + assert.deepStrictEqual( + values.map((record) => record.id), + ['c1', 'c2'] + ); + }); + it('resolves asynchronous constrained relationships in positional single-record selections', async function () { + Child.setComputedAttribute('observed', (record) => Promise.resolve(record.score)); + const select = ['id', { name: 'children', limit: 1, select: ['id', 'observed'] }]; + select.asArray = true; + try { + assert.deepStrictEqual(await Parent.get({ id: 'p1', select }), ['p1', [{ id: 'c1', observed: 10 }]]); + } finally { + Child.setComputedAttribute('observed', (record) => { + projected.push(record.id); + return record.score; + }); + } + }); + it('yields while scanning rejected children before an unsatisfied limit', async function () { + await Parent.put('p-yield', { id: 'p-yield' }); + const rows = Array.from({ length: 2050 }, (_, index) => ({ + id: `yield-${index}`, + parentId: 'p-yield', + state: 'closed', + score: index, + })); + await Child.put(rows); + try { + const results = new Parent(undefined, {}).search({ + conditions: [{ attribute: 'id', value: 'p-yield' }], + select: [ + 'id', + { name: 'children', conditions: [{ attribute: 'state', value: 'absent' }], limit: 1, select: ['id'] }, + ], + }); + let yielded = false; + setImmediate(() => { + yielded = true; + }); + const iterator = results[Symbol.asyncIterator](); + try { + const first = await iterator.next(); + assert.strictEqual(yielded, true, 'rejected candidates yield to the event loop'); + assert.deepStrictEqual(first.value.children, []); + } finally { + await iterator.return(); + } + } finally { + await Child.delete({ conditions: [{ attribute: 'parentId', value: 'p-yield' }] }); + await Parent.delete('p-yield'); + } + }); + for (const id of [undefined, 'p1']) { + it(`rejects unsupported plain JSON query constraints before reading ${id || 'a collection'}`, async function () { + const target = { + select: ['id', { name: 'metadata', conditions: [{ attribute: 'label', value: 'kept' }], select: ['label'] }], + }; + await assert.rejects( + async () => { + if (id) await Parent.get({ id, ...target }); + else for await (const row of Parent.search(target)) void row; + }, + (error) => + error.statusCode === 400 && error.message === 'Nested query constraints require a relationship attribute' + ); + }); + } + it('rejects unsupported options nested below plain JSON projections', async function () { + await assert.rejects( + async () => + Parent.get({ + id: 'p1', + select: [ + { + name: 'metadata', + select: [{ name: 'tags', conditions: [{ attribute: 'label', value: 'kept' }], select: ['label'] }], + }, + ], + }), + (error) => + error.statusCode === 400 && error.message === 'Nested query constraints require a relationship attribute' + ); + }); + it('does not mutate caller selection objects when a nested sort propagates to a relationship', async function () { + const selection = { + name: 'children', + sort: { attribute: ['siblings', 'score'], descending: true }, + limit: 1, + select: Object.freeze(['id', Object.freeze({ name: 'siblings', select: ['id', 'score'] })]), + }; + const before = JSON.stringify(selection); + await query(selection); + await query(selection); + assert.strictEqual(JSON.stringify(selection), before); + }); + it('drops missing FK targets when sorting, matching the existing sort contract', async function () { + const rows = await query({ name: 'references', sort: { attribute: 'score' }, limit: 2, select: ['id'] }); + assert.deepStrictEqual(rows[0].references, [{ id: 'c1' }, { id: 'c2' }]); + }); + it('preserves an absent foreign key under a zero limit without reading children', async function () { + const rows = await query({ name: 'references', limit: 0, select: ['id'] }); + assert.deepStrictEqual(rows[0].references, []); + assert.strictEqual(rows[1].references, undefined); + }); + for (const single of [false, true]) + it(`retains plain JSON bracket projections without constraints (${single ? 'record' : 'collection'})`, async function () { + const result = await get((single ? '/Parent/p1?' : '/Parent/?') + 'select(id,metadata[select(label,)])'); + const row = single ? result : result[0]; + assert.deepStrictEqual(row.metadata, { label: 'kept' }); + }); + it('does not resume a relationship candidate provider after the selection is returned', async function () { + let resolve; + const pending = new Promise((done) => { + resolve = done; + }); + await transaction({}, async () => { + const selection = Parent._prepareRelationshipSelection( + { name: 'child', limit: 1, select: ['id', 'observed'] }, + {}, + { child: { fromRecord: () => pending } } + ); + const result = selection({ id: 'p1', childId: 'c1' }); + selection.return(); + resolve(Child.primaryStore.getEntry('c1')); + assert.strictEqual(await result, undefined); + assert.deepStrictEqual(projected, []); + }); + }); + for (const nestedOnly of [false, true]) + it(`cancels in-flight child projections when the parent iterator is returned (${nestedOnly ? 'nested-only' : 'direct'})`, async function () { + if (!(Child.primaryStore instanceof RocksDatabase)) return this.skip(); + let entered, release; + const started = new Promise((resolve) => { + entered = resolve; + }); + const pending = new Promise((resolve) => { + release = resolve; + }); + Child.setComputedAttribute('observed', (record) => { + entered(); + return pending.then(() => record.score); + }); + try { + await transaction({}, async (txn) => { + const selection = nestedOnly + ? { name: 'children', select: ['id', { name: 'siblings', limit: 2, select: ['id', 'observed'] }] } + : { name: 'children', limit: 2, select: ['id', 'observed'] }; + const results = Parent.search({ select: ['id', selection] }); + const iterator = results[Symbol.asyncIterator](); + const first = Promise.resolve(iterator.next()).then( + (value) => ({ value }), + (error) => ({ error }) + ); + await started; + await iterator.return(); + const readsAtReturn = Child.primaryStore.readCount; + release(); + const settled = await first; + assert.ifError(settled.error); + assert.strictEqual(Child.primaryStore.readCount, readsAtReturn, 'returning the parent must stop child reads'); + assert.strictEqual(txn.closeOwnedReadIterators(), 0, 'no abandoned nested cursor remains owned'); + }); + } finally { + release(); + Child.setComputedAttribute('observed', (record) => { + projected.push(record.id); + return record.score; + }); + } + }); + for (const single of [false, true]) + it(`resolves constrained grandchildren below an unconstrained relationship (${single ? 'record' : 'collection'})`, async function () { + Child.setComputedAttribute('observed', (record) => Promise.resolve(record.score)); + try { + const result = await get( + (single ? '/Parent/p1?' : '/Parent/?') + 'select(id,children{id,siblings[limit(1)&select(id,observed)]})' + ); + const row = single ? result : result[0]; + assert.deepStrictEqual( + row.children, + children + .filter((child) => child.parentId === 'p1') + .map((child) => ({ id: child.id, siblings: [{ id: 'c1', observed: 10 }] })) + ); + } finally { + Child.setComputedAttribute('observed', (record) => { + projected.push(record.id); + return record.score; + }); + } + }); + it('each half of the joined query works independently', async function () { + const expected = [ + ['c1', 'c2', 'c3', 'c4'], + ['c5', 'c6', 'c7'], + ]; + assert.deepStrictEqual(ids(await get('/Parent/?children.siblings.state=open&select(id,children{id})')), expected); + assert.deepStrictEqual( + ids(await get('/Parent/?select(id,children[siblings.score=ge=number:30&select(id,)])')), + expected + ); + }); + it('combines outer joined paths and nested relationship predicates', async function () { + const rows = await get( + '/Parent/?children.siblings.state=open&select(id,children[siblings.score=ge=number:30&select(id,)])' + ); + assert.deepStrictEqual(ids(rows), [ + ['c1', 'c2', 'c3', 'c4'], + ['c5', 'c6', 'c7'], + ]); + }); + for (const abandonTransaction of [false, true]) + it(`cancellation at the candidate yield stops upstream FK reads (${abandonTransaction ? 'transaction' : 'parent'})`, async function () { + if (!(Child.primaryStore instanceof RocksDatabase)) return this.skip(); + await Parent.put('p1', { id: 'p1', childIds: Array(4098).fill('c1') }); + try { + await transaction({}, async (txn) => { + const results = Parent.search({ + conditions: [{ attribute: 'id', value: 'p1' }], + select: [ + 'id', + { name: 'references', conditions: [{ attribute: 'state', value: 'never' }], select: ['id'] }, + ], + }); + const iterator = results[Symbol.asyncIterator](); + let readsAtReturn; + const initialReads = Child.primaryStore.readCount; + const cancelled = new Promise((resolve, reject) => { + const deadline = Date.now() + 5000; + const cancelAtYield = () => { + if (Date.now() > deadline) return reject(new Error('Candidate scan never reached its yield')); + if (Child.primaryStore.readCount - initialReads < 2048) return setImmediate(cancelAtYield); + Promise.resolve(abandonTransaction ? txn.closeOwnedReadIterators() : iterator.return()).then(() => { + readsAtReturn = Child.primaryStore.readCount; + resolve(); + }, reject); + }; + setImmediate(cancelAtYield); + }); + const first = Promise.resolve(iterator.next()); + await cancelled; + await first; + + assert.strictEqual(readsAtReturn - initialReads, 2048, 'must cancel at the first candidate yield'); + assert.strictEqual( + Child.primaryStore.readCount, + readsAtReturn, + 'no child entry reads after parent return during candidate yield' + ); + assert.strictEqual(txn.closeOwnedReadIterators(), 0); + }); + } finally { + await Parent.put('p1', { + id: 'p1', + childId: 'c1', + metadata: { label: 'kept', hidden: 'omit' }, + childIds: ['c1', 'missing', 'c2', 'c3', 'c4'], + }); + } + }); + it('refreshed cache children still satisfy the nested predicate', async function () { + if (!(Child.primaryStore instanceof RocksDatabase)) return this.skip(); + const CachedChild = table({ + database: 'nested_opposition', + table: 'CachedChild', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'state', type: 'String' }, + ], + }); + let loads = 0; + CachedChild.sourcedFrom({ + get(id) { + loads++; + return { id, state: 'closed' }; + }, + }); + const CachedParent = table({ + database: 'nested_opposition', + table: 'CachedParent', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'childId', type: 'ID' }, + { name: 'child', relationship: { from: 'childId' }, definition: { tableClass: CachedChild } }, + ], + }); + await CachedParent.put('p', { id: 'p', childId: 'cached' }); + await CachedChild.put('cached', { id: 'cached', state: 'open' }, { expiresAt: 1 }); + const rows = []; + for await (const row of CachedParent.search({ + select: ['id', { name: 'child', conditions: [{ attribute: 'state', value: 'open' }], select: ['id', 'state'] }], + })) + rows.push(row); + assert.strictEqual(loads, 1, 'must exercise source refresh'); + assert.strictEqual(rows[0].child, undefined, 'refreshed nonmatching child must be excluded'); + }); + + it('nested cache ordering retains the top-level local-entry contract', async function () { + if (!(Child.primaryStore instanceof RocksDatabase)) return this.skip(); + const C = table({ + database: 'nested_opposition_sort', + table: 'Child', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'score', type: 'Int', indexed: true }, + ], + }); + let loads = 0; + C.sourcedFrom({ + get(id) { + loads++; + return { id, score: id === 'a' ? 100 : 1 }; + }, + }); + const P = table({ + database: 'nested_opposition_sort', + table: 'Parent', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'ids', elements: { type: 'ID' } }, + { name: 'children', relationship: { from: 'ids' }, elements: { type: 'Child', definition: { tableClass: C } } }, + ], + }); + await P.put('p', { id: 'p', ids: ['a', 'b'] }); + await C.put('a', { id: 'a', score: 0 }, { expiresAt: 1 }); + await C.put('b', { id: 'b', score: 10 }, { expiresAt: 1 }); + const rows = []; + for await (const row of P.search({ + select: ['id', { name: 'children', sort: { attribute: 'score' }, limit: 1, select: ['id', 'score'] }], + })) + rows.push(row); + + assert.strictEqual(loads, 1, 'only the selected cached row is refreshed'); + assert.deepStrictEqual(rows[0].children, [{ id: 'a', score: 100 }]); + await C.put('a', { id: 'a', score: 0 }, { expiresAt: 1 }); + await C.put('b', { id: 'b', score: 10 }, { expiresAt: 1 }); + const direct = []; + for await (const row of C.search({ sort: { attribute: 'score' }, limit: 1, select: ['id', 'score'] })) + direct.push(row); + assert.deepStrictEqual(rows[0].children, direct); + }); + + it('reordering OR terms leaves projection unchanged', async function () { + const conditions = [ + { attribute: ['siblings', 'state'], value: 'open' }, + { attribute: ['siblings', 'score'], comparator: 'ge', value: 30 }, + ]; + const selection = { + name: 'children', + operator: 'or', + conditions, + select: ['id', { name: 'siblings', select: ['id'] }], + }; + const forward = await query(selection); + const reverse = await query({ ...selection, conditions: [...conditions].reverse() }); + assert.deepStrictEqual(forward[1].children[0].siblings, [{ id: 'c5' }, { id: 'c6' }, { id: 'c7' }]); + assert.deepStrictEqual(forward, reverse, 'OR operand order must not change projected values'); + }); + it('OR scalar branches leave unrelated projected relationships unrestricted', async function () { + const rows = await query({ + name: 'children', + operator: 'or', + conditions: [ + { attribute: 'state', value: 'open' }, + { attribute: ['siblings', 'score'], comparator: 'ge', value: 30 }, + ], + select: ['id', { name: 'siblings', select: ['id'] }], + }); + assert.deepStrictEqual( + rows[0].children[0].siblings, + children.filter((child) => child.parentId === 'p1').map((child) => ({ id: child.id })) + ); + assert.deepStrictEqual( + rows[0].children[1].siblings, + children.filter((child) => child.parentId === 'p1').map((child) => ({ id: child.id })) + ); + }); + it('nested predicates qualify children without implicitly narrowing their descendants', async function () { + const rows = await query({ + name: 'children', + conditions: [ + { attribute: ['siblings', 'state'], value: 'open' }, + { attribute: ['siblings', 'score'], comparator: 'ge', value: 30 }, + ], + select: ['id', { name: 'siblings', select: ['id'] }], + }); + assert.deepStrictEqual( + rows[0].children[0].siblings, + children.filter((child) => child.parentId === 'p1').map((child) => ({ id: child.id })) + ); + assert.deepStrictEqual( + rows[1].children[0].siblings, + children.filter((child) => child.parentId === 'p2').map((child) => ({ id: child.id })) + ); + }); + it('retains inherited join narrowing when the nested predicate uses OR', async function () { + const rows = await query( + { + name: 'children', + operator: 'or', + conditions: [ + { attribute: ['siblings', 'score'], comparator: 'ge', value: 30 }, + { attribute: 'state', value: 'closed' }, + ], + select: ['id', { name: 'siblings', select: ['id'] }], + }, + { conditions: [{ attribute: ['children', 'siblings', 'state'], value: 'open' }] } + ); + assert.deepStrictEqual(rows[1].children[0].siblings, [{ id: 'c6' }]); + }); + it('explicit descendant OR selections return the union of matching children', async function () { + const row = await Parent.get({ + id: 'p2', + select: [ + 'id', + { + name: 'children', + limit: 1, + select: [ + 'id', + { + name: 'siblings', + operator: 'or', + conditions: [ + { attribute: 'state', value: 'open' }, + { attribute: 'score', comparator: 'ge', value: 30 }, + ], + select: ['id'], + }, + ], + }, + ], + }); + assert.deepStrictEqual(row.children[0].siblings, [{ id: 'c6' }, { id: 'c7' }]); + }); + for (const single of [false, true]) + for (const offset of [0, 1]) + it(`refreshed nonmatches do not consume HTTP pagination (${single ? 'record' : 'collection'}, offset ${offset})`, async function () { + if (!(Child.primaryStore instanceof RocksDatabase)) return this.skip(); + const CachedChild = table({ + database: `nested_refresh_${single}_${offset}`, + table: 'Child', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'state', type: 'String' }, + { name: 'observed', computed: true }, + ], + }); + const projectedIds = []; + CachedChild.setComputedAttribute('observed', (record) => { + projectedIds.push(record.id); + return record.state; + }); + const loaded = []; + CachedChild.sourcedFrom({ + get(id) { + loaded.push(id); + return { id, state: id === 'reject' ? 'closed' : 'open' }; + }, + }); + const CachedParent = table({ + database: `nested_refresh_${single}_${offset}`, + table: 'Parent', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'ids', elements: { type: 'ID' } }, + { + name: 'children', + relationship: { from: 'ids' }, + elements: { type: 'Child', definition: { tableClass: CachedChild } }, + }, + ], + }); + await CachedParent.put('p', { id: 'p', ids: ['nonmatch', 'reject', 'first', 'second'] }); + for (const id of ['nonmatch', 'reject', 'first', 'second']) + await CachedChild.put(id, { id, state: id === 'nonmatch' ? 'closed' : 'open' }, { expiresAt: 1 }); + const resourceName = `RefreshParent_${single}_${offset}`; + resources.set(resourceName, CachedParent); + const value = await get( + `/${resourceName}/${single ? 'p' : ''}?select(id,children[state=open&limit(${offset},${offset + 1})&select(id,observed)])` + ); + const row = single ? value : value[0]; + const expected = offset ? 'second' : 'first'; + assert.deepStrictEqual(row.children, [{ id: expected, observed: 'open' }]); + assert.deepStrictEqual(projectedIds, [expected], 'offset and rejected rows must not compute output fields'); + assert.deepStrictEqual(loaded, offset ? ['reject', 'first', 'second'] : ['reject', 'first']); + }); + it('a deleted source candidate does not consume the nested limit', async function () { + if (!(Child.primaryStore instanceof RocksDatabase)) return this.skip(); + const CachedChild = table({ + database: 'nested_source_delete', + table: 'Child', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'state', type: 'String' }, + ], + }); + const loaded = []; + CachedChild.sourcedFrom({ + get(id) { + loaded.push(id); + return undefined; + }, + }); + const CachedParent = table({ + database: 'nested_source_delete', + table: 'Parent', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'ids', elements: { type: 'ID' } }, + { + name: 'children', + relationship: { from: 'ids' }, + elements: { type: 'Child', definition: { tableClass: CachedChild } }, + }, + ], + }); + await CachedParent.put('p', { id: 'p', ids: ['gone', 'kept'] }); + await CachedChild.put('gone', { id: 'gone', state: 'open' }, { expiresAt: 1 }); + await CachedChild.put('kept', { id: 'kept', state: 'open' }); + const row = await CachedParent.get({ + id: 'p', + select: [ + 'id', + { name: 'children', conditions: [{ attribute: 'state', value: 'open' }], limit: 1, select: ['id'] }, + ], + }); + assert.deepStrictEqual(row.children, [{ id: 'kept' }]); + assert.deepStrictEqual(loaded, ['gone']); + }); +}); diff --git a/unitTests/resources/query-parse.test.js b/unitTests/resources/query-parse.test.js index c80093a1e2..bc90cdebbe 100644 --- a/unitTests/resources/query-parse.test.js +++ b/unitTests/resources/query-parse.test.js @@ -174,24 +174,18 @@ describe('Parsing queries', () => { const scalarQuery = parseQuery('select(title)'); assert.strictEqual(scalarQuery.select, 'title'); }); - it('Rejects bracketed filter conditions inside select(...) with 400 SyntaxViolation', function () { + it('Preserves named bracketed filter conditions inside select(...) for schema validation', function () { for (const raw of [ 'select(children[label=child-label-c1])', 'select(id,children[label=child-label-c1])', 'select(children[label=child-label-c1],id)', ]) { - assert.throws( - () => parseQuery(raw), - (err) => { - assert.strictEqual(err.statusCode, 400); - assert.match(err.message, /conditions\/comparisons are not allowed in a property list/); - return true; - } - ); - const target = new RequestTarget(`?${raw}`); - assert.ok(target.parseError); - assert.strictEqual(target.parseError.statusCode, 400); - assert.match(target.parseError.message, /conditions\/comparisons are not allowed in a property list/); + const query = parseQuery(raw); + const child = query.select.find((property) => property.name === 'children'); + assert.strictEqual(child.conditions.length, 1); + assert.strictEqual(child.conditions[0].attribute, 'label'); + assert.strictEqual(child.conditions[0].value, 'child-label-c1'); + assert.strictEqual(new RequestTarget(`?${raw}`).parseError, undefined); } }); it('Allows bracketed non-condition operators like sort() and limit() inside select(...)', function () { diff --git a/unitTests/resources/restSelectProjectionFixes.test.js b/unitTests/resources/restSelectProjectionFixes.test.js index 8fcab6a6b3..3629767981 100644 --- a/unitTests/resources/restSelectProjectionFixes.test.js +++ b/unitTests/resources/restSelectProjectionFixes.test.js @@ -493,23 +493,26 @@ describe('REST and operations API select projection fixes', function () { }); }); - it('returns 400 across collection search and single-record GET when bracketed filter conditions appear in select()', async function () { + it('filters bracketed relationship selections across collection and single-record entry forms', async function () { for (const query of [ 'select(children[label=child-label-v1])', 'select(id,children[label=child-label-v1])', 'select(children[label=child-label-v1],id)', ]) { - const collectionRes = await restGet(`/Parent/?id=p1&${query}`); - assert.strictEqual(collectionRes.status, 400, `expected 400 for collection ?${query}`); - - const singleRes = await restGet(`/Parent/p1?${query}`); - assert.strictEqual(singleRes.status, 400, `expected 400 for single-record ?${query}`); - - const instanceTrueRes = await restGet(`/ParentInstanceTrue/p1?${query}`); - assert.strictEqual(instanceTrueRes.status, 400, `expected 400 for loadAsInstance=true ?${query}`); - - const staticFalseRes = await restGet(`/ParentStaticFalse/p1?${query}`); - assert.strictEqual(staticFalseRes.status, 400, `expected 400 for loadAsInstance=false ?${query}`); + for (const route of ['Parent', 'ParentInstanceTrue', 'ParentStaticFalse']) { + const single = await restGet(`/${route}/p1?${query}`); + assert.strictEqual(single.status, 200, `${route} ${query}`); + assert.deepStrictEqual( + single.body.children.map((child) => child.id), + ['c1'] + ); + } + const collection = await restGet(`/Parent/?id=p1&${query}`); + assert.strictEqual(collection.status, 200, query); + assert.deepStrictEqual( + collection.body[0].children.map((child) => child.id), + ['c1'] + ); } });