From ee0d9c613a2088eb27510b66556b0d8d96584227 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 30 Oct 2026 08:02:50 -0600 Subject: [PATCH 01/19] Apply SQL attribute permissions to every column reference Attribute permissions now cover JOIN ... USING columns, unqualified columns in joins (including inside ON), columns nested deeper than 100 levels, and qualifiers that do not name exactly one table in the statement. The authorization inventory resolves each column with the v2 binder's rules, now shared in sqlEngine/binder/columnScope.ts, so it checks the table the engine reads. JSON literal data and the `*` of a plain COUNT(*) are not column references. Non-super statements that read whole rows without naming a column (AlaSQL `_`, SELECT VALUE/ROW, JavaScript expressions) are refused: attribute permissions cannot apply to them, and legacy SQL's row container always carries the primary key. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RGfHqd6v2q9t9jJ26No4Jc Dispatch-Task: harper-sql-permission-walk-using-and-container --- DESIGN.md | 2 +- .../fixtures/sql-attribute-perms/config.yaml | 3 + .../sql-attribute-perms/schema.graphql | 14 + .../security/sql-attribute-perms.test.ts | 187 ++++++++ sqlEngine/binder/bind.ts | 3 +- sqlEngine/binder/columnScope.ts | 20 + sqlEngine/logical/build.ts | 11 +- sqlTranslator/DESIGN.md | 16 + sqlTranslator/sql_statement_bucket.ts | 404 +++++++----------- unitTests/sqlEngine/columnScope.test.js | 23 + .../sqlTranslator/column-attribution.test.js | 223 ++++++++++ .../sql_statement_bucket.test.js | 24 +- .../utility/operation_authorization.test.js | 26 ++ utility/errors/commonErrors.ts | 4 + utility/operation_authorization.ts | 10 + 15 files changed, 703 insertions(+), 267 deletions(-) create mode 100644 integrationTests/security/fixtures/sql-attribute-perms/config.yaml create mode 100644 integrationTests/security/fixtures/sql-attribute-perms/schema.graphql create mode 100644 integrationTests/security/sql-attribute-perms.test.ts create mode 100644 sqlEngine/binder/columnScope.ts create mode 100644 unitTests/sqlEngine/columnScope.test.js create mode 100644 unitTests/sqlTranslator/column-attribution.test.js diff --git a/DESIGN.md b/DESIGN.md index 22d5f97dfa..cc3c45c264 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -6,7 +6,7 @@ A note lives in the `DESIGN.md` of the directory that owns the code it describes ## sqlTranslator/ — legacy SQL integration -- [CommonJS date functions](sqlTranslator/DESIGN.md#commonjs-date-functions) — Both runtimes use the same CommonJS helper object, enforced by `alasqlFunctionImporter.test.js`. +- [CommonJS date functions](sqlTranslator/DESIGN.md#commonjs-date-functions) — Both runtimes use the same CommonJS helper object, enforced by `alasqlFunctionImporter.test.js`. · [SQL authorization attributes every column with the engine's own resolution rules (`sql_statement_bucket.ts`, `sqlEngine/binder/columnScope.ts`)](sqlTranslator/DESIGN.md#sql-authorization-attributes-every-column-with-the-engines-own-resolution-rules-sql_statement_bucketts-sqlenginebindercolumnscopets) — The bucket and the v2 binder share one column resolver; unresolved qualifiers and whole-row accessors are refused for non-super users. ## .github/ — CI workflows diff --git a/integrationTests/security/fixtures/sql-attribute-perms/config.yaml b/integrationTests/security/fixtures/sql-attribute-perms/config.yaml new file mode 100644 index 0000000000..8005c9df90 --- /dev/null +++ b/integrationTests/security/fixtures/sql-attribute-perms/config.yaml @@ -0,0 +1,3 @@ +graphqlSchema: + files: '*.graphql' +rest: true diff --git a/integrationTests/security/fixtures/sql-attribute-perms/schema.graphql b/integrationTests/security/fixtures/sql-attribute-perms/schema.graphql new file mode 100644 index 0000000000..21fd6f69ce --- /dev/null +++ b/integrationTests/security/fixtures/sql-attribute-perms/schema.graphql @@ -0,0 +1,14 @@ +# The test role reads both tables but is denied `ssn` on ColPermLeft and the primary key and +# `code` on ColPermRight. +type ColPermLeft @table @export { + id: ID @primaryKey + label: String @indexed + ssn: String @indexed +} + +type ColPermRight @table @export { + id: ID @primaryKey + label: String @indexed + ssn: String @indexed + code: String @indexed +} diff --git a/integrationTests/security/sql-attribute-perms.test.ts b/integrationTests/security/sql-attribute-perms.test.ts new file mode 100644 index 0000000000..e90c62a8f1 --- /dev/null +++ b/integrationTests/security/sql-attribute-perms.test.ts @@ -0,0 +1,187 @@ +/** + * Attribute-level read permissions apply to every way a SQL statement can reference a column: + * `JOIN ... USING (col)`, an unqualified column inside `JOIN ... ON`, and row accessors that name + * no column (`SELECT VALUE`, `_`, AlaSQL JavaScript expressions). The authorization check runs + * before the engine router, so legacy, auto and new must all give the same answer. + * + * npm run build && npm run test:integration -- "integrationTests/security/sql-attribute-perms.test.ts" + */ +import { suite, test, before, after } from 'node:test'; +import { deepStrictEqual, notStrictEqual, ok, strictEqual } from 'node:assert'; +import { resolve } from 'node:path'; + +import { setupHarperWithFixture, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; +// @ts-expect-error utils/client.mjs has no type declarations; runtime resolves fine +import { createApiClient, createHeaders } from '../apiTests/utils/client.mjs'; + +const FIXTURE_PATH = resolve(import.meta.dirname, 'fixtures/sql-attribute-perms'); +const skipSuite = process.env.HARPER_RUNTIME === 'bun' || process.platform === 'win32'; + +const USER = { username: 'colperm_reader', password: 'Colperm-reader-pw-1!' }; +const ROLE = 'colperm_reader_role'; +const RIGHT_PK = 'r-primary-key-1'; + +type Client = ReturnType; + +function attr(attribute_name: string, read: boolean) { + return { attribute_name, read, insert: false, update: false }; +} + +function readOnly(attribute_permissions: ReturnType[]) { + return { read: true, insert: false, update: false, delete: false, attribute_permissions }; +} + +// sql.engine=new answers only indexed queries, so every statement carries an indexed predicate. +const USING_DENIED = `SELECT l.label FROM data.ColPermLeft l JOIN data.ColPermRight r USING (ssn) WHERE l.id = 'l1'`; +const USING_DENIED_SELF = `SELECT a.label FROM data.ColPermLeft a JOIN data.ColPermLeft b USING (ssn) WHERE a.id = 'l1'`; +const USING_PERMITTED = `SELECT l.label, r.label AS rl FROM data.ColPermLeft l JOIN data.ColPermRight r USING (label) WHERE l.id = 'l1'`; +const ON_PERMITTED = `SELECT l.label, r.label AS rl FROM data.ColPermLeft l JOIN data.ColPermRight r ON l.label = r.label WHERE l.id = 'l1'`; + +for (const sqlEngine of ['legacy', 'auto', 'new']) { + suite( + `SQL attribute permissions cover every column reference [sql.engine=${sqlEngine}]`, + { skip: skipSuite }, + (ctx: ContextWithHarper) => { + let client: Client; + let readerHeaders: Record; + + const asReader = (sql: string) => client.reqAs(readerHeaders).send({ operation: 'sql', sql }); + + function assertRefused(r: { status: number; body: unknown }, sql: string) { + strictEqual(r.status, 403, `${sql} -> ${r.status} ${JSON.stringify(r.body).slice(0, 300)}`); + ok(!JSON.stringify(r.body).includes(RIGHT_PK), 'the refusal must not carry the primary key value'); + } + + before(async () => { + await setupHarperWithFixture(ctx, FIXTURE_PATH, { config: {}, env: { HARPER_SQL_ENGINE: sqlEngine } }); + client = createApiClient(ctx.harper); + readerHeaders = createHeaders(USER.username, USER.password); + + await client + .req() + .send({ + operation: 'add_role', + role: ROLE, + permission: { + super_user: false, + data: { + tables: { + ColPermLeft: readOnly([attr('id', true), attr('label', true), attr('ssn', false)]), + ColPermRight: readOnly([ + attr('id', false), + attr('label', true), + attr('ssn', true), + attr('code', false), + ]), + }, + }, + }, + }) + .expect(200); + await client + .req() + .send({ operation: 'add_user', role: ROLE, ...USER, active: true }) + .expect(200); + await client + .req() + .send({ + operation: 'insert', + schema: 'data', + table: 'ColPermLeft', + records: [ + { id: 'l1', label: 'shared', ssn: '111' }, + { id: 'l2', label: 'other', ssn: '222' }, + ], + }) + .expect(200); + await client + .req() + .send({ + operation: 'insert', + schema: 'data', + table: 'ColPermRight', + records: [ + { id: RIGHT_PK, label: 'shared', ssn: '111', code: 'c1' }, + { id: 'r-primary-key-2', label: 'unmatched', ssn: '333', code: 'c2' }, + ], + }) + .expect(200); + }); + + after(async () => { + await teardownHarper(ctx); + }); + + test('JOIN USING on a read-denied column is refused, as the equivalent ON is', async () => { + assertRefused(await asReader(USING_DENIED), USING_DENIED); + const on = `SELECT l.label FROM data.ColPermLeft l JOIN data.ColPermRight r ON l.ssn = r.ssn WHERE l.id = 'l1'`; + assertRefused(await asReader(on), on); + }); + + test('self-join USING on a read-denied column is refused', async () => { + assertRefused(await asReader(USING_DENIED_SELF), USING_DENIED_SELF); + }); + + test('an unqualified read-denied column inside JOIN ON is refused', async () => { + const sql = `SELECT l.label FROM data.ColPermLeft l JOIN data.ColPermRight r ON l.label = r.label AND code = 'c1' WHERE l.id = 'l1'`; + assertRefused(await asReader(sql), sql); + }); + + test('a column qualified by a name that is not a table in the statement is refused', async () => { + const sql = `SELECT bogus.ssn FROM data.ColPermLeft WHERE id = 'l1'`; + assertRefused(await asReader(sql), sql); + }); + + test('a read-denied column nested deep inside an expression is refused', async () => { + const nested = 'UPPER('.repeat(80) + 'ssn' + ')'.repeat(80); + const sql = `SELECT label FROM data.ColPermLeft WHERE id = 'l1' AND ${nested} = '111'`; + assertRefused(await asReader(sql), sql); + }); + + test("a qualifier that is one table's alias and another's base name is refused", async () => { + // v2 reads the aliased ColPermRight's read-denied primary key; ColPermLeft.id is readable. + const sql = `SELECT ColPermLeft.id FROM data.ColPermLeft l JOIN data.ColPermRight ColPermLeft ON l.label = ColPermLeft.label WHERE l.id = 'l1'`; + assertRefused(await asReader(sql), sql); + }); + + test('row accessors that name no column are refused when the primary key is read-denied', async () => { + for (const sql of [ + `SELECT VALUE FROM data.ColPermRight r WHERE r.label = 'shared'`, + `SELECT _->id AS k FROM data.ColPermRight r WHERE r.label = 'shared'`, + "SELECT ``p`` AS rec FROM data.ColPermRight r WHERE r.label = 'shared'", + ]) { + assertRefused(await asReader(sql), sql); + } + }); + + test('JOIN USING on a permitted column is authorized and matches the equivalent ON', async () => { + const using = await asReader(USING_PERMITTED); + notStrictEqual(using.status, 403, `${USING_PERMITTED} -> ${JSON.stringify(using.body)}`); + // Legacy SQL rejects every USING clause, super user included, so only the v2 engine + // can show row parity with ON. + if (sqlEngine !== 'legacy') { + strictEqual(using.status, 200, JSON.stringify(using.body)); + const on = await asReader(ON_PERMITTED); + strictEqual(on.status, 200, JSON.stringify(on.body)); + deepStrictEqual(using.body, on.body); + deepStrictEqual(using.body, [{ label: 'shared', rl: 'shared' }]); + } + }); + + test('super user is unaffected', async () => { + const r = await client.req().send({ operation: 'sql', sql: USING_DENIED }); + if (sqlEngine === 'legacy') { + notStrictEqual(r.status, 403, JSON.stringify(r.body)); + } else { + strictEqual(r.status, 200, JSON.stringify(r.body)); + deepStrictEqual(r.body, [{ label: 'shared' }]); + } + const row = await client + .req() + .send({ operation: 'sql', sql: `SELECT id FROM data.ColPermRight WHERE label = 'shared'` }); + strictEqual(row.status, 200, JSON.stringify(row.body)); + deepStrictEqual(row.body, [{ id: RIGHT_PK }]); + }); + } + ); +} diff --git a/sqlEngine/binder/bind.ts b/sqlEngine/binder/bind.ts index 0ad51fffb1..2bac79416a 100644 --- a/sqlEngine/binder/bind.ts +++ b/sqlEngine/binder/bind.ts @@ -28,6 +28,7 @@ import type { } from '../parser/ast.ts'; import { EngineUnsupportedError } from '../errors.ts'; import { _setDatabasesLoader, loadDatabases, findDatabasesWithTable } from './defaultDatabase.ts'; +import { declaringTables } from './columnScope.ts'; interface AttributeInfo { name: string; @@ -263,7 +264,7 @@ function resolveColumns(expr: ExprNode, scope: BoundTable[]): void { return; } // Unqualified: bind to the unique table that declares this attribute. - const owners = scope.filter((t) => t.attributes.some((a) => a.name === expr.name)); + const owners = declaringTables(expr.name, scope); if (owners.length === 1) { expr.table = owners[0].effectiveAlias; } else if (owners.length > 1) { diff --git a/sqlEngine/binder/columnScope.ts b/sqlEngine/binder/columnScope.ts new file mode 100644 index 0000000000..9869064bff --- /dev/null +++ b/sqlEngine/binder/columnScope.ts @@ -0,0 +1,20 @@ +// Shared with SQL authorization (sqlTranslator/sql_statement_bucket.ts) so a column is authorized +// against the table the engine reads it from. + +export interface ScopeTable { + table: string; + effectiveAlias: string; + attributes: readonly { name: string }[]; +} + +export function declaringTables(name: string, scope: readonly T[]): T[] { + return scope.filter((t) => t.attributes.some((a) => a.name === name)); +} + +/** The left operand of `JOIN scope[rightIdx] USING (name)`: the nearest preceding declarer, else FROM. */ +export function usingLeftOwner(name: string, scope: readonly T[], rightIdx: number): T { + for (let i = rightIdx - 1; i >= 0; i--) { + if (scope[i].attributes.some((a) => a.name === name)) return scope[i]; + } + return scope[0]; +} diff --git a/sqlEngine/logical/build.ts b/sqlEngine/logical/build.ts index cfb50e0de1..d647fd8001 100644 --- a/sqlEngine/logical/build.ts +++ b/sqlEngine/logical/build.ts @@ -21,6 +21,7 @@ import type { ExprNode, JoinNode, ProjectionNode, SelectNode, SortNode, Statemen import type { BoundSelect, BoundTable } from '../binder/bind.ts'; import { normalizePredicate } from '../optimizer/rules/predicateNormalize.ts'; import { EngineUnsupportedError } from '../errors.ts'; +import { usingLeftOwner } from '../binder/columnScope.ts'; export function buildLogicalPlan(stmt: StatementNode): LogicalPlan { if (stmt.kind !== 'select') { @@ -231,18 +232,10 @@ function usingToOn(join: JoinNode, scope: BoundTable[], rightIdx: number): ExprN if (!join.using || join.using.length === 0) return undefined; const rightAlias = scope[rightIdx].effectiveAlias; const eqs: ExprNode[] = join.using.map((col) => { - // Left owner: nearest preceding table that declares the column. - let leftAlias = scope[0].effectiveAlias; - for (let i = rightIdx - 1; i >= 0; i--) { - if (scope[i].attributes.some((a) => a.name === col)) { - leftAlias = scope[i].effectiveAlias; - break; - } - } return { kind: 'binop', op: '=', - left: { kind: 'column', table: leftAlias, name: col }, + left: { kind: 'column', table: usingLeftOwner(col, scope, rightIdx).effectiveAlias, name: col }, right: { kind: 'column', table: rightAlias, name: col }, }; }); diff --git a/sqlTranslator/DESIGN.md b/sqlTranslator/DESIGN.md index d53dbe39dd..eb853906bf 100644 --- a/sqlTranslator/DESIGN.md +++ b/sqlTranslator/DESIGN.md @@ -3,3 +3,19 @@ ## CommonJS date functions `alasqlFunctionImporter.ts` imports the CommonJS date-helper object by default; native namespace imports did not expose its object-literal properties in [#562](https://github.com/HarperFast/harper/pull/562). `unitTests/sqlTranslator/alasqlFunctionImporter.test.js` verifies the registered date functions in both test runtimes. + +## SQL authorization attributes every column with the engine's own resolution rules (`sql_statement_bucket.ts`, `sqlEngine/binder/columnScope.ts`) + +`verifyPermsAST` checks attribute permissions only for the attributes `sqlStatementBucket` records per table, and it runs once, before the router picks legacy or v2, so the bucket alone decides what every engine may read. A column the bucket misses, or records against the wrong table, is read unchecked. + +So the bucket attributes columns with the binder's own rules, which live in `columnScope.ts` and are called by `bind.ts`, `logical/build.ts` and the bucket alike: + +- A qualified column is attributed only when exactly one table in the statement (not only the tables an ON has joined so far) has the qualifier as its alias or base name. v2 prefers an alias over a base name, while legacy SQL matches either, so only then do both engines read the same table. Any other qualifier is reported by `getUnresolvedColumnRefs()`, and non-super statements are refused. +- An unqualified column in a join goes to every in-scope table that declares it. Each JOIN ON only sees the tables joined so far. If no table declares the column, it stays on the FROM table, where a role with attribute permissions is refused because the attribute is not found. +- `USING (c)` records `c` on the joined table and on `usingLeftOwner` (the nearest preceding declarer, else FROM): the same two operands v2 compares. +- When the registry cannot describe a table, unqualified and USING columns go to every candidate table, so the check fails closed. +- Every walk uses `astNodes()`: iterative, with no depth limit, and visiting shared nodes once. `recursive-iterator` silently stopped at depth 100, so a column nested in 80 function calls went unchecked, and it threw on the subquery node AlaSQL shares between `where` and `exists`. + +Row accessors that name no column, AlaSQL's `_` and JavaScript expressions, are refused for non-super users by `getRowAccessors()`. Legacy SQL's row container always carries the primary key (`dataLayer/SQLSearch.ts` keys rows by it), so a whole-row read would expose it whatever the role allows. The JavaScript refusal stays unconditional for non-super users, even if the `_` refusal is ever relaxed for roles without attribute permissions. + +Enforced by `unitTests/sqlTranslator/column-attribution.test.js`, `unitTests/sqlEngine/columnScope.test.js` and `integrationTests/security/sql-attribute-perms.test.ts`. diff --git a/sqlTranslator/sql_statement_bucket.ts b/sqlTranslator/sql_statement_bucket.ts index 48e336761e..3bfb191b3e 100644 --- a/sqlTranslator/sql_statement_bucket.ts +++ b/sqlTranslator/sql_statement_bucket.ts @@ -5,12 +5,12 @@ **/ import alasql from 'alasql'; -import RecursiveIterator from 'recursive-iterator'; import _harperLogger from '../utility/logging/harper_logger.ts'; const harperLogger = _harperLogger; import * as hdbUtils from '../utility/common_utils.ts'; import * as terms from '../utility/hdbTerms.ts'; import { findDatabasesWithTable, loadDatabaseRegistry } from '../sqlEngine/binder/defaultDatabase.ts'; +import { declaringTables, usingLeftOwner, type ScopeTable } from '../sqlEngine/binder/columnScope.ts'; class sqlStatementBucket { ast: any; @@ -18,6 +18,7 @@ class sqlStatementBucket { table_lookup: any; schema_lookup: any; table_to_schema_lookup: any; + unresolved_columns: Set; constructor(ast) { this.ast = ast; // affectedAttributes stores a table and it's attributes as a Map [schema, Map[table, [attributesArray]]]. @@ -25,12 +26,14 @@ class sqlStatementBucket { this.table_lookup = new Map(); this.schema_lookup = new Map(); this.table_to_schema_lookup = new Map(); + this.unresolved_columns = new Set(); interpretAST( this.ast, this.affected_attributes, this.table_lookup, this.schema_lookup, - this.table_to_schema_lookup + this.table_to_schema_lookup, + this.unresolved_columns ); } @@ -126,6 +129,28 @@ class sqlStatementBucket { return unauthorized; } + /** + * Column references whose table qualifier names no table in scope, or several. They were not + * attributed to any table, so the caller must refuse the statement. + */ + getUnresolvedColumnRefs(): string[] { + return [...this.unresolved_columns]; + } + + /** + * Expressions that read a whole row instead of naming a column: AlaSQL's `_` (`SELECT VALUE`, + * `ROW`, `x._`, `_->attr`) and JavaScript expressions. Attribute permissions cannot apply to them, + * and legacy SQL's row container always carries the primary key. + */ + getRowAccessors(): string[] { + const accessors = []; + for (const node of astNodes(this.ast)) { + if (node instanceof (alasql as any).yy.Column && node.columnid === '_') accessors.push('_'); + else if (node instanceof (alasql as any).yy.JavaScript) accessors.push('JavaScript expression'); + } + return accessors; + } + /** *When a SELECT * is included in the AST for a non-SU, we need to convert the star into the specific attributes the * user has READ permissions @@ -206,6 +231,52 @@ function filterReadRestrictedAttrs(attrPerms: any[]) { return attrPerms.filter((perm) => perm[terms.PERMS_CRUD_ENUM.READ]); } +/** + * Every object node under `root`, each once. Iterative and unbounded: an expression nested arbitrarily + * deep must still be seen, and AlaSQL shares one subquery node between two fields (EXISTS). + */ +function* astNodes(root: any): Generator { + const seen = new Set(); + const pending = root !== null && typeof root === 'object' ? [root] : []; + while (pending.length > 0) { + const node = pending.pop(); + if (seen.has(node)) continue; + seen.add(node); + yield node; + for (const key in node) { + if (!Object.hasOwn(node, key)) continue; + const child = node[key]; + if (child !== null && typeof child === 'object') pending.push(child); + } + } +} + +/** + * Column references under `root`. Inside a JSON literal only real Column nodes count; its own objects are + * data. The `*` of a plain COUNT(*) names no attribute. + */ +function* columnNodes(root: any): Generator { + const notColumns = new Set(); + for (const node of astNodes(root)) { + if (node instanceof (alasql as any).yy.Json) { + // Only the literal's own plain objects and arrays: nested AST nodes, a nested JSON literal + // included, are reached by the outer walk, so each is marked once. + const pending = [node.value]; + while (pending.length > 0) { + const value = pending.pop(); + if (value === null || typeof value !== 'object' || notColumns.has(value)) continue; + if (!Array.isArray(value) && Object.getPrototypeOf(value) !== Object.prototype) continue; + notColumns.add(value); + for (const child of Object.values(value)) pending.push(child); + } + } else if (node.expression && isRowCountWildcard(node, node.expression)) { + notColumns.add(node.expression); + } else if (node.columnid && !notColumns.has(node)) { + yield node; + } + } +} + // FIRST(*), ARRAY(*) and COUNT(DISTINCT *) read the value of an attribute literally named `*`, so they stay collected. function isRowCountWildcard(parent: any, node: any): boolean { return node.columnid === '*' && parent?.aggregatorid === 'COUNT' && parent.expression === node && !parent.distinct; @@ -216,10 +287,11 @@ function interpretAST( affectedAttributes: any, tableLookup: any, schemaLookup: any, - tableToSchemaLookup: any + tableToSchemaLookup: any, + unresolvedColumns: Set ) { qualifyBareTableRefs(ast); - getRecordAttributesAST(ast, affectedAttributes, tableLookup, schemaLookup, tableToSchemaLookup); + getRecordAttributesAST(ast, affectedAttributes, tableLookup, schemaLookup, tableToSchemaLookup, unresolvedColumns); } interface TableTargets { @@ -389,7 +461,8 @@ function getRecordAttributesAST( affectedAttributes: any, tableLookup: any, schemaLookup: any, - tableToSchemaLookup: any + tableToSchemaLookup: any, + unresolvedColumns?: Set ) { if (!ast) { harperLogger.info(`getRecordAttributesAST: invalid SQL syntax tree`); @@ -400,7 +473,7 @@ function getRecordAttributesAST( if (ast instanceof (alasql as any).yy.Insert) { getInsertAttributes(ast, affectedAttributes, tableLookup); } else if (ast instanceof (alasql as any).yy.Select) { - getSelectAttributes(ast, affectedAttributes, tableLookup, schemaLookup, tableToSchemaLookup); + getSelectAttributes(ast, affectedAttributes, tableLookup, schemaLookup, tableToSchemaLookup, unresolvedColumns); } else if (ast instanceof (alasql as any).yy.Update) { getUpdateAttributes(ast, affectedAttributes, tableLookup); } else if (ast instanceof (alasql as any).yy.Delete) { @@ -410,35 +483,49 @@ function getRecordAttributesAST( } } +interface AttributionTable extends ScopeTable { + databaseid: string; + /** False when the registry could not describe the table; attribution then fails closed. */ + attributesKnown: boolean; +} + /** - * Retrieve the schemas, tables, and attributes from the source Select AST. - * - * @param ast - SQL command converted to an AST - * @param affectedAttributes - A map containing attributes affected by the statement. Defined as [schema, Map[table, [attributesArray]]]. - * @param tableLookup - A map that will be filled in. This map contains alias to table definitions as [alias, tableName]. - */ -/** - * The database a column's table belongs to. - * - * `tableToSchemaLookup` is keyed by alias and by base table name, so it is the only mapping that - * resolves an unaliased table in a cross-database statement. Falling back to the FROM table's - * database (as the code did before schema-unqualified names could resolve to different databases) - * silently attributes a joined table's columns to the wrong database, and they then get dropped - * without an attribute-level permission check. + * FROM entries then JOIN targets, in the order the engines bind them. A derived JOIN keeps its slot + * (so later ON/USING indexes line up) but records nothing — getTableTargets() refuses it. */ -function resolveColumnSchema(tableName: any, schemaLookup: any, tableToSchemaLookup: any, fallbackSchema: any) { - if (!tableName) return fallbackSchema; - if (tableToSchemaLookup?.has(tableName)) return tableToSchemaLookup.get(tableName); - if (schemaLookup?.has(tableName)) return schemaLookup.get(tableName); - return fallbackSchema; +function getAttributionScope(ast: any): AttributionTable[] { + const entries = [...ast.from, ...(ast.joins ?? []).map((join) => join?.table ?? { as: join?.as })]; + const scope: AttributionTable[] = entries.map((ref) => ({ + databaseid: ref.databaseid, + table: ref.tableid ?? '', + effectiveAlias: ref.as ?? ref.tableid ?? '', + attributes: [], + attributesKnown: false, + })); + if (scope.length < 2) return scope; + const databases = loadDatabaseRegistry(); + for (const entry of scope) { + const tables = entry.databaseid && databases?.[entry.databaseid]; + const table: any = tables && Object.hasOwn(tables, entry.table) ? tables[entry.table] : undefined; + if (Array.isArray(table?.attributes)) { + entry.attributes = table.attributes; + entry.attributesKnown = true; + } + } + return scope; } +/** + * Attributes every column reference in a SELECT to the table the engines read it from, using the v2 + * binder's rules (sqlEngine/binder/columnScope.ts). See sqlTranslator/DESIGN.md. + */ function getSelectAttributes( ast: any, affectedAttributes: any, tableLookup: any, schemaLookup: any, - tableToSchemaLookup: any + tableToSchemaLookup: any, + unresolvedColumns: Set = new Set() ) { if (!ast) { harperLogger.info(`getSelectAttributes: invalid SQL syntax tree`); @@ -447,8 +534,7 @@ function getSelectAttributes( if (!ast.from || ast.from[0] === undefined) { return; } - let schema = ast.from[0].databaseid; - if (hdbUtils.isEmptyOrZeroLength(schema)) { + if (hdbUtils.isEmptyOrZeroLength(ast.from[0].databaseid)) { harperLogger.error('No schema specified'); return; } @@ -470,212 +556,55 @@ function getSelectAttributes( }); } - const iterator = new RecursiveIterator(ast.columns); - for (let { node, parent } of iterator) { - if (node && node.columnid && !isRowCountWildcard(parent, node)) { - let tableName = node.tableid; - // tableToSchemaLookup is keyed by alias AND by base table name, so it resolves - // `SELECT vault.Secret.ssn FROM data.Public JOIN vault.Secret` where schemaLookup — which - // only ever holds aliases — would fall back to the FROM table's database and drop the - // column, skipping its attribute-level permission check. - const columnSchema = resolveColumnSchema(tableName, schemaLookup, tableToSchemaLookup, schema); - - if (!tableName) { - tableName = ast.from[0].tableid; - } - - const schemaTables = affectedAttributes.get(columnSchema); - if (!schemaTables) { - // The column's table was never recorded — an unresolvable reference alongside a - // resolvable one. getUnauthorizedTableRefs() reports it and the statement is denied. - harperLogger.info(`schema for table ${tableName} not resolved; skipping its attributes.`); - continue; - } - - if (!schemaTables.has(tableName)) { - if (!tableLookup.has(tableName)) { - harperLogger.info(`table specified as ${tableName} not found.`); - return; - } else { - tableName = tableLookup.get(tableName); - } - } - - if (schemaTables.get(tableName).indexOf(node.columnid) < 0) { - schemaTables.get(tableName).push(node.columnid); - } - } - } - - // It's important to iterate through the WHERE clause in case there are other columns that are not included in - // the SELECT clause - if (ast.where) { - const iterator = new RecursiveIterator(ast.where); - const fromTable = ast.from[0].tableid; - - for (let { node, parent } of iterator) { - if (node && node.columnid && !isRowCountWildcard(parent, node)) { - let table = node.tableid ? node.tableid : fromTable; - - const whereSchema = resolveColumnSchema(node.tableid, schemaLookup, tableToSchemaLookup, schema); - const schemaTables = affectedAttributes.get(whereSchema); - if (!schemaTables) { - harperLogger.info(`schema for table ${table} not resolved; skipping its attributes.`); - continue; - } - - if (!schemaTables.has(table)) { - if (!tableLookup.has(table)) { - harperLogger.info(`table specified as ${table} not found.`); - continue; - } else { - table = tableLookup.get(table); - } - } - //We need to check to ensure this columnid wasn't already set in the Map - if (schemaTables.get(table).indexOf(node.columnid) < 0) { - schemaTables.get(table).push(node.columnid); - } - } - } - } - - // GROUP BY and HAVING can reference an attribute the SELECT list, WHERE, and JOIN ON never - // mention (`SELECT COUNT(*) FROM t GROUP BY ssn`, `... HAVING ssn = 'x'`) — without recording - // it here, checkAttributePerms() never sees that reference and a role denied READ on the - // attribute can still observe its distribution via the grouped/filtered result. - if (ast.group) { - const iterator = new RecursiveIterator(ast.group); - const fromTable = ast.from[0].tableid; + const scope = getAttributionScope(ast); - for (let { node, parent } of iterator) { - if (node && node.columnid && !isRowCountWildcard(parent, node)) { - let table = node.tableid ? node.tableid : fromTable; - - const groupSchema = resolveColumnSchema(node.tableid, schemaLookup, tableToSchemaLookup, schema); - const schemaTables = affectedAttributes.get(groupSchema); - if (!schemaTables) { - harperLogger.info(`schema for table ${table} not resolved; skipping its attributes.`); - continue; - } + const record = (entry: AttributionTable, columnid: string) => { + const attributes = affectedAttributes.get(entry.databaseid)?.get(entry.table); + if (attributes && !attributes.includes(columnid)) attributes.push(columnid); + }; - if (!schemaTables.has(table)) { - if (!tableLookup.has(table)) { - harperLogger.info(`table specified as ${table} not found.`); - continue; - } else { - table = tableLookup.get(table); - } - } - if (schemaTables.get(table).indexOf(node.columnid) < 0) { - schemaTables.get(table).push(node.columnid); - } - } + const recordColumn = (node: any, clauseScope: AttributionTable[]) => { + if (node.tableid) { + // v2 prefers an alias over a base table name; legacy SQL matches either, against every table in + // the statement. Only a qualifier exactly one table claims means the same table to both. + const claimants = scope.filter((entry) => entry.effectiveAlias === node.tableid || entry.table === node.tableid); + if (claimants.length === 1) record(claimants[0], node.columnid); + else unresolvedColumns.add(`${node.tableid}.${node.columnid}`); + return; } - } - - if (ast.having) { - const iterator = new RecursiveIterator(ast.having); - const fromTable = ast.from[0].tableid; - - for (let { node, parent } of iterator) { - if (node && node.columnid && !isRowCountWildcard(parent, node)) { - let table = node.tableid ? node.tableid : fromTable; - - const havingSchema = resolveColumnSchema(node.tableid, schemaLookup, tableToSchemaLookup, schema); - const schemaTables = affectedAttributes.get(havingSchema); - if (!schemaTables) { - harperLogger.info(`schema for table ${table} not resolved; skipping its attributes.`); - continue; - } - - if (!schemaTables.has(table)) { - if (!tableLookup.has(table)) { - harperLogger.info(`table specified as ${table} not found.`); - continue; - } else { - table = tableLookup.get(table); - } - } - if (schemaTables.get(table).indexOf(node.columnid) < 0) { - schemaTables.get(table).push(node.columnid); - } - } + // Wildcards stay with the FROM table; updateAttributeWildcardsForRolePerms() expands them there. + if (clauseScope.length === 1 || terms.SEARCH_WILDCARDS.includes(node.columnid)) { + record(scope[0], node.columnid); + return; } - } - - // It's important to also iterate through the JOIN clause in case there are other columns that are not included in - // the SELECT clause - if (ast.joins) { - ast.joins.forEach((join) => { - const iterator = new RecursiveIterator(join.on); - - for (let { node, parent } of iterator) { - if (node && node.columnid && !isRowCountWildcard(parent, node)) { - let table = node.tableid; - let schema = tableToSchemaLookup.get(table); - - const schemaTables = affectedAttributes.get(schema); - if (!schemaTables) { - harperLogger.info(`schema for table ${table} not resolved; skipping its attributes.`); - continue; - } - - if (!schemaTables.has(table)) { - if (!tableLookup.has(table)) { - harperLogger.info(`table specified as ${table} not found.`); - continue; - } else { - table = tableLookup.get(table); - } - } - //We need to check to ensure this columnid wasn't already set in the Map - if (schemaTables.get(table).indexOf(node.columnid) < 0) { - schemaTables.get(table).push(node.columnid); - } - } - } - }); - } - - // It's important to iterate through the ORDER clause in case there are other columns that are not included in - // the SELECT clause with wildcard - if (ast.order) { - const orderIterator = new RecursiveIterator(ast.order); - for (let { node, parent } of orderIterator) { - if (node && node.columnid && !isRowCountWildcard(parent, node)) { - let tableName = node.tableid; - // tableToSchemaLookup is keyed by alias AND by base table name — schemaLookup alone - // (aliases only) silently fell back to the FROM table's schema for an unaliased joined - // table, misattributing its column to the wrong database and dropping it from that - // table's attribute list without a permission check. - const orderSchema = resolveColumnSchema(tableName, schemaLookup, tableToSchemaLookup, schema); - - if (!tableName) { - tableName = ast.from[0].tableid; - } - - const schemaTables = affectedAttributes.get(orderSchema); - if (!schemaTables) { - harperLogger.info(`schema for table ${tableName} not resolved; skipping its attributes.`); - continue; - } + const owners = clauseScope.every((entry) => entry.attributesKnown) + ? declaringTables(node.columnid, clauseScope) + : clauseScope; + if (owners.length === 0) record(scope[0], node.columnid); + for (const owner of owners) record(owner, node.columnid); + }; - if (!schemaTables.has(tableName)) { - if (!tableLookup.has(tableName)) { - harperLogger.info(`table specified as ${tableName} not found.`); - return; - } else { - tableName = tableLookup.get(tableName); - } - } + const recordColumns = (tree: any, clauseScope: AttributionTable[]) => { + for (const node of columnNodes(tree)) recordColumn(node, clauseScope); + }; - if (schemaTables.get(tableName).indexOf(node.columnid) < 0) { - schemaTables.get(tableName).push(node.columnid); - } - } + recordColumns(ast.columns, scope); + recordColumns(ast.where, scope); + recordColumns(ast.group, scope); + recordColumns(ast.having, scope); + recordColumns(ast.order, scope); + (ast.joins ?? []).forEach((join, index) => { + const joinedIdx = ast.from.length + index; + recordColumns(join?.on, scope.slice(0, joinedIdx + 1)); + for (const column of join?.using ?? []) { + const columnid = column?.columnid; + if (!columnid) continue; + record(scope[joinedIdx], columnid); + const left = scope.slice(0, joinedIdx); + if (left.every((entry) => entry.attributesKnown)) record(usingLeftOwner(columnid, scope, joinedIdx), columnid); + else for (const entry of left) record(entry, columnid); } - } + }); } /** @@ -689,15 +618,12 @@ function getUpdateAttributes(ast: any, affectedAttributes: any, tableLookup: any harperLogger.info(`getUpdateAttributes: invalid SQL syntax tree`); return; } - let iterator = new RecursiveIterator(ast.columns); let schema = ast.table.databaseid; addSchemaTableToMap(ast.table, affectedAttributes, tableLookup); - for (let { node } of iterator) { - if (node && node.columnid) { - pushAttribute(ast.table.tableid, schema, node.columnid, affectedAttributes, tableLookup); - } + for (const node of columnNodes(ast.columns)) { + pushAttribute(ast.table.tableid, schema, node.columnid, affectedAttributes, tableLookup); } } @@ -712,15 +638,12 @@ function getDeleteAttributes(ast: any, affectedAttributes: any, tableLookup: any harperLogger.info(`getDeleteAttributes: invalid SQL syntax tree`); return; } - let iterator = new RecursiveIterator(ast.where); let schema = ast.table.databaseid; addSchemaTableToMap(ast.table, affectedAttributes, tableLookup); - for (let { node } of iterator) { - if (node && node.columnid) { - pushAttribute(ast.table.tableid, schema, node.columnid, affectedAttributes, tableLookup); - } + for (const node of columnNodes(ast.where)) { + pushAttribute(ast.table.tableid, schema, node.columnid, affectedAttributes, tableLookup); } } @@ -735,15 +658,12 @@ function getInsertAttributes(ast: any, affectedAttributes: any, tableLookup: any harperLogger.info(`getInsertAttributes: invalid SQL syntax tree`); return; } - let iterator = new RecursiveIterator(ast.columns); let schema = ast.into.databaseid; addSchemaTableToMap(ast.into, affectedAttributes, tableLookup); - for (let { node } of iterator) { - if (node && node.columnid) { - pushAttribute(ast.into.tableid, schema, node.columnid, affectedAttributes, tableLookup); - } + for (const node of columnNodes(ast.columns)) { + pushAttribute(ast.into.tableid, schema, node.columnid, affectedAttributes, tableLookup); } } diff --git a/unitTests/sqlEngine/columnScope.test.js b/unitTests/sqlEngine/columnScope.test.js new file mode 100644 index 0000000000..3e5aa61c48 --- /dev/null +++ b/unitTests/sqlEngine/columnScope.test.js @@ -0,0 +1,23 @@ +'use strict'; + +const assert = require('assert'); +const { declaringTables, usingLeftOwner } = require('#src/sqlEngine/binder/columnScope'); + +const table = (name, alias, ...attributes) => ({ + table: name, + effectiveAlias: alias ?? name, + attributes: attributes.map((attribute) => ({ name: attribute })), +}); + +describe('sqlEngine columnScope', () => { + it('declaringTables lists every table that declares the attribute', () => { + const scope = [table('L', 'a', 'ssn'), table('R', 'b', 'code'), table('C', 'c', 'ssn')]; + assert.deepStrictEqual(declaringTables('ssn', scope), [scope[0], scope[2]]); + }); + + it('usingLeftOwner picks the nearest preceding declarer, else the FROM table', () => { + const scope = [table('L', 'a', 'ssn'), table('R', 'b', 'ssn'), table('C', 'c', 'ssn', 'code')]; + assert.strictEqual(usingLeftOwner('ssn', scope, 2), scope[1]); + assert.strictEqual(usingLeftOwner('code', scope, 2), scope[0]); + }); +}); diff --git a/unitTests/sqlTranslator/column-attribution.test.js b/unitTests/sqlTranslator/column-attribution.test.js new file mode 100644 index 0000000000..993c1937a0 --- /dev/null +++ b/unitTests/sqlTranslator/column-attribution.test.js @@ -0,0 +1,223 @@ +'use strict'; +/** + * Every column reference in a SELECT is attributed to the table the SQL engines read it from, so its + * attribute permission is checked on that table. End-to-end enforcement is covered by + * integrationTests/security/sql-attribute-perms.test.ts. + */ + +const assert = require('assert'); +const alasql = require('alasql'); + +const SqlStatementBucket = require('#src/sqlTranslator/sql_statement_bucket').default; +const { _setDatabasesLoader } = require('#src/sqlEngine/binder/defaultDatabase'); + +const attributes = (...names) => ({ attributes: names.map((name) => ({ name })) }); +const REGISTRY = { + data: { + L: attributes('id', 'label', 'ssn'), + R: attributes('id', 'label', 'ssn', 'code'), + C: attributes('id', 'ssn', 'code'), + }, + vault: { L: attributes('id', 'secret') }, +}; + +let registry = REGISTRY; +let registryLoads = 0; + +function bucketFor(sql) { + return new SqlStatementBucket(alasql.parse(sql).statements[0]); +} + +function attributesOf(bucket, database, table) { + return bucket.affected_attributes.get(database).get(table); +} + +describe('SQL column attribution for attribute permissions', () => { + before(() => { + _setDatabasesLoader(() => { + registryLoads++; + return registry; + }); + }); + + beforeEach(() => { + registry = REGISTRY; + registryLoads = 0; + }); + + after(() => { + _setDatabasesLoader(null); + }); + + describe('JOIN USING', () => { + it('records the column on both joined tables', () => { + const bucket = bucketFor('SELECT l.label FROM data.L l JOIN data.R r USING (ssn)'); + assert.ok(attributesOf(bucket, 'data', 'L').includes('ssn')); + assert.ok(attributesOf(bucket, 'data', 'R').includes('ssn')); + }); + + it('records the column on a self-joined table', () => { + const bucket = bucketFor('SELECT a.label FROM data.L a JOIN data.L b USING (ssn)'); + assert.ok(attributesOf(bucket, 'data', 'L').includes('ssn')); + }); + + it('records only the operands the engine compares: the nearest preceding declarer and the joined table', () => { + const bucket = bucketFor('SELECT a.id FROM data.L a JOIN data.R b ON a.id = b.id JOIN data.C c USING (ssn)'); + assert.ok(attributesOf(bucket, 'data', 'R').includes('ssn')); + assert.ok(attributesOf(bucket, 'data', 'C').includes('ssn')); + assert.ok(!attributesOf(bucket, 'data', 'L').includes('ssn')); + }); + + it('falls back to the FROM table when no preceding table declares the column', () => { + const bucket = bucketFor('SELECT a.id FROM data.L a JOIN data.C c USING (code)'); + assert.ok(attributesOf(bucket, 'data', 'L').includes('code')); + assert.ok(attributesOf(bucket, 'data', 'C').includes('code')); + }); + }); + + describe('unqualified columns in a join', () => { + it('attributes an unqualified ON column to the joined table that declares it', () => { + const bucket = bucketFor(`SELECT l.label FROM data.L l JOIN data.R r ON l.label = r.label AND code = 'c1'`); + assert.ok(attributesOf(bucket, 'data', 'R').includes('code')); + assert.ok(!attributesOf(bucket, 'data', 'L').includes('code')); + }); + + it('resolves an ON column only against the tables joined so far', () => { + const bucket = bucketFor( + `SELECT a.id FROM data.L a JOIN data.R b ON a.id = b.id AND code = 'c1' JOIN data.C c ON b.id = c.id` + ); + assert.ok(attributesOf(bucket, 'data', 'R').includes('code')); + assert.ok(!attributesOf(bucket, 'data', 'C').includes('code')); + }); + + it('attributes an unqualified select column to its declaring table, including in a comma join', () => { + for (const sql of ['SELECT code FROM data.L l JOIN data.R r ON l.id = r.id', 'SELECT code FROM data.L, data.R']) { + const bucket = bucketFor(sql); + assert.ok(attributesOf(bucket, 'data', 'R').includes('code'), sql); + assert.ok(!attributesOf(bucket, 'data', 'L').includes('code'), sql); + } + }); + + it('keeps same-named tables in different databases apart', () => { + const bucket = bucketFor(`SELECT l.label FROM data.L l JOIN vault.L v ON l.id = v.id WHERE secret = 'x'`); + assert.ok(attributesOf(bucket, 'vault', 'L').includes('secret')); + assert.ok(!attributesOf(bucket, 'data', 'L').includes('secret')); + }); + + it('records on every candidate table when the registry cannot describe them', () => { + registry = { data: {} }; + const bucket = bucketFor( + `SELECT a.id FROM data.L a JOIN data.R b ON a.id = b.id AND code = 'c1' JOIN data.C c USING (ssn)` + ); + assert.ok(attributesOf(bucket, 'data', 'L').includes('code')); + assert.ok(attributesOf(bucket, 'data', 'R').includes('code')); + assert.ok(!attributesOf(bucket, 'data', 'C').includes('code')); + for (const table of ['L', 'R', 'C']) assert.ok(attributesOf(bucket, 'data', table).includes('ssn'), table); + }); + }); + + describe('qualifiers', () => { + it('attributes a qualifier to the one table that has it as alias or base name', () => { + const bucket = bucketFor('SELECT A.ssn FROM data.L a JOIN data.R A ON a.id = A.id'); + assert.ok(attributesOf(bucket, 'data', 'R').includes('ssn')); + assert.ok(!attributesOf(bucket, 'data', 'L').includes('ssn')); + assert.deepStrictEqual(bucket.getUnresolvedColumnRefs(), []); + }); + + it("reports a qualifier that is one table's alias and another's base name", () => { + const bucket = bucketFor('SELECT L.ssn FROM data.L a JOIN data.R L ON a.id = L.id'); + assert.ok(bucket.getUnresolvedColumnRefs().includes('L.ssn')); + assert.ok(!attributesOf(bucket, 'data', 'R').includes('ssn')); + }); + + it('counts claimants across the whole statement, not just the tables an ON has joined so far', () => { + const bucket = bucketFor( + 'SELECT a.id FROM data.L a JOIN data.R b ON a.id = b.id AND L.ssn = 1 JOIN data.C L ON b.id = L.id' + ); + assert.ok(bucket.getUnresolvedColumnRefs().includes('L.ssn')); + }); + + it('reports a qualifier shared by same-named tables in different databases', () => { + const bucket = bucketFor('SELECT L.secret FROM data.L JOIN vault.L USING (id)'); + assert.deepStrictEqual(bucket.getUnresolvedColumnRefs(), ['L.secret']); + }); + + it('reports a qualifier that names no table in scope', () => { + const bucket = bucketFor('SELECT bogus.ssn FROM data.L WHERE id = 1'); + assert.deepStrictEqual(bucket.getUnresolvedColumnRefs(), ['bogus.ssn']); + assert.ok(!attributesOf(bucket, 'data', 'L').includes('ssn')); + }); + + it('reports a base table name that is ambiguous in a self-join', () => { + const bucket = bucketFor('SELECT L.ssn FROM data.L a JOIN data.L b ON a.id = b.id'); + assert.deepStrictEqual(bucket.getUnresolvedColumnRefs(), ['L.ssn']); + }); + + it('accepts a base table name for an aliased single table', () => { + const bucket = bucketFor('SELECT L.ssn FROM data.L AS l'); + assert.deepStrictEqual(bucket.getUnresolvedColumnRefs(), []); + assert.ok(attributesOf(bucket, 'data', 'L').includes('ssn')); + }); + }); + + it('records a column nested deeper than 100 levels', () => { + const nested = 'UPPER('.repeat(80) + 'ssn' + ')'.repeat(80); + const bucket = bucketFor(`SELECT label FROM data.L WHERE ${nested} = 'X'`); + assert.ok(attributesOf(bucket, 'data', 'L').includes('ssn')); + }); + + it('treats objects inside a JSON literal as data, but a column inside one as a column', () => { + const literal = bucketFor(`SELECT {columnid:'id',tableid:'bogus'} AS obj FROM data.L WHERE id = 'x'`); + assert.deepStrictEqual(literal.getUnresolvedColumnRefs(), []); + assert.deepStrictEqual(attributesOf(literal, 'data', 'L'), ['id']); + const update = bucketFor(`UPDATE data.L SET label = {columnid:'ssn'} WHERE id = 1`); + assert.ok(!attributesOf(update, 'data', 'L').includes('ssn')); + const nested = bucketFor(`SELECT {a: {columnid:'x', tableid:'zz'}, b: {c: {columnid:'y'}}} AS obj FROM data.L`); + assert.deepStrictEqual(nested.getUnresolvedColumnRefs(), []); + assert.ok(!attributesOf(nested, 'data', 'L').includes('y')); + const wrapped = bucketFor(`SELECT {a: ssn} AS obj FROM data.L WHERE id = 'x'`); + assert.ok(attributesOf(wrapped, 'data', 'L').includes('ssn')); + }); + + it('does not load the registry for a single-table SELECT', () => { + const bucket = bucketFor(`SELECT ssn FROM data.L WHERE label = 'x' ORDER BY id`); + assert.deepStrictEqual(attributesOf(bucket, 'data', 'L').sort(), ['id', 'label', 'ssn']); + assert.strictEqual(registryLoads, 0); + }); + + describe('row accessors', () => { + it('lists `_` and JavaScript expressions anywhere in the statement', () => { + assert.deepStrictEqual(bucketFor('SELECT VALUE FROM data.L').getRowAccessors(), ['_']); + assert.deepStrictEqual(bucketFor('SELECT label FROM data.L WHERE _->id = 1').getRowAccessors(), ['_']); + assert.deepStrictEqual(bucketFor('UPDATE data.L SET label = ``1`` WHERE id = 1').getRowAccessors(), [ + 'JavaScript expression', + ]); + }); + + it('finds an accessor nested deeper than 100 levels', () => { + const nested = 'UPPER('.repeat(80) + '_->id' + ')'.repeat(80); + assert.deepStrictEqual(bucketFor(`SELECT label FROM data.L WHERE ${nested} = 'X'`).getRowAccessors(), ['_']); + }); + + it('walks a subquery node AlaSQL shares between two fields', () => { + const bucket = bucketFor('SELECT label FROM data.L WHERE EXISTS (SELECT id FROM data.R)'); + assert.deepStrictEqual(bucket.getRowAccessors(), []); + // Nested queries are walked with the outer scope, which is safe only because they are refused. + assert.deepStrictEqual(bucket.getUnauthorizedTableRefs(), ['nested query in "exists"']); + for (const sql of [ + 'SELECT (SELECT ssn FROM data.R) AS s FROM data.L', + 'SELECT label FROM data.L WHERE id IN (SELECT id FROM data.R)', + ]) { + assert.deepStrictEqual(bucketFor(sql).getUnauthorizedTableRefs(), ['nested query in "queries"'], sql); + } + }); + + it('lists nothing for a statement that names its columns', () => { + assert.deepStrictEqual(bucketFor(`SELECT VALUE label FROM data.L WHERE id = 1`).getRowAccessors(), []); + assert.deepStrictEqual( + bucketFor("INSERT INTO data.L (id, label) VALUES ('x', {columnid:'_'})").getRowAccessors(), + [] + ); + }); + }); +}); diff --git a/unitTests/sqlTranslator/sql_statement_bucket.test.js b/unitTests/sqlTranslator/sql_statement_bucket.test.js index 4a1f0f2171..1604147e6a 100644 --- a/unitTests/sqlTranslator/sql_statement_bucket.test.js +++ b/unitTests/sqlTranslator/sql_statement_bucket.test.js @@ -284,14 +284,7 @@ function clone(a) { return JSON.parse(JSON.stringify(a)); } -let logger_info_spy; - describe('Test sql_statement_bucket Class', () => { - before(() => { - const logger_rw = sql_statement_rewire.__get__('harperLogger'); - logger_info_spy = sandbox.spy(logger_rw, 'info'); - }); - after(() => { sandbox.restore(); rewire('#js/sqlTranslator/sql_statement_bucket'); @@ -671,7 +664,7 @@ describe('Test sql_statement_bucket Class', () => { }); }); - it('Nominal case, INVALID table in WHERE, reasonably complex AST with attributes. ', function () { + it('Reports a WHERE column qualified by an unknown table as unresolved', function () { let getRecordAttributesAST = sql_statement_rewire.__get__('getRecordAttributesAST'); let test_copy = clone(TEST_COMPLEX_AST); const invalid_table = 'steffen'; @@ -681,13 +674,16 @@ describe('Test sql_statement_bucket Class', () => { let table_lookup = new Map(); let schema_lookup = new Map(); let table_to_schema_lookup = new Map(); - getRecordAttributesAST(temp_select, affected_attributes, table_lookup, schema_lookup, table_to_schema_lookup); - assert.equal(logger_info_spy.calledOnce, true, 'invalid table was not logged'); - assert.equal( - logger_info_spy.args[0], - `table specified as ${invalid_table} not found.`, - 'invalid table was not logged' + const unresolved = new Set(); + getRecordAttributesAST( + temp_select, + affected_attributes, + table_lookup, + schema_lookup, + table_to_schema_lookup, + unresolved ); + assert.deepEqual([...unresolved], [`${invalid_table}.${test_copy.where.expression.left.left.columnid}`]); }); }); diff --git a/unitTests/utility/operation_authorization.test.js b/unitTests/utility/operation_authorization.test.js index d04aa3c332..f0498cbe62 100644 --- a/unitTests/utility/operation_authorization.test.js +++ b/unitTests/utility/operation_authorization.test.js @@ -607,6 +607,32 @@ describe('Test operation_authorization', function () { assert.equal(result, null); }); + it('refuses a non-super whole-row accessor or unresolved column qualifier, but not a super user', function () { + for (const sql of [ + 'SELECT VALUE FROM dev.dog', + 'SELECT ``1`` AS one FROM dev.dog', + 'SELECT bogus.name FROM dev.dog', + ]) { + const req_json = getRequestJson(TEST_JSON); + req_json.hdb_user.role.permission.dev.tables.dog.read = true; + const result = op_auth_rewire.verifyPermsAST( + alasql.parse(sql).statements[0], + req_json.hdb_user, + search.search.name + ); + assert.equal(result instanceof PermissionResponseObject, true, sql); + assert.equal(result.unauthorized_access.length, 1, sql); + + const super_json = getRequestJson(TEST_JSON); + super_json.hdb_user.role.permission.super_user = true; + assert.equal( + op_auth_rewire.verifyPermsAST(alasql.parse(sql).statements[0], super_json.hdb_user, search.search.name), + null, + sql + ); + } + }); + it('Test cannot delete from system table error returned', () => { let ast_test = { table: { diff --git a/utility/errors/commonErrors.ts b/utility/errors/commonErrors.ts index 62d2300630..13407b3ac4 100644 --- a/utility/errors/commonErrors.ts +++ b/utility/errors/commonErrors.ts @@ -143,6 +143,10 @@ const OPERATION_AUTH_ERROR_MSGS = { UNKNOWN_OP_AUTH_ERROR: (op, schema, table) => `There was an error authorizing ${op} op on table '${schema}.${table}'`, UNRESOLVED_SQL_TABLE: 'This operation is not authorized: the statement references a table that could not be resolved to a single database. Qualify the table with its database name (e.g. `database.table`).', + UNRESOLVED_SQL_COLUMN: (columns: string[]) => + `This operation is not authorized: the statement qualifies ${columns.join(', ')} with a table name or alias that does not identify exactly one table in the statement.`, + SQL_ROW_ACCESSOR: + 'This operation is not authorized: the statement reads whole rows through `_`, `SELECT VALUE`/`ROW`, or a JavaScript expression, which attribute permissions cannot be applied to. Name the attributes to read instead.', USER_HAS_NO_PERMS: (user) => `User ${user} has no role or permissions. Please assign the user a valid role.`, DROP_SYSTEM: "The 'system' database, tables and records are used internally by Harper and cannot be updated or removed.", diff --git a/utility/operation_authorization.ts b/utility/operation_authorization.ts index f18fe125f1..2be4417709 100644 --- a/utility/operation_authorization.ts +++ b/utility/operation_authorization.ts @@ -637,6 +637,16 @@ export function verifyPermsAST(ast, userObject, operation, apiOperation = terms. return permsResponse.handleUnauthorizedItem(HDB_ERROR_MSGS.UNRESOLVED_SQL_TABLE); } + if (!isSuperUser) { + const unresolvedColumns = parsedAst.getUnresolvedColumnRefs(); + if (unresolvedColumns.length > 0) { + return permsResponse.handleUnauthorizedItem(HDB_ERROR_MSGS.UNRESOLVED_SQL_COLUMN(unresolvedColumns)); + } + if (parsedAst.getRowAccessors().length > 0) { + return permsResponse.handleUnauthorizedItem(HDB_ERROR_MSGS.SQL_ROW_ACCESSOR); + } + } + const fullRolePerms = permsTranslator.getRolePermissions(userObject.role); userObject.role.permission = fullRolePerms; From 612708842ed681f5f283b2972be80557f37fb859 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 30 Oct 2026 08:11:15 -0600 Subject: [PATCH 02/19] Check read permission on the columns an SQL UPDATE or DELETE reads UPDATE's WHERE and SET expressions and DELETE's WHERE now need `read` on each column they reference, separately from the write permission; the primary key is exempt, since the statement reports the keys it touched. A joined SELECT * now expands to the FROM table's readable attributes qualified by its alias, so each name binds to the table whose permissions chose it. Before, the unqualified expansion was ambiguous for any name the joined table shared, and could bind to the joined table for a name only it declared. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RGfHqd6v2q9t9jJ26No4Jc Dispatch-Task: harper-sql-permission-walk-using-and-container --- .../security/sql-attribute-perms.test.ts | 54 ++++++++++++++++++- sqlTranslator/DESIGN.md | 2 + sqlTranslator/sql_statement_bucket.ts | 30 ++++++++++- .../sqlTranslator/column-attribution.test.js | 27 +++++++++- utility/operation_authorization.ts | 13 +++++ 5 files changed, 122 insertions(+), 4 deletions(-) diff --git a/integrationTests/security/sql-attribute-perms.test.ts b/integrationTests/security/sql-attribute-perms.test.ts index e90c62a8f1..ef7a17d6c7 100644 --- a/integrationTests/security/sql-attribute-perms.test.ts +++ b/integrationTests/security/sql-attribute-perms.test.ts @@ -66,7 +66,18 @@ for (const sqlEngine of ['legacy', 'auto', 'new']) { super_user: false, data: { tables: { - ColPermLeft: readOnly([attr('id', true), attr('label', true), attr('ssn', false)]), + // Writable, so UPDATE/DELETE predicates can be checked for read on their own. + ColPermLeft: { + read: true, + insert: false, + update: true, + delete: true, + attribute_permissions: [ + attr('id', true), + { attribute_name: 'label', read: true, insert: false, update: true }, + attr('ssn', false), + ], + }, ColPermRight: readOnly([ attr('id', false), attr('label', true), @@ -91,6 +102,7 @@ for (const sqlEngine of ['legacy', 'auto', 'new']) { records: [ { id: 'l1', label: 'shared', ssn: '111' }, { id: 'l2', label: 'other', ssn: '222' }, + { id: 'l3', label: 'disposable', ssn: '333' }, ], }) .expect(200); @@ -168,6 +180,46 @@ for (const sqlEngine of ['legacy', 'auto', 'new']) { } }); + async function adminRow(id: string) { + const r = await client + .req() + .send({ operation: 'sql', sql: `SELECT id, label FROM data.ColPermLeft WHERE id = '${id}'` }); + strictEqual(r.status, 200, JSON.stringify(r.body)); + return r.body[0]; + } + + test('UPDATE or DELETE that filters on a read-denied column is refused and changes nothing', async () => { + const update = `UPDATE data.ColPermLeft SET label = 'changed' WHERE ssn = '111'`; + assertRefused(await asReader(update), update); + const remove = `DELETE FROM data.ColPermLeft WHERE ssn = '222'`; + assertRefused(await asReader(remove), remove); + deepStrictEqual(await adminRow('l1'), { id: 'l1', label: 'shared' }); + deepStrictEqual(await adminRow('l2'), { id: 'l2', label: 'other' }); + }); + + test('UPDATE and DELETE on readable columns and the primary key still run', async () => { + for (const sql of [ + `UPDATE data.ColPermLeft SET label = 'other' WHERE id = 'l2'`, + `UPDATE data.ColPermLeft SET label = 'other' WHERE label = 'other'`, + `DELETE FROM data.ColPermLeft WHERE id = 'l3'`, + ]) { + const r = await asReader(sql); + strictEqual(r.status, 200, `${sql} -> ${r.status} ${JSON.stringify(r.body).slice(0, 300)}`); + } + strictEqual(await adminRow('l3'), undefined); + }); + + test('a joined SELECT * expands to the FROM table columns the role may read', async () => { + const star = `SELECT * FROM data.ColPermLeft l JOIN data.ColPermRight r ON l.label = r.label WHERE l.id = 'l1'`; + const r = await asReader(star); + strictEqual(r.status, 200, `${star} -> ${r.status} ${JSON.stringify(r.body).slice(0, 300)}`); + deepStrictEqual(r.body, [{ id: 'l1', label: 'shared' }]); + const joined = `SELECT r.* FROM data.ColPermLeft l JOIN data.ColPermRight r ON l.label = r.label WHERE l.id = 'l1'`; + const rStar = await asReader(joined); + strictEqual(rStar.status, 200, JSON.stringify(rStar.body)); + deepStrictEqual(rStar.body, [{ label: 'shared', ssn: '111' }]); + }); + test('super user is unaffected', async () => { const r = await client.req().send({ operation: 'sql', sql: USING_DENIED }); if (sqlEngine === 'legacy') { diff --git a/sqlTranslator/DESIGN.md b/sqlTranslator/DESIGN.md index eb853906bf..c9ae6c3356 100644 --- a/sqlTranslator/DESIGN.md +++ b/sqlTranslator/DESIGN.md @@ -14,6 +14,8 @@ So the bucket attributes columns with the binder's own rules, which live in `col - An unqualified column in a join goes to every in-scope table that declares it. Each JOIN ON only sees the tables joined so far. If no table declares the column, it stays on the FROM table, where a role with attribute permissions is refused because the attribute is not found. - `USING (c)` records `c` on the joined table and on `usingLeftOwner` (the nearest preceding declarer, else FROM): the same two operands v2 compares. - When the registry cannot describe a table, unqualified and USING columns go to every candidate table, so the check fails closed. +- UPDATE and DELETE also need `read` on every column their WHERE or SET expressions read (`getWriteStatementReads()`), separately from the write permission. The primary key is exempt: the statement reports the keys it touched, as NoSQL update and delete do. +- An unqualified `*` in a join expands to the FROM table's readable attributes qualified by its alias, so each expanded name binds to the table whose permissions chose it rather than to whichever table declares it. - Every walk uses `astNodes()`: iterative, with no depth limit, and visiting shared nodes once. `recursive-iterator` silently stopped at depth 100, so a column nested in 80 function calls went unchecked, and it threw on the subquery node AlaSQL shares between `where` and `exists`. Row accessors that name no column, AlaSQL's `_` and JavaScript expressions, are refused for non-super users by `getRowAccessors()`. Legacy SQL's row container always carries the primary key (`dataLayer/SQLSearch.ts` keys rows by it), so a whole-row read would expose it whatever the role allows. The JavaScript refusal stays unconditional for non-super users, even if the `_` refusal is ever relaxed for roles without attribute permissions. diff --git a/sqlTranslator/sql_statement_bucket.ts b/sqlTranslator/sql_statement_bucket.ts index 3bfb191b3e..94611fe6e2 100644 --- a/sqlTranslator/sql_statement_bucket.ts +++ b/sqlTranslator/sql_statement_bucket.ts @@ -151,6 +151,28 @@ class sqlStatementBucket { return accessors; } + /** + * Attributes an UPDATE or DELETE reads to pick rows (WHERE) or compute values (SET expressions). + * They need read permission on top of the write permission the statement already needs. The primary + * key is left out: the statement reports the keys of the rows it touched, as NoSQL update and delete do. + */ + getWriteStatementReads(): { schema: string; table: string; attributes: string[] } | undefined { + const ast = this.ast; + const isUpdate = ast instanceof (alasql as any).yy.Update; + if (!isUpdate && !(ast instanceof (alasql as any).yy.Delete)) return undefined; + const { databaseid: schema, tableid: table } = ast.table; + const tables = loadDatabaseRegistry()?.[schema]; + const primaryKey = tables && Object.hasOwn(tables, table) ? (tables[table] as any)?.primaryKey : undefined; + const attributes = []; + const trees = [ast.where, ...(isUpdate ? (ast.columns ?? []).map((column) => column?.expression) : [])]; + for (const tree of trees) { + for (const node of columnNodes(tree)) { + if (node.columnid !== primaryKey && !attributes.includes(node.columnid)) attributes.push(node.columnid); + } + } + return { schema, table, attributes }; + } + /** *When a SELECT * is included in the AST for a non-SU, we need to convert the star into the specific attributes the * user has READ permissions @@ -202,10 +224,14 @@ class sqlStatementBucket { .get(colSchema) .get(colTable) .filter((attr) => !terms.SEARCH_WILDCARDS.includes(attr)); + // In a join an unqualified expansion would bind to whichever table declares the name, not + // necessarily the one whose permissions were just applied. + const isJoin = this.ast.from.length > 1 || this.ast.joins?.length > 0; + const qualifier = val.tableid ?? (isJoin ? (this.ast.from[0].as ?? this.ast.from[0].tableid) : undefined); finalTableAttrs.forEach(({ attribute_name }) => { let newColumn = new (alasql as any).yy.Column({ columnid: attribute_name }); - if (val.tableid) { - newColumn.tableid = val.tableid; + if (qualifier) { + newColumn.tableid = qualifier; } this.ast.columns.push(newColumn); if (!tableAffectedAttrs.includes(attribute_name)) { diff --git a/unitTests/sqlTranslator/column-attribution.test.js b/unitTests/sqlTranslator/column-attribution.test.js index 993c1937a0..c5263e1c74 100644 --- a/unitTests/sqlTranslator/column-attribution.test.js +++ b/unitTests/sqlTranslator/column-attribution.test.js @@ -14,7 +14,7 @@ const { _setDatabasesLoader } = require('#src/sqlEngine/binder/defaultDatabase') const attributes = (...names) => ({ attributes: names.map((name) => ({ name })) }); const REGISTRY = { data: { - L: attributes('id', 'label', 'ssn'), + L: { ...attributes('id', 'label', 'ssn'), primaryKey: 'id' }, R: attributes('id', 'label', 'ssn', 'code'), C: attributes('id', 'ssn', 'code'), }, @@ -185,6 +185,31 @@ describe('SQL column attribution for attribute permissions', () => { assert.strictEqual(registryLoads, 0); }); + describe('reads made by UPDATE and DELETE', () => { + it('lists WHERE and SET-expression columns, but not the primary key or a SET target', () => { + const update = bucketFor(`UPDATE data.L SET label = UPPER(ssn) WHERE id = 'x' AND label = 'y'`); + assert.deepStrictEqual(update.getWriteStatementReads(), { + schema: 'data', + table: 'L', + attributes: ['label', 'ssn'], + }); + const remove = bucketFor(`DELETE FROM data.L WHERE ssn = '1' OR id = 'x'`); + assert.deepStrictEqual(remove.getWriteStatementReads().attributes, ['ssn']); + }); + + it('lists nothing for a SELECT or INSERT', () => { + assert.strictEqual(bucketFor('SELECT label FROM data.L').getWriteStatementReads(), undefined); + assert.strictEqual(bucketFor("INSERT INTO data.L (id) VALUES ('x')").getWriteStatementReads(), undefined); + }); + + it('keeps the primary key when the registry cannot name it', () => { + registry = { data: {} }; + assert.deepStrictEqual(bucketFor(`DELETE FROM data.L WHERE id = 'x'`).getWriteStatementReads().attributes, [ + 'id', + ]); + }); + }); + describe('row accessors', () => { it('lists `_` and JavaScript expressions anywhere in the statement', () => { assert.deepStrictEqual(bucketFor('SELECT VALUE FROM data.L').getRowAccessors(), ['_']); diff --git a/utility/operation_authorization.ts b/utility/operation_authorization.ts index 2be4417709..53d8d1890e 100644 --- a/utility/operation_authorization.ts +++ b/utility/operation_authorization.ts @@ -685,6 +685,19 @@ export function verifyPermsAST(ast, userObject, operation, apiOperation = terms. } }); + const writeReads = isSuperUser ? undefined : parsedAst.getWriteStatementReads(); + if (writeReads?.attributes.length) { + checkAttributePerms( + writeReads.attributes, + getAttributePermissions(userObject.role.permission, writeReads.schema, writeReads.table), + terms.VALID_SQL_OPS_ENUM.SELECT, + writeReads.table, + writeReads.schema, + permsResponse, + undefined + ); + } + return permsResponse.getPermsResponse(); } catch (e) { throw handleHDBError(e); From 1a69ae4b0dbd6a6717403218d3ccba3f8cf0c507 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 30 Oct 2026 08:29:16 -0600 Subject: [PATCH 03/19] Close the remaining SQL column-attribution gaps the review found - A joined SELECT * is refused when the FROM table's alias is also claimed by another table, since the expansion is qualified by it. - An unqualified ON column no table joined so far declares falls back to the declaring tables anywhere in the statement. - UPDATE checks only SET targets for update; what SET expressions read is checked for read. UPDATE and DELETE refuse a column qualified by a name other than their table. - A JSON literal that sets __proto__ is refused for non-super users: every walk reads own properties, but AlaSQL evaluates the prototype. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RGfHqd6v2q9t9jJ26No4Jc Dispatch-Task: harper-sql-permission-walk-using-and-container --- .../security/sql-attribute-perms.test.ts | 10 ++- sqlTranslator/sql_statement_bucket.ts | 67 ++++++++++++++++--- .../sqlTranslator/column-attribution.test.js | 39 +++++++++++ utility/errors/commonErrors.ts | 2 +- 4 files changed, 104 insertions(+), 14 deletions(-) diff --git a/integrationTests/security/sql-attribute-perms.test.ts b/integrationTests/security/sql-attribute-perms.test.ts index ef7a17d6c7..5c6fcaa077 100644 --- a/integrationTests/security/sql-attribute-perms.test.ts +++ b/integrationTests/security/sql-attribute-perms.test.ts @@ -161,6 +161,7 @@ for (const sqlEngine of ['legacy', 'auto', 'new']) { `SELECT VALUE FROM data.ColPermRight r WHERE r.label = 'shared'`, `SELECT _->id AS k FROM data.ColPermRight r WHERE r.label = 'shared'`, "SELECT ``p`` AS rec FROM data.ColPermRight r WHERE r.label = 'shared'", + "SELECT {__proto__:{rec:_}} AS obj FROM data.ColPermRight r WHERE r.label LIKE '%shared%'", ]) { assertRefused(await asReader(sql), sql); } @@ -191,14 +192,19 @@ for (const sqlEngine of ['legacy', 'auto', 'new']) { test('UPDATE or DELETE that filters on a read-denied column is refused and changes nothing', async () => { const update = `UPDATE data.ColPermLeft SET label = 'changed' WHERE ssn = '111'`; assertRefused(await asReader(update), update); - const remove = `DELETE FROM data.ColPermLeft WHERE ssn = '222'`; - assertRefused(await asReader(remove), remove); + for (const remove of [ + `DELETE FROM data.ColPermLeft WHERE ssn = '222'`, + `DELETE FROM data.ColPermLeft WHERE bogus.id IS NULL`, + ]) { + assertRefused(await asReader(remove), remove); + } deepStrictEqual(await adminRow('l1'), { id: 'l1', label: 'shared' }); deepStrictEqual(await adminRow('l2'), { id: 'l2', label: 'other' }); }); test('UPDATE and DELETE on readable columns and the primary key still run', async () => { for (const sql of [ + `UPDATE data.ColPermLeft SET label = id WHERE id = 'l2'`, `UPDATE data.ColPermLeft SET label = 'other' WHERE id = 'l2'`, `UPDATE data.ColPermLeft SET label = 'other' WHERE label = 'other'`, `DELETE FROM data.ColPermLeft WHERE id = 'l3'`, diff --git a/sqlTranslator/sql_statement_bucket.ts b/sqlTranslator/sql_statement_bucket.ts index 94611fe6e2..49740e8fb0 100644 --- a/sqlTranslator/sql_statement_bucket.ts +++ b/sqlTranslator/sql_statement_bucket.ts @@ -147,6 +147,9 @@ class sqlStatementBucket { for (const node of astNodes(this.ast)) { if (node instanceof (alasql as any).yy.Column && node.columnid === '_') accessors.push('_'); else if (node instanceof (alasql as any).yy.JavaScript) accessors.push('JavaScript expression'); + // A JSON literal's `__proto__` key hides an expression in the prototype, where every walk here + // (own properties only) misses it but AlaSQL's JSON compiler still evaluates it. + else if (!isKnownPrototype(Object.getPrototypeOf(node))) accessors.push('JSON literal with __proto__'); } return accessors; } @@ -257,6 +260,21 @@ function filterReadRestrictedAttrs(attrPerms: any[]) { return attrPerms.filter((perm) => perm[terms.PERMS_CRUD_ENUM.READ]); } +let astPrototypes: Set | undefined; +function isKnownPrototype(prototype: object | null): boolean { + astPrototypes ??= new Set( + Object.values((alasql as any).yy) + .filter((value) => typeof value === 'function') + .map((astClass: any) => astClass.prototype) + ); + return ( + prototype === null || + prototype === Object.prototype || + prototype === Array.prototype || + astPrototypes.has(prototype) + ); +} + /** * Every object node under `root`, each once. Iterative and unbounded: an expression nested arbitrarily * deep must still be seen, and AlaSQL shares one subquery node between two fields (EXISTS). @@ -501,9 +519,9 @@ function getRecordAttributesAST( } else if (ast instanceof (alasql as any).yy.Select) { getSelectAttributes(ast, affectedAttributes, tableLookup, schemaLookup, tableToSchemaLookup, unresolvedColumns); } else if (ast instanceof (alasql as any).yy.Update) { - getUpdateAttributes(ast, affectedAttributes, tableLookup); + getUpdateAttributes(ast, affectedAttributes, tableLookup, unresolvedColumns); } else if (ast instanceof (alasql as any).yy.Delete) { - getDeleteAttributes(ast, affectedAttributes, tableLookup); + getDeleteAttributes(ast, affectedAttributes, tableLookup, unresolvedColumns); } else { harperLogger.error(`AST in getRecordAttributesAST() is not a valid SQL type.`); } @@ -598,14 +616,24 @@ function getSelectAttributes( else unresolvedColumns.add(`${node.tableid}.${node.columnid}`); return; } - // Wildcards stay with the FROM table; updateAttributeWildcardsForRolePerms() expands them there. - if (clauseScope.length === 1 || terms.SEARCH_WILDCARDS.includes(node.columnid)) { + // updateAttributeWildcardsForRolePerms() expands a wildcard on the FROM table, qualified by its alias + // in a join, so that alias must name one table. + if (terms.SEARCH_WILDCARDS.includes(node.columnid)) { + const fromName = scope[0].effectiveAlias; + const claimants = scope.filter((entry) => entry.effectiveAlias === fromName || entry.table === fromName); + if (claimants.length === 1) record(scope[0], node.columnid); + else unresolvedColumns.add(`${fromName}.${node.columnid}`); + return; + } + if (clauseScope.length === 1) { record(scope[0], node.columnid); return; } - const owners = clauseScope.every((entry) => entry.attributesKnown) - ? declaringTables(node.columnid, clauseScope) - : clauseScope; + const declarers = (tables: AttributionTable[]) => + tables.every((entry) => entry.attributesKnown) ? declaringTables(node.columnid, tables) : tables; + // Legacy SQL can bind an ON column to a table joined after that ON. + let owners = declarers(clauseScope); + if (owners.length === 0 && clauseScope.length < scope.length) owners = declarers(scope); if (owners.length === 0) record(scope[0], node.columnid); for (const owner of owners) record(owner, node.columnid); }; @@ -639,7 +667,7 @@ function getSelectAttributes( * @param affectedAttributes - - A map containing attributes affected by the statement. Defined as [schema, Map[table, [attributesArray]]]. * @param tableLookup - A map that will be filled in. This map contains alias to table definitions as [alias, tableName]. */ -function getUpdateAttributes(ast: any, affectedAttributes: any, tableLookup: any) { +function getUpdateAttributes(ast: any, affectedAttributes: any, tableLookup: any, unresolvedColumns?: Set) { if (!ast) { harperLogger.info(`getUpdateAttributes: invalid SQL syntax tree`); return; @@ -648,8 +676,24 @@ function getUpdateAttributes(ast: any, affectedAttributes: any, tableLookup: any addSchemaTableToMap(ast.table, affectedAttributes, tableLookup); - for (const node of columnNodes(ast.columns)) { - pushAttribute(ast.table.tableid, schema, node.columnid, affectedAttributes, tableLookup); + // SET targets need update; what SET expressions and WHERE read needs read (getWriteStatementReads()). + for (const assignment of ast.columns ?? []) { + if (assignment?.column?.columnid) { + pushAttribute(ast.table.tableid, schema, assignment.column.columnid, affectedAttributes, tableLookup); + } + } + reportForeignQualifiers(ast, [ast.columns, ast.where], unresolvedColumns); +} + +/** A write statement has one table, so a column qualified by any other name has no table to check it on. */ +function reportForeignQualifiers(ast: any, trees: any[], unresolvedColumns?: Set) { + const { tableid, as } = ast.table; + for (const tree of trees) { + for (const node of columnNodes(tree)) { + if (node.tableid && node.tableid !== tableid && node.tableid !== as) { + unresolvedColumns?.add(`${node.tableid}.${node.columnid}`); + } + } } } @@ -659,7 +703,7 @@ function getUpdateAttributes(ast: any, affectedAttributes: any, tableLookup: any * @param affectedAttributes - - A map containing attributes affected by the statement. Defined as [schema, Map[table, [attributesArray]]]. * @param tableLookup - A map that will be filled in. This map contains alias to table definitions as [alias, tableName]. */ -function getDeleteAttributes(ast: any, affectedAttributes: any, tableLookup: any) { +function getDeleteAttributes(ast: any, affectedAttributes: any, tableLookup: any, unresolvedColumns?: Set) { if (!ast) { harperLogger.info(`getDeleteAttributes: invalid SQL syntax tree`); return; @@ -671,6 +715,7 @@ function getDeleteAttributes(ast: any, affectedAttributes: any, tableLookup: any for (const node of columnNodes(ast.where)) { pushAttribute(ast.table.tableid, schema, node.columnid, affectedAttributes, tableLookup); } + reportForeignQualifiers(ast, [ast.where], unresolvedColumns); } /** diff --git a/unitTests/sqlTranslator/column-attribution.test.js b/unitTests/sqlTranslator/column-attribution.test.js index c5263e1c74..5014d74109 100644 --- a/unitTests/sqlTranslator/column-attribution.test.js +++ b/unitTests/sqlTranslator/column-attribution.test.js @@ -17,6 +17,7 @@ const REGISTRY = { L: { ...attributes('id', 'label', 'ssn'), primaryKey: 'id' }, R: attributes('id', 'label', 'ssn', 'code'), C: attributes('id', 'ssn', 'code'), + S: attributes('id', 'secret'), }, vault: { L: attributes('id', 'secret') }, }; @@ -98,6 +99,13 @@ describe('SQL column attribution for attribute permissions', () => { } }); + it('falls back to a table joined after the ON when none joined so far declares the column', () => { + const bucket = bucketFor( + `SELECT a.id FROM data.L a JOIN data.R b ON a.id = b.id AND secret = 'x' JOIN data.S c ON b.id = c.id` + ); + assert.ok(attributesOf(bucket, 'data', 'S').includes('secret')); + }); + it('keeps same-named tables in different databases apart', () => { const bucket = bucketFor(`SELECT l.label FROM data.L l JOIN vault.L v ON l.id = v.id WHERE secret = 'x'`); assert.ok(attributesOf(bucket, 'vault', 'L').includes('secret')); @@ -153,6 +161,22 @@ describe('SQL column attribution for attribute permissions', () => { assert.deepStrictEqual(bucket.getUnresolvedColumnRefs(), ['L.ssn']); }); + it('reports a joined SELECT * whose FROM alias another table also claims', () => { + const bucket = bucketFor('SELECT * FROM data.L JOIN data.R L ON L.id = L.id'); + assert.ok(bucket.getUnresolvedColumnRefs().includes('L.*')); + assert.ok(!bucketFor('SELECT * FROM data.L l JOIN data.R r ON l.id = r.id').getUnresolvedColumnRefs().length); + }); + + it('reports a write statement column qualified by a name other than its table', () => { + assert.deepStrictEqual(bucketFor('DELETE FROM data.L WHERE bogus.id IS NULL').getUnresolvedColumnRefs(), [ + 'bogus.id', + ]); + assert.deepStrictEqual( + bucketFor('UPDATE data.L SET label = bogus.ssn WHERE L.id = 1').getUnresolvedColumnRefs(), + ['bogus.ssn'] + ); + }); + it('accepts a base table name for an aliased single table', () => { const bucket = bucketFor('SELECT L.ssn FROM data.L AS l'); assert.deepStrictEqual(bucket.getUnresolvedColumnRefs(), []); @@ -197,6 +221,12 @@ describe('SQL column attribution for attribute permissions', () => { assert.deepStrictEqual(remove.getWriteStatementReads().attributes, ['ssn']); }); + it('checks SET targets for update but not the columns a SET expression reads', () => { + const update = bucketFor(`UPDATE data.L SET label = ssn WHERE id = 'x'`); + assert.deepStrictEqual(attributesOf(update, 'data', 'L'), ['label']); + assert.deepStrictEqual(update.getWriteStatementReads().attributes, ['ssn']); + }); + it('lists nothing for a SELECT or INSERT', () => { assert.strictEqual(bucketFor('SELECT label FROM data.L').getWriteStatementReads(), undefined); assert.strictEqual(bucketFor("INSERT INTO data.L (id) VALUES ('x')").getWriteStatementReads(), undefined); @@ -219,6 +249,15 @@ describe('SQL column attribution for attribute permissions', () => { ]); }); + it('refuses a JSON literal that hides an expression behind __proto__', () => { + for (const sql of [ + "SELECT {__proto__:{rec:_}} AS obj FROM data.L WHERE label = 'x'", + "SELECT {a:{__proto__:{rec:label}}} AS obj FROM data.L WHERE label = 'x'", + ]) { + assert.ok(bucketFor(sql).getRowAccessors().includes('JSON literal with __proto__'), sql); + } + }); + it('finds an accessor nested deeper than 100 levels', () => { const nested = 'UPPER('.repeat(80) + '_->id' + ')'.repeat(80); assert.deepStrictEqual(bucketFor(`SELECT label FROM data.L WHERE ${nested} = 'X'`).getRowAccessors(), ['_']); diff --git a/utility/errors/commonErrors.ts b/utility/errors/commonErrors.ts index 13407b3ac4..8ed14337e0 100644 --- a/utility/errors/commonErrors.ts +++ b/utility/errors/commonErrors.ts @@ -146,7 +146,7 @@ const OPERATION_AUTH_ERROR_MSGS = { UNRESOLVED_SQL_COLUMN: (columns: string[]) => `This operation is not authorized: the statement qualifies ${columns.join(', ')} with a table name or alias that does not identify exactly one table in the statement.`, SQL_ROW_ACCESSOR: - 'This operation is not authorized: the statement reads whole rows through `_`, `SELECT VALUE`/`ROW`, or a JavaScript expression, which attribute permissions cannot be applied to. Name the attributes to read instead.', + 'This operation is not authorized: the statement reads whole rows through `_`, `SELECT VALUE`/`ROW`, a JavaScript expression or a JSON literal that sets `__proto__`, which attribute permissions cannot be applied to. Name the attributes to read instead.', USER_HAS_NO_PERMS: (user) => `User ${user} has no role or permissions. Please assign the user a valid role.`, DROP_SYSTEM: "The 'system' database, tables and records are used internally by Harper and cannot be updated or removed.", From e0d0bb933a8a877eaf5c1ac2ba7954027c5a8bed Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 30 Oct 2026 08:31:38 -0600 Subject: [PATCH 04/19] Test that a JSON literal in an UPDATE SET adds no read requirement Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RGfHqd6v2q9t9jJ26No4Jc Dispatch-Task: harper-sql-permission-walk-using-and-container --- unitTests/sqlTranslator/column-attribution.test.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unitTests/sqlTranslator/column-attribution.test.js b/unitTests/sqlTranslator/column-attribution.test.js index 5014d74109..e6cedbeff8 100644 --- a/unitTests/sqlTranslator/column-attribution.test.js +++ b/unitTests/sqlTranslator/column-attribution.test.js @@ -225,6 +225,8 @@ describe('SQL column attribution for attribute permissions', () => { const update = bucketFor(`UPDATE data.L SET label = ssn WHERE id = 'x'`); assert.deepStrictEqual(attributesOf(update, 'data', 'L'), ['label']); assert.deepStrictEqual(update.getWriteStatementReads().attributes, ['ssn']); + const literal = bucketFor(`UPDATE data.L SET label = {columnid:'ssn'} WHERE id = 'x'`); + assert.deepStrictEqual(literal.getWriteStatementReads().attributes, []); }); it('lists nothing for a SELECT or INSERT', () => { From 1387b721bcc823a89b42a53de80a61a6e00ad0ce Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 30 Oct 2026 08:45:26 -0600 Subject: [PATCH 05/19] Bring the SQL attribution design note up to date Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RGfHqd6v2q9t9jJ26No4Jc Dispatch-Task: harper-sql-permission-walk-using-and-container --- sqlTranslator/DESIGN.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sqlTranslator/DESIGN.md b/sqlTranslator/DESIGN.md index c9ae6c3356..e16c7d1f12 100644 --- a/sqlTranslator/DESIGN.md +++ b/sqlTranslator/DESIGN.md @@ -11,13 +11,13 @@ So the bucket attributes columns with the binder's own rules, which live in `columnScope.ts` and are called by `bind.ts`, `logical/build.ts` and the bucket alike: - A qualified column is attributed only when exactly one table in the statement (not only the tables an ON has joined so far) has the qualifier as its alias or base name. v2 prefers an alias over a base name, while legacy SQL matches either, so only then do both engines read the same table. Any other qualifier is reported by `getUnresolvedColumnRefs()`, and non-super statements are refused. -- An unqualified column in a join goes to every in-scope table that declares it. Each JOIN ON only sees the tables joined so far. If no table declares the column, it stays on the FROM table, where a role with attribute permissions is refused because the attribute is not found. +- An unqualified column in a join goes to every in-scope table that declares it. Each JOIN ON first looks only at the tables joined so far, then, when none of them declares the column, at every table in the statement (legacy SQL can bind it to a later join). If no table declares the column, it stays on the FROM table, where a role with attribute permissions is refused because the attribute is not found. - `USING (c)` records `c` on the joined table and on `usingLeftOwner` (the nearest preceding declarer, else FROM): the same two operands v2 compares. - When the registry cannot describe a table, unqualified and USING columns go to every candidate table, so the check fails closed. -- UPDATE and DELETE also need `read` on every column their WHERE or SET expressions read (`getWriteStatementReads()`), separately from the write permission. The primary key is exempt: the statement reports the keys it touched, as NoSQL update and delete do. +- UPDATE and DELETE also need `read` on every column their WHERE or SET expressions read (`getWriteStatementReads()`), separately from the write permission; only SET targets need `update`. A column qualified by a name other than the target table is refused. The primary key is exempt: the statement reports the keys it touched, as NoSQL update and delete do. - An unqualified `*` in a join expands to the FROM table's readable attributes qualified by its alias, so each expanded name binds to the table whose permissions chose it rather than to whichever table declares it. - Every walk uses `astNodes()`: iterative, with no depth limit, and visiting shared nodes once. `recursive-iterator` silently stopped at depth 100, so a column nested in 80 function calls went unchecked, and it threw on the subquery node AlaSQL shares between `where` and `exists`. -Row accessors that name no column, AlaSQL's `_` and JavaScript expressions, are refused for non-super users by `getRowAccessors()`. Legacy SQL's row container always carries the primary key (`dataLayer/SQLSearch.ts` keys rows by it), so a whole-row read would expose it whatever the role allows. The JavaScript refusal stays unconditional for non-super users, even if the `_` refusal is ever relaxed for roles without attribute permissions. +Row accessors that name no column, AlaSQL's `_`, JavaScript expressions and a JSON literal that sets `__proto__` (every walk here reads own properties, but AlaSQL evaluates the prototype), are refused for non-super users by `getRowAccessors()`. Legacy SQL's row container always carries the primary key (`dataLayer/SQLSearch.ts` keys rows by it), so a whole-row read would expose it whatever the role allows. The JavaScript refusal stays unconditional for non-super users, even if the `_` refusal is ever relaxed for roles without attribute permissions. Enforced by `unitTests/sqlTranslator/column-attribution.test.js`, `unitTests/sqlEngine/columnScope.test.js` and `integrationTests/security/sql-attribute-perms.test.ts`. From 3ccb9fcdfb0a7093280432d36b35f88355ea2626 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 31 Oct 2026 00:31:35 -0600 Subject: [PATCH 06/19] Let SQL DELETE filter on a read-denied primary key, as UPDATE can DELETE recorded its WHERE columns in the DELETE attribute list as well as in the read check, so a primary key with no attribute grants refused DELETE ... WHERE id = ... while the same UPDATE ran. The read check already covers every non-key predicate column, so DELETE now registers only its table there. The integration role makes the write predicates discriminating: an updatable but read-denied column, an UPDATE whose SET reads it, and a writable table whose primary key is read-denied. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: pr-maint-abc8bf322b5ccfd671a9379f3776cea4 --- .../security/sql-attribute-perms.test.ts | 49 +++++++++++++------ sqlTranslator/DESIGN.md | 2 +- sqlTranslator/sql_statement_bucket.ts | 6 +-- .../sqlTranslator/column-attribution.test.js | 6 +++ .../sql_statement_bucket.test.js | 5 +- 5 files changed, 45 insertions(+), 23 deletions(-) diff --git a/integrationTests/security/sql-attribute-perms.test.ts b/integrationTests/security/sql-attribute-perms.test.ts index 5c6fcaa077..612a6886d4 100644 --- a/integrationTests/security/sql-attribute-perms.test.ts +++ b/integrationTests/security/sql-attribute-perms.test.ts @@ -27,10 +27,6 @@ function attr(attribute_name: string, read: boolean) { return { attribute_name, read, insert: false, update: false }; } -function readOnly(attribute_permissions: ReturnType[]) { - return { read: true, insert: false, update: false, delete: false, attribute_permissions }; -} - // sql.engine=new answers only indexed queries, so every statement carries an indexed predicate. const USING_DENIED = `SELECT l.label FROM data.ColPermLeft l JOIN data.ColPermRight r USING (ssn) WHERE l.id = 'l1'`; const USING_DENIED_SELF = `SELECT a.label FROM data.ColPermLeft a JOIN data.ColPermLeft b USING (ssn) WHERE a.id = 'l1'`; @@ -66,7 +62,7 @@ for (const sqlEngine of ['legacy', 'auto', 'new']) { super_user: false, data: { tables: { - // Writable, so UPDATE/DELETE predicates can be checked for read on their own. + // Writable, and `ssn` updatable, so only the read check can refuse a predicate on it. ColPermLeft: { read: true, insert: false, @@ -75,15 +71,22 @@ for (const sqlEngine of ['legacy', 'auto', 'new']) { attribute_permissions: [ attr('id', true), { attribute_name: 'label', read: true, insert: false, update: true }, - attr('ssn', false), + { attribute_name: 'ssn', read: false, insert: false, update: true }, + ], + }, + // Writable with a read-denied primary key, which write predicates may still use. + ColPermRight: { + read: true, + insert: false, + update: true, + delete: true, + attribute_permissions: [ + attr('id', false), + { attribute_name: 'label', read: true, insert: false, update: true }, + attr('ssn', true), + attr('code', false), ], }, - ColPermRight: readOnly([ - attr('id', false), - attr('label', true), - attr('ssn', true), - attr('code', false), - ]), }, }, }, @@ -190,8 +193,12 @@ for (const sqlEngine of ['legacy', 'auto', 'new']) { } test('UPDATE or DELETE that filters on a read-denied column is refused and changes nothing', async () => { - const update = `UPDATE data.ColPermLeft SET label = 'changed' WHERE ssn = '111'`; - assertRefused(await asReader(update), update); + for (const update of [ + `UPDATE data.ColPermLeft SET label = 'changed' WHERE ssn = '111'`, + `UPDATE data.ColPermLeft SET label = ssn WHERE id = 'l2'`, + ]) { + assertRefused(await asReader(update), update); + } for (const remove of [ `DELETE FROM data.ColPermLeft WHERE ssn = '222'`, `DELETE FROM data.ColPermLeft WHERE bogus.id IS NULL`, @@ -215,6 +222,20 @@ for (const sqlEngine of ['legacy', 'auto', 'new']) { strictEqual(await adminRow('l3'), undefined); }); + test('UPDATE and DELETE may filter on a read-denied primary key', async () => { + for (const sql of [ + `UPDATE data.ColPermRight SET label = 'unmatched' WHERE id = 'r-primary-key-2'`, + `DELETE FROM data.ColPermRight WHERE id = 'r-primary-key-2'`, + ]) { + const r = await asReader(sql); + strictEqual(r.status, 200, `${sql} -> ${r.status} ${JSON.stringify(r.body).slice(0, 300)}`); + } + const left = await client + .req() + .send({ operation: 'sql', sql: `SELECT id FROM data.ColPermRight WHERE id = 'r-primary-key-2'` }); + deepStrictEqual(left.body, []); + }); + test('a joined SELECT * expands to the FROM table columns the role may read', async () => { const star = `SELECT * FROM data.ColPermLeft l JOIN data.ColPermRight r ON l.label = r.label WHERE l.id = 'l1'`; const r = await asReader(star); diff --git a/sqlTranslator/DESIGN.md b/sqlTranslator/DESIGN.md index e16c7d1f12..6e72289646 100644 --- a/sqlTranslator/DESIGN.md +++ b/sqlTranslator/DESIGN.md @@ -14,7 +14,7 @@ So the bucket attributes columns with the binder's own rules, which live in `col - An unqualified column in a join goes to every in-scope table that declares it. Each JOIN ON first looks only at the tables joined so far, then, when none of them declares the column, at every table in the statement (legacy SQL can bind it to a later join). If no table declares the column, it stays on the FROM table, where a role with attribute permissions is refused because the attribute is not found. - `USING (c)` records `c` on the joined table and on `usingLeftOwner` (the nearest preceding declarer, else FROM): the same two operands v2 compares. - When the registry cannot describe a table, unqualified and USING columns go to every candidate table, so the check fails closed. -- UPDATE and DELETE also need `read` on every column their WHERE or SET expressions read (`getWriteStatementReads()`), separately from the write permission; only SET targets need `update`. A column qualified by a name other than the target table is refused. The primary key is exempt: the statement reports the keys it touched, as NoSQL update and delete do. +- UPDATE and DELETE also need `read` on every column their WHERE or SET expressions read (`getWriteStatementReads()`), separately from the write permission; only SET targets need `update`. A column qualified by a name other than the target table is refused. The primary key is exempt: the statement reports the keys it touched, as NoSQL update and delete do. Only attribute-level denials apply; table-level `read` is not required. - An unqualified `*` in a join expands to the FROM table's readable attributes qualified by its alias, so each expanded name binds to the table whose permissions chose it rather than to whichever table declares it. - Every walk uses `astNodes()`: iterative, with no depth limit, and visiting shared nodes once. `recursive-iterator` silently stopped at depth 100, so a column nested in 80 function calls went unchecked, and it threw on the subquery node AlaSQL shares between `where` and `exists`. diff --git a/sqlTranslator/sql_statement_bucket.ts b/sqlTranslator/sql_statement_bucket.ts index 49740e8fb0..2d61c78566 100644 --- a/sqlTranslator/sql_statement_bucket.ts +++ b/sqlTranslator/sql_statement_bucket.ts @@ -708,13 +708,9 @@ function getDeleteAttributes(ast: any, affectedAttributes: any, tableLookup: any harperLogger.info(`getDeleteAttributes: invalid SQL syntax tree`); return; } - let schema = ast.table.databaseid; - addSchemaTableToMap(ast.table, affectedAttributes, tableLookup); - for (const node of columnNodes(ast.where)) { - pushAttribute(ast.table.tableid, schema, node.columnid, affectedAttributes, tableLookup); - } + // DELETE writes no attribute; what WHERE reads needs read (getWriteStatementReads()), primary key exempt. reportForeignQualifiers(ast, [ast.where], unresolvedColumns); } diff --git a/unitTests/sqlTranslator/column-attribution.test.js b/unitTests/sqlTranslator/column-attribution.test.js index e6cedbeff8..05529717d8 100644 --- a/unitTests/sqlTranslator/column-attribution.test.js +++ b/unitTests/sqlTranslator/column-attribution.test.js @@ -229,6 +229,12 @@ describe('SQL column attribution for attribute permissions', () => { assert.deepStrictEqual(literal.getWriteStatementReads().attributes, []); }); + it('checks DELETE predicates only for read, so the primary key stays exempt', () => { + const remove = bucketFor(`DELETE FROM data.L WHERE id = 'x' AND ssn = '1'`); + assert.deepStrictEqual(attributesOf(remove, 'data', 'L'), []); + assert.deepStrictEqual(remove.getWriteStatementReads().attributes, ['ssn']); + }); + it('lists nothing for a SELECT or INSERT', () => { assert.strictEqual(bucketFor('SELECT label FROM data.L').getWriteStatementReads(), undefined); assert.strictEqual(bucketFor("INSERT INTO data.L (id) VALUES ('x')").getWriteStatementReads(), undefined); diff --git a/unitTests/sqlTranslator/sql_statement_bucket.test.js b/unitTests/sqlTranslator/sql_statement_bucket.test.js index 1604147e6a..e8b65d855c 100644 --- a/unitTests/sqlTranslator/sql_statement_bucket.test.js +++ b/unitTests/sqlTranslator/sql_statement_bucket.test.js @@ -291,13 +291,12 @@ describe('Test sql_statement_bucket Class', () => { }); describe(`Test getDeleteAttributes`, function () { - it('Nominal, pull attributes in delete statement', function () { + it('registers the table but no WHERE columns, which are checked for read instead', function () { let getDeleteAttributes = sql_statement_rewire.__get__('getDeleteAttributes'); - //let statement = new sql_statement_bucket(TEST_DELETE); let statement = new Map(); let table_lookup = new Map(); getDeleteAttributes(TEST_DELETE, statement, table_lookup); - assert.equal(statement.get(SCHEMA_NAME).get(TABLE_NAME).length, 2); + assert.equal(statement.get(SCHEMA_NAME).get(TABLE_NAME).length, 0); assert.equal(Array.from(statement.get(SCHEMA_NAME).keys()).length, 1); assert.equal(Array.from(statement.keys()).length, 1); }); From cd7c7f26038e7e470a3713c2ef2b48078d18f018 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 31 Oct 2026 01:01:56 -0600 Subject: [PATCH 07/19] Refuse SQL row accessors after wildcard expansion The whole-row accessor check ran before SELECT * was expanded to the role's readable attributes, so a table declaring an attribute named `_` let SELECT * produce the `_` column a direct SELECT _ is refused for. The check now runs on the expanded statement, still once and still skipped for super users. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: pr-maint-abc8bf322b5ccfd671a9379f3776cea4 --- .../utility/operation_authorization.test.js | 23 +++++++++++++++++++ utility/operation_authorization.ts | 7 +++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/unitTests/utility/operation_authorization.test.js b/unitTests/utility/operation_authorization.test.js index f0498cbe62..0e54896928 100644 --- a/unitTests/utility/operation_authorization.test.js +++ b/unitTests/utility/operation_authorization.test.js @@ -27,6 +27,7 @@ const PermissionTableResponseObject = require('#src/security/data_objects/Permis const PermissionAttributeResponseObject = require('#src/security/data_objects/PermissionAttributeResponseObject').default; const { TEST_SCHEMA_OP_ERROR, TEST_OPERATION_AUTH_ERROR } = require('../commonTestErrors'); +const { HDB_ERROR_MSGS } = require('#src/utility/errors/commonErrors'); // rewire() bypasses the require cache and re-executes serverUtilities' module body in an isolated // instance so we can reach its private initializeOperationFunctionMap(). But that module body also @@ -633,6 +634,28 @@ describe('Test operation_authorization', function () { } }); + it('refuses a wildcard that expands into an attribute named `_`', function () { + const req_json = getRequestJson(TEST_JSON); + req_json.hdb_user.role.permission.dev.tables.dog.read = true; + req_json.hdb_user.role.permission.dev.tables.dog.attribute_permissions = ATTRIBUTE_PERMISSION_BASE( + ['_'], + crud_keys.READ, + true + ); + const attributes = global.hdb_schema.dev.dog.attributes; + attributes.push({ attribute: '_' }); + try { + const result = op_auth_rewire.verifyPermsAST( + alasql.parse('SELECT * FROM dev.dog').statements[0], + req_json.hdb_user, + search.search.name + ); + assert.deepEqual(result?.unauthorized_access, [HDB_ERROR_MSGS.SQL_ROW_ACCESSOR]); + } finally { + attributes.pop(); + } + }); + it('Test cannot delete from system table error returned', () => { let ast_test = { table: { diff --git a/utility/operation_authorization.ts b/utility/operation_authorization.ts index 53d8d1890e..586d891c2a 100644 --- a/utility/operation_authorization.ts +++ b/utility/operation_authorization.ts @@ -642,9 +642,6 @@ export function verifyPermsAST(ast, userObject, operation, apiOperation = terms. if (unresolvedColumns.length > 0) { return permsResponse.handleUnauthorizedItem(HDB_ERROR_MSGS.UNRESOLVED_SQL_COLUMN(unresolvedColumns)); } - if (parsedAst.getRowAccessors().length > 0) { - return permsResponse.handleUnauthorizedItem(HDB_ERROR_MSGS.SQL_ROW_ACCESSOR); - } } const fullRolePerms = permsTranslator.getRolePermissions(userObject.role); @@ -655,6 +652,10 @@ export function verifyPermsAST(ast, userObject, operation, apiOperation = terms. if (!isSuperUser && ast instanceof (alasql.yy as any).Select) { ast = parsedAst.updateAttributeWildcardsForRolePerms(fullRolePerms); } + // After expansion, so a wildcard cannot expand into an attribute named `_`. + if (!isSuperUser && parsedAst.getRowAccessors().length > 0) { + return permsResponse.handleUnauthorizedItem(HDB_ERROR_MSGS.SQL_ROW_ACCESSOR); + } for (let s = 0; s < schemas.length; s++) { //NOSONAR From 0273b4eed01053ef75b2b4b8062d0edc0a6969f3 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 31 Oct 2026 01:25:20 -0600 Subject: [PATCH 08/19] Skip SQL AST walk setup for an absent clause Most statements leave several of the clauses the attribution walks (group, having, order, joins) empty; return before allocating the visited set and stack for them. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: pr-maint-abc8bf322b5ccfd671a9379f3776cea4 --- sqlTranslator/sql_statement_bucket.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sqlTranslator/sql_statement_bucket.ts b/sqlTranslator/sql_statement_bucket.ts index 2d61c78566..0446698a7c 100644 --- a/sqlTranslator/sql_statement_bucket.ts +++ b/sqlTranslator/sql_statement_bucket.ts @@ -280,8 +280,9 @@ function isKnownPrototype(prototype: object | null): boolean { * deep must still be seen, and AlaSQL shares one subquery node between two fields (EXISTS). */ function* astNodes(root: any): Generator { + if (root === null || typeof root !== 'object') return; const seen = new Set(); - const pending = root !== null && typeof root === 'object' ? [root] : []; + const pending = [root]; while (pending.length > 0) { const node = pending.pop(); if (seen.has(node)) continue; From 02dfd6884a04526183740042615cfcf082d44589 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 1 Nov 2026 00:17:23 -0600 Subject: [PATCH 09/19] Keep SQL literals and names intact when legacy SQL re-runs a statement Legacy SQL checks permissions on the parsed statement, then runs AlaSQL's re-parse of its toString() text. AlaSQL writes string and JSON literals without escaping, so a literal such as 'x'', ssn, ''y' came back as three columns and an UPDATE SET expression could read a column the check never saw. 'O''Brien' failed for the same reason. Harper now writes StringValue and Json literals so they decode to the same value, and the legacy path refuses with a 400 what has no spelling: a name containing a backtick, bracket or quote, or a string whose backslashes and quotes AlaSQL's lexer cannot reproduce. v2 binds the AST directly and is unaffected. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RGfHqd6v2q9t9jJ26No4Jc Dispatch-Task: pr-maint-abc8bf322b5ccfd671a9379f3776cea4 --- DESIGN.md | 2 +- .../security/sql-attribute-perms.test.ts | 25 ++++ sqlTranslator/DESIGN.md | 6 + sqlTranslator/alasqlFunctionImporter.ts | 3 + sqlTranslator/index.ts | 10 +- sqlTranslator/legacySqlText.ts | 135 ++++++++++++++++++ sqlTranslator/sql_statement_bucket.ts | 2 +- unitTests/sqlTranslator/legacySqlText.test.js | 124 ++++++++++++++++ 8 files changed, 304 insertions(+), 3 deletions(-) create mode 100644 sqlTranslator/legacySqlText.ts create mode 100644 unitTests/sqlTranslator/legacySqlText.test.js diff --git a/DESIGN.md b/DESIGN.md index cc3c45c264..93a61eab58 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -6,7 +6,7 @@ A note lives in the `DESIGN.md` of the directory that owns the code it describes ## sqlTranslator/ — legacy SQL integration -- [CommonJS date functions](sqlTranslator/DESIGN.md#commonjs-date-functions) — Both runtimes use the same CommonJS helper object, enforced by `alasqlFunctionImporter.test.js`. · [SQL authorization attributes every column with the engine's own resolution rules (`sql_statement_bucket.ts`, `sqlEngine/binder/columnScope.ts`)](sqlTranslator/DESIGN.md#sql-authorization-attributes-every-column-with-the-engines-own-resolution-rules-sql_statement_bucketts-sqlenginebindercolumnscopets) — The bucket and the v2 binder share one column resolver; unresolved qualifiers and whole-row accessors are refused for non-super users. +- [CommonJS date functions](sqlTranslator/DESIGN.md#commonjs-date-functions) — Both runtimes use the same CommonJS helper object, enforced by `alasqlFunctionImporter.test.js`. · [SQL authorization attributes every column with the engine's own resolution rules (`sql_statement_bucket.ts`, `sqlEngine/binder/columnScope.ts`)](sqlTranslator/DESIGN.md#sql-authorization-attributes-every-column-with-the-engines-own-resolution-rules-sql_statement_bucketts-sqlenginebindercolumnscopets) — The bucket and the v2 binder share one column resolver; unresolved qualifiers and whole-row accessors are refused for non-super users. · [Legacy SQL runs the re-parse of the authorized statement's text (`legacySqlText.ts`)](sqlTranslator/DESIGN.md#legacy-sql-runs-the-re-parse-of-the-authorized-statements-text-legacysqltextts) — Literals are written so they decode to the same value; names and strings with no spelling get a 400 on legacy. ## .github/ — CI workflows diff --git a/integrationTests/security/sql-attribute-perms.test.ts b/integrationTests/security/sql-attribute-perms.test.ts index 612a6886d4..33eaa78cdb 100644 --- a/integrationTests/security/sql-attribute-perms.test.ts +++ b/integrationTests/security/sql-attribute-perms.test.ts @@ -247,6 +247,31 @@ for (const sqlEngine of ['legacy', 'auto', 'new']) { deepStrictEqual(rStar.body, [{ label: 'shared', ssn: '111' }]); }); + test('a quoted string or name that spells out a column list reads no other column', async () => { + const literal = `SELECT 'x'', ssn, ''y' AS v FROM data.ColPermLeft WHERE id = 'l1'`; + const r = await asReader(literal); + strictEqual(r.status, 200, `${literal} -> ${r.status} ${JSON.stringify(r.body).slice(0, 300)}`); + deepStrictEqual(r.body, [{ v: "x', ssn, 'y" }]); + + // v2 has no CONCAT. + if (sqlEngine !== 'new') { + const update = `UPDATE data.ColPermLeft SET label = CONCAT('x'', ssn, ''y') WHERE id = 'l2'`; + const u = await asReader(update); + strictEqual(u.status, 200, `${update} -> ${u.status} ${JSON.stringify(u.body).slice(0, 300)}`); + deepStrictEqual(await adminRow('l2'), { id: 'l2', label: "x', ssn, 'y" }); + await client + .req() + .send({ operation: 'sql', sql: `UPDATE data.ColPermLeft SET label = 'other' WHERE id = 'l2'` }) + .expect(200); + } + + // Legacy SQL cannot write this name back out and refuses it; v2 reads it as one name. + const name = "SELECT label AS [x`,ssn,`y] FROM data.ColPermLeft WHERE id = 'l1'"; + const n = await asReader(name); + ok(n.status === 200 || n.status === 400, `${name} -> ${n.status} ${JSON.stringify(n.body).slice(0, 300)}`); + ok(!JSON.stringify(n.body).includes('111'), 'the response must not carry the read-denied value'); + }); + test('super user is unaffected', async () => { const r = await client.req().send({ operation: 'sql', sql: USING_DENIED }); if (sqlEngine === 'legacy') { diff --git a/sqlTranslator/DESIGN.md b/sqlTranslator/DESIGN.md index 6e72289646..e4df69851f 100644 --- a/sqlTranslator/DESIGN.md +++ b/sqlTranslator/DESIGN.md @@ -21,3 +21,9 @@ So the bucket attributes columns with the binder's own rules, which live in `col Row accessors that name no column, AlaSQL's `_`, JavaScript expressions and a JSON literal that sets `__proto__` (every walk here reads own properties, but AlaSQL evaluates the prototype), are refused for non-super users by `getRowAccessors()`. Legacy SQL's row container always carries the primary key (`dataLayer/SQLSearch.ts` keys rows by it), so a whole-row read would expose it whatever the role allows. The JavaScript refusal stays unconditional for non-super users, even if the `_` refusal is ever relaxed for roles without attribute permissions. Enforced by `unitTests/sqlTranslator/column-attribution.test.js`, `unitTests/sqlEngine/columnScope.test.js` and `integrationTests/security/sql-attribute-perms.test.ts`. + +## Legacy SQL runs the re-parse of the authorized statement's text (`legacySqlText.ts`) + +Legacy SQL writes the authorized AST back out with `toString()` and AlaSQL parses that text again (`dataLayer/SQLSearch.ts` `_buildSQL()`, `dataLayer/update.ts`, `deleteTranslator.ts`, and the compiled SET and INSERT expressions), so the permission check holds only if the text parses back to the same statement. AlaSQL writes string and JSON literals unescaped, so `'x'', ssn, ''y'` came back as three columns. `installLegacySqlText()`, run by `alasqlFunctionImporter`, replaces the `StringValue` and `Json` writers with ones that decode to the same value. `assertLegacySqlText()` refuses, with a 400 on the legacy path only, what has no spelling: a name containing `` ` ``, `[`, `]` or `'`, which backticks and brackets cannot escape, or a string whose backslashes and quotes AlaSQL's lexer cannot reproduce. v2 binds the AST directly. + +Enforced by `unitTests/sqlTranslator/legacySqlText.test.js` and the quoted-string case in `integrationTests/security/sql-attribute-perms.test.ts`. diff --git a/sqlTranslator/alasqlFunctionImporter.ts b/sqlTranslator/alasqlFunctionImporter.ts index 35858a56c9..8a094c4275 100644 --- a/sqlTranslator/alasqlFunctionImporter.ts +++ b/sqlTranslator/alasqlFunctionImporter.ts @@ -7,9 +7,12 @@ import * as alasqlExtension from '../utility/functions/sql/alaSQLExtension.ts'; import dateFunctions from '../utility/functions/date/dateFunctions.js'; import * as geo from '../utility/functions/geo.ts'; +import { installLegacySqlText } from './legacySqlText.ts'; //import the custom function, need to define an upper and lower case version of the function so it is parsed properly in alasql export default function (alasql: any) { + installLegacySqlText(); + /* AGGREGATE FUNCTIONS */ diff --git a/sqlTranslator/index.ts b/sqlTranslator/index.ts index 049a007689..ff4b6d4864 100644 --- a/sqlTranslator/index.ts +++ b/sqlTranslator/index.ts @@ -12,6 +12,7 @@ import alasql from 'alasql'; import * as opAuth from '../utility/operation_authorization.ts'; import logger from '../utility/logging/harper_logger.ts'; import alasqlFunctionImporter from './alasqlFunctionImporter.ts'; +import { assertLegacySqlText } from './legacySqlText.ts'; import * as hdbUtils from '../utility/common_utils.ts'; import * as terms from '../utility/hdbTerms.ts'; import { handleHDBError } from '../utility/errors/hdbError.ts'; @@ -193,7 +194,14 @@ export function processAST(jsonMessage: any, parsedSqlObject: any, callback: any variant: parsedSqlObject.variant, jsonMessage, statement, - legacy: sqlFunction, + legacy: (legacyStatement, legacyCallback) => { + try { + assertLegacySqlText(parsedSqlObject.ast.statements[0]); + } catch (err) { + return legacyCallback(err); + } + sqlFunction(legacyStatement, legacyCallback); + }, }, (err, data) => { if (err) { diff --git a/sqlTranslator/legacySqlText.ts b/sqlTranslator/legacySqlText.ts new file mode 100644 index 0000000000..76104b7fe0 --- /dev/null +++ b/sqlTranslator/legacySqlText.ts @@ -0,0 +1,135 @@ +/** + * Legacy SQL authorizes the parsed statement but runs AlaSQL's re-parse of its `toString()` text, so that text + * must parse back to the same statement. AlaSQL writes string and JSON literals without escaping, and + * identifiers inside backticks or brackets that cannot hold every character. The serializers below quote + * literals so they decode to the same value, and `assertLegacySqlText` refuses whatever cannot be written. + */ +import alasql from 'alasql'; +import { handleHDBError } from '../utility/errors/hdbError.ts'; +import { HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; +import { astNodes } from './sql_statement_bucket.ts'; + +const yy = (alasql as any).yy; + +// Harper writes names inside backticks, and AlaSQL inside brackets; neither can escape these. +const UNQUOTABLE_NAME = /[`[\]']/; + +/** + * Single-quoted text that AlaSQL decodes back to `value`, or undefined when there is none. AlaSQL lexes a + * backslash as taking the next character, then decodes `\'` and `''` to `'`, so a quote after an even run of + * backslashes, or an odd run at the end, has no spelling. + */ +export function quoteString(value: string): string | undefined { + let text = "'"; + let i = 0; + while (i < value.length) { + let backslashes = 0; + while (value[i + backslashes] === '\\') backslashes++; + const next = value[i + backslashes]; + if (next === undefined) { + if (backslashes % 2) return undefined; + text += value.slice(i); + break; + } + if (next === "'") { + if (backslashes === 0) text += "''"; + else if (backslashes % 2) text += '\\'.repeat(backslashes + 1) + "''"; + else return undefined; + } else { + text += value.slice(i, i + backslashes + 1); + } + i += backslashes + 1; + } + return text + "'"; +} + +// A JSON key is the token body undecoded, so it must lex as it stands inside one of the quotes. +function quoteKey(key: string): string | undefined { + if (lexesInside(key, '"')) return `"${key}"`; + if (lexesInside(key, "'")) return `'${key}'`; +} + +function lexesInside(body: string, quote: string): boolean { + for (let i = 0; i < body.length; i++) { + if (body[i] === '\\') { + if (++i >= body.length) return false; + } else if (body[i] === quote && body[++i] !== quote) { + return false; + } + } + return true; +} + +function unwritable(what: string) { + return handleHDBError( + new Error(), + `${what} cannot be written back to SQL text for the legacy SQL engine. Remove the quotes, backticks, brackets or backslashes, or use the new SQL engine.`, + HTTP_STATUS_CODES.BAD_REQUEST + ); +} + +function isJsonData(value: any): boolean { + return Array.isArray(value) || Object.getPrototypeOf(value) === Object.prototype; +} + +function jsonText(value: any): string { + if (typeof value === 'string') { + const text = quoteString(value); + if (text === undefined) throw unwritable('A string in a JSON literal'); + return text; + } + if (value == null) return 'NULL'; + if (typeof value !== 'object') return typeof value === 'bigint' ? `${value}n` : String(value); + // Any other object is an AST node inside the literal, a nested JSON literal included. + if (!isJsonData(value)) return value.toString(); + if (Array.isArray(value)) return `@[${value.map(jsonText).join(',')}]`; + const properties = []; + for (const key in value) { + const keyText = quoteKey(key); + if (keyText === undefined) throw unwritable('A key in a JSON literal'); + properties.push(`${keyText}:${jsonText(value[key])}`); + } + return `{${properties.join(',')}}`; +} + +export function installLegacySqlText() { + yy.StringValue.prototype.toString = function () { + const text = quoteString(String(this.value)); + if (text === undefined) throw unwritable('A string literal'); + return text; + }; + yy.Json.prototype.toString = function () { + return jsonText(this.value); + }; +} + +/** Throws a 400 for a statement whose SQL text would not parse back to it. */ +export function assertLegacySqlText(statement: any) { + const literalData = new Set(); + for (const node of astNodes(statement)) { + if (literalData.has(node)) continue; + if (node instanceof yy.StringValue) { + if (quoteString(String(node.value)) === undefined) throw unwritable('A string literal'); + continue; + } + if (node instanceof yy.Json) { + jsonText(node.value); + const pending = [node.value]; + while (pending.length > 0) { + const value = pending.pop(); + if (value === null || typeof value !== 'object' || literalData.has(value) || !isJsonData(value)) continue; + literalData.add(value); + pending.push(...Object.values(value)); + } + continue; + } + // Its text is written verbatim by design. + if (node instanceof yy.JavaScript) continue; + for (const key in node) { + const value = node[key]; + if (Object.hasOwn(node, key) && typeof value === 'string' && UNQUOTABLE_NAME.test(value)) { + throw unwritable(`The name ${JSON.stringify(value)}`); + } + } + } +} diff --git a/sqlTranslator/sql_statement_bucket.ts b/sqlTranslator/sql_statement_bucket.ts index 0446698a7c..8c28f6b30c 100644 --- a/sqlTranslator/sql_statement_bucket.ts +++ b/sqlTranslator/sql_statement_bucket.ts @@ -279,7 +279,7 @@ function isKnownPrototype(prototype: object | null): boolean { * Every object node under `root`, each once. Iterative and unbounded: an expression nested arbitrarily * deep must still be seen, and AlaSQL shares one subquery node between two fields (EXISTS). */ -function* astNodes(root: any): Generator { +export function* astNodes(root: any): Generator { if (root === null || typeof root !== 'object') return; const seen = new Set(); const pending = [root]; diff --git a/unitTests/sqlTranslator/legacySqlText.test.js b/unitTests/sqlTranslator/legacySqlText.test.js new file mode 100644 index 0000000000..9c720472eb --- /dev/null +++ b/unitTests/sqlTranslator/legacySqlText.test.js @@ -0,0 +1,124 @@ +'use strict'; +/** + * Legacy SQL runs the re-parse of a statement's SQL text, so that text must parse back to the statement that + * was authorized. End-to-end coverage is in integrationTests/security/sql-attribute-perms.test.ts. + */ + +const assert = require('assert'); +const sinon = require('sinon'); +const alasql = require('alasql'); + +const sqlTranslator = require('#js/sqlTranslator/index'); +const opAuth = require('#src/utility/operation_authorization'); +const sqlEngineRouter = require('#src/sqlEngine/router'); +const { assertLegacySqlText, installLegacySqlText, quoteString } = require('#src/sqlTranslator/legacySqlText'); +const { backtickASTSchemaItems } = require('#src/utility/common_utils'); + +installLegacySqlText(); + +const parse = (sql) => alasql.parse(sql).statements[0]; + +// The text legacy SQL runs: Harper backticks names before writing the statement out. +function reparsed(sql) { + const statement = parse(sql); + assertLegacySqlText(statement); + backtickASTSchemaItems(statement); + return parse(statement.toString()); +} + +const assertUnwritable = (sql) => + assert.throws( + () => assertLegacySqlText(parse(sql)), + (err) => err.statusCode === 400, + sql + ); + +describe('legacy SQL text', function () { + it('writes a string literal that spells out a column list as one literal', function () { + for (const literal of ["'x'', ssn, ''y'", "'O''Brien'", "'x\\', ssn, \\'y'", "\"x'', ssn, ''y\""]) { + const columns = reparsed(`SELECT ${literal} AS v FROM dev.t`).columns; + assert.strictEqual(columns.length, 1, literal); + assert.strictEqual(columns[0].value, parse(`SELECT ${literal} AS v FROM dev.t`).columns[0].value, literal); + } + }); + + it('writes every short string of letters, quotes and backslashes back to its value, or refuses it', function () { + const alphabet = ['a', "'", '\\']; + let bodies = ['']; + for (let length = 0; length < 6; length++) { + for (const body of bodies) { + let columns; + try { + columns = parse(`SELECT '${body}' AS v FROM t`).columns; + } catch { + continue; + } + if (columns.length !== 1 || columns[0].as !== 'v') continue; + const text = quoteString(columns[0].value); + if (text === undefined) continue; + const again = parse(`SELECT ${text} AS v FROM t WHERE w = 'z'`).columns; + assert.deepStrictEqual( + again.map((c) => c.value), + [columns[0].value], + body + ); + } + bodies = bodies.flatMap((body) => alphabet.map((c) => body + c)); + } + }); + + it('writes JSON literal keys, strings and arrays back to the same value', function () { + for (const json of [ + `{'k':'x"} , ssn , {"z":"y'}`, + `{'x",ssn,"y':1}`, + `{"k":'a"b', "n":@[1,'x''y',{"z":id}], "o":@{"p":'q'}}`, + `@['x'', ssn, ''y']`, + ]) { + const before = parse(`SELECT ${json} AS v FROM dev.t`).columns; + const after = reparsed(`SELECT ${json} AS v FROM dev.t`).columns; + assert.strictEqual(after.length, 1, json); + assert.strictEqual(JSON.stringify(after[0].value), JSON.stringify(before[0].value), json); + } + }); + + it('refuses a name that backticks or brackets cannot hold', function () { + for (const sql of [ + 'SELECT [x`,ssn,`y] FROM dev.t', + 'SELECT `x],ssn,[y` FROM dev.t', + 'SELECT label AS [x`,ssn,`y] FROM dev.t', + 'SELECT label AS `[x` FROM dev.t', + "SELECT label FROM dev.t WHERE [x`,ssn,`y] = 'a'", + ]) { + assertUnwritable(sql); + } + }); + + it('refuses a string literal that has no spelling', function () { + assertUnwritable("SELECT 'a\\' AS v FROM dev.t"); + assertUnwritable("UPDATE dev.t SET label = {'k':'a\\'} WHERE id = 1"); + }); + + it('accepts ordinary statements', function () { + for (const sql of [ + "SELECT d.name, COUNT(*) AS c FROM dev.t AS d INNER JOIN dev.u AS u ON d.id = u.tid WHERE d.x LIKE 'a%' GROUP BY d.name", + "UPDATE dev.t SET label = CONCAT('a', id), meta = {'k':@[1,2]} WHERE id IN (1, 2) AND name IS NOT NULL", + "DELETE FROM dev.t WHERE [first name] = 'O''Brien'", + ]) { + assertLegacySqlText(parse(sql)); + } + }); + + describe('engine routing', function () { + const sandbox = sinon.createSandbox(); + afterEach(() => sandbox.restore()); + + it('refuses an unwritable statement on the legacy path before it runs', async function () { + sandbox.stub(opAuth, 'verifyPermsAST').returns(null); + sandbox.stub(sqlEngineRouter, 'route').callsFake((opts, cb) => opts.legacy(opts.statement, cb)); + const err = await new Promise((resolve) => + sqlTranslator.evaluateSQL({ sql: 'SELECT [x`,ssn,`y] FROM dev.dog', hdb_user: { username: 'u' } }, resolve) + ); + assert.strictEqual(err?.statusCode, 400); + }); + }); +}); From 87c2912dbee8c72d64080b7e8af852d0b4db0529 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 1 Nov 2026 22:15:35 -0700 Subject: [PATCH 10/19] Refuse legacy SQL names that its text cannot hold, aliases included The name check now covers every node's own string properties, so an alias on a literal is checked too, and requires @variable and $param names to be plain identifiers because AlaSQL writes them unquoted. The node walk is a loop, and refusals are ClientError. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RGfHqd6v2q9t9jJ26No4Jc Dispatch-Task: pr-maint-abc8bf322b5ccfd671a9379f3776cea4 --- .../security/sql-attribute-perms.test.ts | 15 +++-- sqlTranslator/DESIGN.md | 2 +- sqlTranslator/legacySqlText.ts | 63 +++++++++++-------- unitTests/sqlTranslator/legacySqlText.test.js | 30 ++++----- 4 files changed, 59 insertions(+), 51 deletions(-) diff --git a/integrationTests/security/sql-attribute-perms.test.ts b/integrationTests/security/sql-attribute-perms.test.ts index 33eaa78cdb..3af2d7d1ca 100644 --- a/integrationTests/security/sql-attribute-perms.test.ts +++ b/integrationTests/security/sql-attribute-perms.test.ts @@ -265,11 +265,16 @@ for (const sqlEngine of ['legacy', 'auto', 'new']) { .expect(200); } - // Legacy SQL cannot write this name back out and refuses it; v2 reads it as one name. - const name = "SELECT label AS [x`,ssn,`y] FROM data.ColPermLeft WHERE id = 'l1'"; - const n = await asReader(name); - ok(n.status === 200 || n.status === 400, `${name} -> ${n.status} ${JSON.stringify(n.body).slice(0, 300)}`); - ok(!JSON.stringify(n.body).includes('111'), 'the response must not carry the read-denied value'); + // Legacy SQL cannot write these names back out and refuses them; v2 reads each as one name. + for (const name of [ + "SELECT label AS [x`,ssn,`y] FROM data.ColPermLeft WHERE id = 'l1'", + "SELECT 'x' AS [x`,ssn,`y] FROM data.ColPermLeft WHERE id = 'l1'", + ]) { + const n = await asReader(name); + const result = `${name} -> ${n.status} ${JSON.stringify(n.body).slice(0, 300)}`; + if (sqlEngine === 'legacy') strictEqual(n.status, 400, result); + ok(!JSON.stringify(n.body).includes('111'), result); + } }); test('super user is unaffected', async () => { diff --git a/sqlTranslator/DESIGN.md b/sqlTranslator/DESIGN.md index e4df69851f..135f748534 100644 --- a/sqlTranslator/DESIGN.md +++ b/sqlTranslator/DESIGN.md @@ -24,6 +24,6 @@ Enforced by `unitTests/sqlTranslator/column-attribution.test.js`, `unitTests/sql ## Legacy SQL runs the re-parse of the authorized statement's text (`legacySqlText.ts`) -Legacy SQL writes the authorized AST back out with `toString()` and AlaSQL parses that text again (`dataLayer/SQLSearch.ts` `_buildSQL()`, `dataLayer/update.ts`, `deleteTranslator.ts`, and the compiled SET and INSERT expressions), so the permission check holds only if the text parses back to the same statement. AlaSQL writes string and JSON literals unescaped, so `'x'', ssn, ''y'` came back as three columns. `installLegacySqlText()`, run by `alasqlFunctionImporter`, replaces the `StringValue` and `Json` writers with ones that decode to the same value. `assertLegacySqlText()` refuses, with a 400 on the legacy path only, what has no spelling: a name containing `` ` ``, `[`, `]` or `'`, which backticks and brackets cannot escape, or a string whose backslashes and quotes AlaSQL's lexer cannot reproduce. v2 binds the AST directly. +Legacy SQL writes the authorized AST back out with `toString()` and AlaSQL parses that text again (`dataLayer/SQLSearch.ts` `_buildSQL()`, `dataLayer/update.ts`, `deleteTranslator.ts`, and the compiled SET and INSERT expressions), so the permission check holds only if the text parses back to the same statement. AlaSQL writes string and JSON literals unescaped, so `'x'', ssn, ''y'` came back as three columns. `installLegacySqlText()`, run by `alasqlFunctionImporter`, replaces the `StringValue` and `Json` writers with ones that decode to the same value. `assertLegacySqlText()` refuses, with a 400 on the legacy path only, what has no spelling: a name containing `` ` ``, `[`, `]` or `'`, which backticks and brackets cannot escape, an `@variable` or `$param` name other than letters, digits and `_`, which AlaSQL writes unquoted, or a string whose backslashes and quotes AlaSQL's lexer cannot reproduce. v2 binds the AST directly. Enforced by `unitTests/sqlTranslator/legacySqlText.test.js` and the quoted-string case in `integrationTests/security/sql-attribute-perms.test.ts`. diff --git a/sqlTranslator/legacySqlText.ts b/sqlTranslator/legacySqlText.ts index 76104b7fe0..f7e3f67d4e 100644 --- a/sqlTranslator/legacySqlText.ts +++ b/sqlTranslator/legacySqlText.ts @@ -1,18 +1,18 @@ /** * Legacy SQL authorizes the parsed statement but runs AlaSQL's re-parse of its `toString()` text, so that text - * must parse back to the same statement. AlaSQL writes string and JSON literals without escaping, and - * identifiers inside backticks or brackets that cannot hold every character. The serializers below quote - * literals so they decode to the same value, and `assertLegacySqlText` refuses whatever cannot be written. + * must keep every literal a literal and every name one name. AlaSQL writes string and JSON literals without + * escaping. */ import alasql from 'alasql'; -import { handleHDBError } from '../utility/errors/hdbError.ts'; -import { HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; +import { ClientError } from '../utility/errors/hdbError.ts'; import { astNodes } from './sql_statement_bucket.ts'; const yy = (alasql as any).yy; // Harper writes names inside backticks, and AlaSQL inside brackets; neither can escape these. const UNQUOTABLE_NAME = /[`[\]']/; +// AlaSQL writes `@variable` and `$param` with no quoting at all. +const BARE_NAME = /^\w+$/; /** * Single-quoted text that AlaSQL decodes back to `value`, or undefined when there is none. AlaSQL lexes a @@ -61,10 +61,8 @@ function lexesInside(body: string, quote: string): boolean { } function unwritable(what: string) { - return handleHDBError( - new Error(), - `${what} cannot be written back to SQL text for the legacy SQL engine. Remove the quotes, backticks, brackets or backslashes, or use the new SQL engine.`, - HTTP_STATUS_CODES.BAD_REQUEST + return new ClientError( + `${what} cannot be written back to SQL text for the legacy SQL engine. Remove the quotes, backticks, brackets or backslashes, or use the new SQL engine.` ); } @@ -103,31 +101,44 @@ export function installLegacySqlText() { }; } -/** Throws a 400 for a statement whose SQL text would not parse back to it. */ -export function assertLegacySqlText(statement: any) { +// AST nodes, without the plain objects and arrays that are a JSON literal's data. +function* statementNodes(statement: any): Generator { const literalData = new Set(); for (const node of astNodes(statement)) { if (literalData.has(node)) continue; - if (node instanceof yy.StringValue) { - if (quoteString(String(node.value)) === undefined) throw unwritable('A string literal'); - continue; + yield node; + if (!(node instanceof yy.Json)) continue; + const pending = [node.value]; + while (pending.length > 0) { + const value = pending.pop(); + if (value === null || typeof value !== 'object' || literalData.has(value) || !isJsonData(value)) continue; + literalData.add(value); + for (const child of Object.values(value)) pending.push(child); + } + } +} + +/** Throws a 400 for a statement whose SQL text would not parse back to what it reads. */ +export function assertLegacySqlText(statement: any) { + for (const node of statementNodes(statement)) { + if (node instanceof yy.StringValue && quoteString(String(node.value)) === undefined) { + throw unwritable('A string literal'); } - if (node instanceof yy.Json) { - jsonText(node.value); - const pending = [node.value]; - while (pending.length > 0) { - const value = pending.pop(); - if (value === null || typeof value !== 'object' || literalData.has(value) || !isJsonData(value)) continue; - literalData.add(value); - pending.push(...Object.values(value)); + if (node instanceof yy.Json) jsonText(node.value); + for (const key of ['variable', 'param']) { + const value = node[key]; + if (Object.hasOwn(node, key) && typeof value === 'string' && !BARE_NAME.test(value)) { + throw unwritable(`The name ${JSON.stringify(value)}`); } - continue; } - // Its text is written verbatim by design. - if (node instanceof yy.JavaScript) continue; for (const key in node) { const value = node[key]; - if (Object.hasOwn(node, key) && typeof value === 'string' && UNQUOTABLE_NAME.test(value)) { + if ( + Object.hasOwn(node, key) && + typeof value === 'string' && + UNQUOTABLE_NAME.test(value) && + !(key === 'value' && (node instanceof yy.StringValue || node instanceof yy.JavaScript)) + ) { throw unwritable(`The name ${JSON.stringify(value)}`); } } diff --git a/unitTests/sqlTranslator/legacySqlText.test.js b/unitTests/sqlTranslator/legacySqlText.test.js index 9c720472eb..ad6efb0b30 100644 --- a/unitTests/sqlTranslator/legacySqlText.test.js +++ b/unitTests/sqlTranslator/legacySqlText.test.js @@ -5,12 +5,8 @@ */ const assert = require('assert'); -const sinon = require('sinon'); const alasql = require('alasql'); -const sqlTranslator = require('#js/sqlTranslator/index'); -const opAuth = require('#src/utility/operation_authorization'); -const sqlEngineRouter = require('#src/sqlEngine/router'); const { assertLegacySqlText, installLegacySqlText, quoteString } = require('#src/sqlTranslator/legacySqlText'); const { backtickASTSchemaItems } = require('#src/utility/common_utils'); @@ -73,6 +69,8 @@ describe('legacy SQL text', function () { `{'x",ssn,"y':1}`, `{"k":'a"b', "n":@[1,'x''y',{"z":id}], "o":@{"p":'q'}}`, `@['x'', ssn, ''y']`, + `{'a"", ssn, ""b':1}`, + `{'a"", ssn, ""b''c':1}`, ]) { const before = parse(`SELECT ${json} AS v FROM dev.t`).columns; const after = reparsed(`SELECT ${json} AS v FROM dev.t`).columns; @@ -81,13 +79,18 @@ describe('legacy SQL text', function () { } }); - it('refuses a name that backticks or brackets cannot hold', function () { + it('refuses a name that backticks, brackets or a bare name cannot hold', function () { for (const sql of [ 'SELECT [x`,ssn,`y] FROM dev.t', 'SELECT `x],ssn,[y` FROM dev.t', 'SELECT label AS [x`,ssn,`y] FROM dev.t', 'SELECT label AS `[x` FROM dev.t', "SELECT label FROM dev.t WHERE [x`,ssn,`y] = 'a'", + "SELECT 'x' AS [x`,ssn,`y] FROM dev.t", + "SELECT 'x' AS `x],ssn,[y` FROM dev.t", + "SELECT {'k':1} AS [x`,ssn,`y] FROM dev.t", + 'SELECT @`x,ssn` AS v FROM dev.t', + 'SELECT $[x,ssn] AS v FROM dev.t', ]) { assertUnwritable(sql); } @@ -103,22 +106,11 @@ describe('legacy SQL text', function () { "SELECT d.name, COUNT(*) AS c FROM dev.t AS d INNER JOIN dev.u AS u ON d.id = u.tid WHERE d.x LIKE 'a%' GROUP BY d.name", "UPDATE dev.t SET label = CONCAT('a', id), meta = {'k':@[1,2]} WHERE id IN (1, 2) AND name IS NOT NULL", "DELETE FROM dev.t WHERE [first name] = 'O''Brien'", + 'SELECT DISTINCT lower(a) AS l FROM dev.t WHERE EXISTS (SELECT 1 FROM dev.u WHERE u.id = t.id) ORDER BY l DESC LIMIT 2', + "SELECT CASE WHEN a = 1 THEN 'x' ELSE 'y' END AS c, CAST(b AS INT) AS n FROM dev.t WHERE b NOT IN (1, 2)", + 'SELECT @x AS v, $1 AS w, ? AS z FROM dev.t', ]) { assertLegacySqlText(parse(sql)); } }); - - describe('engine routing', function () { - const sandbox = sinon.createSandbox(); - afterEach(() => sandbox.restore()); - - it('refuses an unwritable statement on the legacy path before it runs', async function () { - sandbox.stub(opAuth, 'verifyPermsAST').returns(null); - sandbox.stub(sqlEngineRouter, 'route').callsFake((opts, cb) => opts.legacy(opts.statement, cb)); - const err = await new Promise((resolve) => - sqlTranslator.evaluateSQL({ sql: 'SELECT [x`,ssn,`y] FROM dev.dog', hdb_user: { username: 'u' } }, resolve) - ); - assert.strictEqual(err?.statusCode, 400); - }); - }); }); From ebf4229324e1eed138a73da8ce52116c6de63f04 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 1 Nov 2026 22:35:23 -0700 Subject: [PATCH 11/19] Check that legacy SQL's statement text reads nothing new, and keep numbers intact Legacy SQL now re-parses the backticked text it will run and refuses a statement whose text names a column, table, function or variable the authorized statement does not, so an AlaSQL writer this code does not replace, such as the right side of `->`, fails closed. Numbers are written in plain decimal: AlaSQL wrote 1e+25 and Infinity, which read back as columns. A quote after an even run of backslashes is now written instead of refused. The alias cases check the status on every engine. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RGfHqd6v2q9t9jJ26No4Jc Dispatch-Task: pr-maint-abc8bf322b5ccfd671a9379f3776cea4 --- .../security/sql-attribute-perms.test.ts | 2 + sqlTranslator/DESIGN.md | 2 +- sqlTranslator/legacySqlText.ts | 69 +++++++++++++++++-- unitTests/sqlTranslator/legacySqlText.test.js | 42 ++++++++++- 4 files changed, 106 insertions(+), 9 deletions(-) diff --git a/integrationTests/security/sql-attribute-perms.test.ts b/integrationTests/security/sql-attribute-perms.test.ts index 3af2d7d1ca..8c68523b86 100644 --- a/integrationTests/security/sql-attribute-perms.test.ts +++ b/integrationTests/security/sql-attribute-perms.test.ts @@ -273,6 +273,8 @@ for (const sqlEngine of ['legacy', 'auto', 'new']) { const n = await asReader(name); const result = `${name} -> ${n.status} ${JSON.stringify(n.body).slice(0, 300)}`; if (sqlEngine === 'legacy') strictEqual(n.status, 400, result); + else if (sqlEngine === 'new') strictEqual(n.status, 200, result); + else ok(n.status === 200 || n.status === 400, result); ok(!JSON.stringify(n.body).includes('111'), result); } }); diff --git a/sqlTranslator/DESIGN.md b/sqlTranslator/DESIGN.md index 135f748534..a7410f9e62 100644 --- a/sqlTranslator/DESIGN.md +++ b/sqlTranslator/DESIGN.md @@ -24,6 +24,6 @@ Enforced by `unitTests/sqlTranslator/column-attribution.test.js`, `unitTests/sql ## Legacy SQL runs the re-parse of the authorized statement's text (`legacySqlText.ts`) -Legacy SQL writes the authorized AST back out with `toString()` and AlaSQL parses that text again (`dataLayer/SQLSearch.ts` `_buildSQL()`, `dataLayer/update.ts`, `deleteTranslator.ts`, and the compiled SET and INSERT expressions), so the permission check holds only if the text parses back to the same statement. AlaSQL writes string and JSON literals unescaped, so `'x'', ssn, ''y'` came back as three columns. `installLegacySqlText()`, run by `alasqlFunctionImporter`, replaces the `StringValue` and `Json` writers with ones that decode to the same value. `assertLegacySqlText()` refuses, with a 400 on the legacy path only, what has no spelling: a name containing `` ` ``, `[`, `]` or `'`, which backticks and brackets cannot escape, an `@variable` or `$param` name other than letters, digits and `_`, which AlaSQL writes unquoted, or a string whose backslashes and quotes AlaSQL's lexer cannot reproduce. v2 binds the AST directly. +Legacy SQL writes the authorized AST back out with `toString()` and AlaSQL parses that text again (`dataLayer/SQLSearch.ts` `_buildSQL()`, `dataLayer/update.ts`, `deleteTranslator.ts`, and the compiled SET and INSERT expressions), so the permission check holds only if the text parses back to the same statement. AlaSQL writes string and JSON literals unescaped, so `'x'', ssn, ''y'` came back as three columns, and numbers as `1e+25` or `Infinity`, which read back as columns. `installLegacySqlText()`, run by `alasqlFunctionImporter`, replaces the `StringValue`, `Json` and `NumValue` writers with ones that read back as the same value. `assertLegacySqlText()` refuses, with a 400 on the legacy path only, what has no spelling: a name containing `` ` ``, `[`, `]` or `'`, which backticks and brackets cannot escape, an `@variable` or `$param` name other than letters, digits and `_`, which AlaSQL writes unquoted, a string ending in an odd run of backslashes, or a number that overflows. It then re-parses the backticked text legacy runs and refuses a statement whose text names a column, table, function or variable the statement does not, so an AlaSQL writer not replaced here (the right side of `->`, for one) fails closed. v2 binds the AST directly. Enforced by `unitTests/sqlTranslator/legacySqlText.test.js` and the quoted-string case in `integrationTests/security/sql-attribute-perms.test.ts`. diff --git a/sqlTranslator/legacySqlText.ts b/sqlTranslator/legacySqlText.ts index f7e3f67d4e..7687ce17fb 100644 --- a/sqlTranslator/legacySqlText.ts +++ b/sqlTranslator/legacySqlText.ts @@ -1,9 +1,11 @@ /** * Legacy SQL authorizes the parsed statement but runs AlaSQL's re-parse of its `toString()` text, so that text * must keep every literal a literal and every name one name. AlaSQL writes string and JSON literals without - * escaping. + * escaping, and numbers in a form it reads back as a column. */ import alasql from 'alasql'; +import clone from 'clone'; +import { backtickASTSchemaItems } from '../utility/common_utils.ts'; import { ClientError } from '../utility/errors/hdbError.ts'; import { astNodes } from './sql_statement_bucket.ts'; @@ -16,8 +18,8 @@ const BARE_NAME = /^\w+$/; /** * Single-quoted text that AlaSQL decodes back to `value`, or undefined when there is none. AlaSQL lexes a - * backslash as taking the next character, then decodes `\'` and `''` to `'`, so a quote after an even run of - * backslashes, or an odd run at the end, has no spelling. + * backslash as taking the next character, then decodes `\'` and `''` to `'`, so an odd run of backslashes at + * the end has no spelling. */ export function quoteString(value: string): string | undefined { let text = "'"; @@ -32,9 +34,8 @@ export function quoteString(value: string): string | undefined { break; } if (next === "'") { - if (backslashes === 0) text += "''"; - else if (backslashes % 2) text += '\\'.repeat(backslashes + 1) + "''"; - else return undefined; + if (backslashes % 2) text += '\\'.repeat(backslashes + 1) + "''"; + else text += '\\'.repeat(backslashes) + (backslashes ? "\\'" : "''"); } else { text += value.slice(i, i + backslashes + 1); } @@ -60,6 +61,21 @@ function lexesInside(body: string, quote: string): boolean { return true; } +// AlaSQL reads `1e+25` as `1e + 25` and `Infinity` as a column, so write every number in plain decimal. +function numberText(value: number): string { + if (!Number.isFinite(value)) throw unwritable(`The number ${value}`); + const text = String(value); + const e = text.indexOf('e'); + if (e < 0) return text; + const sign = text[0] === '-' ? '-' : ''; + const [whole, fraction = ''] = text.slice(sign.length, e).split('.'); + const digits = whole + fraction; + const point = whole.length + Number(text.slice(e + 1)); + if (point <= 0) return `${sign}0.${'0'.repeat(-point)}${digits}`; + if (point >= digits.length) return sign + digits + '0'.repeat(point - digits.length); + return `${sign}${digits.slice(0, point)}.${digits.slice(point)}`; +} + function unwritable(what: string) { return new ClientError( `${what} cannot be written back to SQL text for the legacy SQL engine. Remove the quotes, backticks, brackets or backslashes, or use the new SQL engine.` @@ -77,7 +93,8 @@ function jsonText(value: any): string { return text; } if (value == null) return 'NULL'; - if (typeof value !== 'object') return typeof value === 'bigint' ? `${value}n` : String(value); + if (typeof value === 'number') return numberText(value); + if (typeof value !== 'object') return String(value); // Any other object is an AST node inside the literal, a nested JSON literal included. if (!isJsonData(value)) return value.toString(); if (Array.isArray(value)) return `@[${value.map(jsonText).join(',')}]`; @@ -99,6 +116,29 @@ export function installLegacySqlText() { yy.Json.prototype.toString = function () { return jsonText(this.value); }; + yy.NumValue.prototype.toString = function () { + return typeof this.value === 'number' ? numberText(this.value) : String(this.value); + }; +} + +// What a statement reads or calls by name. +const NAME_KEYS = ['columnid', 'tableid', 'databaseid', 'funcid', 'aggregatorid', 'variable', 'param']; + +function referencedNames(statement: any): Set { + const names = new Set(); + for (const node of statementNodes(statement)) { + if (node instanceof yy.JavaScript) names.add(`javascript=${node.value}`); + for (const key of NAME_KEYS) { + if (!Object.hasOwn(node, key)) continue; + // `?` is param 0, written back as `$0`. + let name = typeof node[key] === 'number' ? String(node[key]) : node[key]; + if (typeof name !== 'string') continue; + if (name.length > 1 && name.startsWith('`') && name.endsWith('`')) name = name.slice(1, -1); + // AlaSQL writes function names upper-cased. + names.add(`${key}=${key === 'funcid' || key === 'aggregatorid' ? name.toUpperCase() : name}`); + } + } + return names; } // AST nodes, without the plain objects and arrays that are a JSON literal's data. @@ -143,4 +183,19 @@ export function assertLegacySqlText(statement: any) { } } } + // Catches any writer not replaced here: the text legacy runs, with Harper's backticks, may read nothing new. + const written = clone(statement); + // `SELECT *` for a role that may read no column leaves no columns, which does not parse. + if (written instanceof yy.Select && written.columns?.length === 0) written.columns = [new yy.NumValue({ value: 1 })]; + backtickASTSchemaItems(written); + let reparsed: any; + try { + reparsed = (alasql as any).parse(written.toString()).statements[0]; + } catch { + throw unwritable('The statement'); + } + const names = referencedNames(statement); + for (const name of referencedNames(reparsed)) { + if (!names.has(name)) throw unwritable('The statement'); + } } diff --git a/unitTests/sqlTranslator/legacySqlText.test.js b/unitTests/sqlTranslator/legacySqlText.test.js index ad6efb0b30..e18faeedef 100644 --- a/unitTests/sqlTranslator/legacySqlText.test.js +++ b/unitTests/sqlTranslator/legacySqlText.test.js @@ -51,7 +51,14 @@ describe('legacy SQL text', function () { } if (columns.length !== 1 || columns[0].as !== 'v') continue; const text = quoteString(columns[0].value); - if (text === undefined) continue; + if (text === undefined) { + assert.match( + columns[0].value, + /(^|[^\\])(\\\\)*\\$/, + `only a trailing odd run of backslashes is refused: ${body}` + ); + continue; + } const again = parse(`SELECT ${text} AS v FROM t WHERE w = 'z'`).columns; assert.deepStrictEqual( again.map((c) => c.value), @@ -96,6 +103,35 @@ describe('legacy SQL text', function () { } }); + it('writes numbers in plain decimal, which reads back as the same number', function () { + for (const literal of [ + '10000000000000000000000000', + '0.0000001', + '123.5', + "{'a':0.0000001,'b':@[10000000000000000000000000]}", + ]) { + const before = parse(`SELECT ${literal} AS v FROM dev.t`).columns; + const after = reparsed(`SELECT ${literal} AS v FROM dev.t`).columns; + assert.strictEqual(after.length, 1, literal); + assert.deepStrictEqual(after[0].value, before[0].value, literal); + } + assertUnwritable(`SELECT ${'9'.repeat(400)} AS v FROM dev.t`); + assertUnwritable(`SELECT {'a':${'9'.repeat(400)}} AS v FROM dev.t`); + }); + + it('refuses a statement whose text reads a name the statement does not', function () { + assertUnwritable('SELECT label->[x, ssn] AS v FROM dev.t'); + const writer = alasql.yy.StringValue.prototype.toString; + alasql.yy.StringValue.prototype.toString = function () { + return "'" + this.value + "'"; + }; + try { + assertUnwritable("SELECT 'x'', ssn, ''y' AS v FROM dev.t"); + } finally { + alasql.yy.StringValue.prototype.toString = writer; + } + }); + it('refuses a string literal that has no spelling', function () { assertUnwritable("SELECT 'a\\' AS v FROM dev.t"); assertUnwritable("UPDATE dev.t SET label = {'k':'a\\'} WHERE id = 1"); @@ -109,8 +145,12 @@ describe('legacy SQL text', function () { 'SELECT DISTINCT lower(a) AS l FROM dev.t WHERE EXISTS (SELECT 1 FROM dev.u WHERE u.id = t.id) ORDER BY l DESC LIMIT 2', "SELECT CASE WHEN a = 1 THEN 'x' ELSE 'y' END AS c, CAST(b AS INT) AS n FROM dev.t WHERE b NOT IN (1, 2)", 'SELECT @x AS v, $1 AS w, ? AS z FROM dev.t', + 'SELECT label->x AS v, COUNT(*) AS `count`, `select` FROM dev.t GROUP BY label->x, `select`', ]) { assertLegacySqlText(parse(sql)); } + const noReadableColumn = parse("SELECT * FROM dev.t WHERE label = 'a'"); + noReadableColumn.columns = []; + assertLegacySqlText(noReadableColumn); }); }); From 230286b837de2ad6af3032f2b5ead4ab9b684aa0 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 1 Nov 2026 22:55:16 -0700 Subject: [PATCH 12/19] Compare legacy SQL's re-parsed text by table-qualified column and count The re-parse check compared flat names, so an injected r.id passed when r and id each appeared elsewhere in the statement. It now counts each node's database, table and column together and refuses a re-parse that reads any of them more often, or that holds more than one statement. The name after -> or ! must be a bare name, since AlaSQL writes it unquoted. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RGfHqd6v2q9t9jJ26No4Jc Dispatch-Task: pr-maint-abc8bf322b5ccfd671a9379f3776cea4 --- .../security/sql-attribute-perms.test.ts | 6 ++++ sqlTranslator/DESIGN.md | 2 +- sqlTranslator/legacySqlText.ts | 32 +++++++++++-------- unitTests/sqlTranslator/legacySqlText.test.js | 12 +++++-- 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/integrationTests/security/sql-attribute-perms.test.ts b/integrationTests/security/sql-attribute-perms.test.ts index 8c68523b86..77febe29ed 100644 --- a/integrationTests/security/sql-attribute-perms.test.ts +++ b/integrationTests/security/sql-attribute-perms.test.ts @@ -277,6 +277,12 @@ for (const sqlEngine of ['legacy', 'auto', 'new']) { else ok(n.status === 200 || n.status === 400, result); ok(!JSON.stringify(n.body).includes('111'), result); } + + // Legacy writes the name after `->` unquoted; v2 has no `->`. + const member = `SELECT l.label->[x, r.id] AS v FROM data.ColPermLeft l JOIN data.ColPermRight r ON l.label = r.label WHERE l.id = 'l1'`; + const m = await asReader(member); + strictEqual(m.status, 400, `${member} -> ${m.status} ${JSON.stringify(m.body).slice(0, 300)}`); + ok(!JSON.stringify(m.body).includes(RIGHT_PK), JSON.stringify(m.body)); }); test('super user is unaffected', async () => { diff --git a/sqlTranslator/DESIGN.md b/sqlTranslator/DESIGN.md index a7410f9e62..769691dc9f 100644 --- a/sqlTranslator/DESIGN.md +++ b/sqlTranslator/DESIGN.md @@ -24,6 +24,6 @@ Enforced by `unitTests/sqlTranslator/column-attribution.test.js`, `unitTests/sql ## Legacy SQL runs the re-parse of the authorized statement's text (`legacySqlText.ts`) -Legacy SQL writes the authorized AST back out with `toString()` and AlaSQL parses that text again (`dataLayer/SQLSearch.ts` `_buildSQL()`, `dataLayer/update.ts`, `deleteTranslator.ts`, and the compiled SET and INSERT expressions), so the permission check holds only if the text parses back to the same statement. AlaSQL writes string and JSON literals unescaped, so `'x'', ssn, ''y'` came back as three columns, and numbers as `1e+25` or `Infinity`, which read back as columns. `installLegacySqlText()`, run by `alasqlFunctionImporter`, replaces the `StringValue`, `Json` and `NumValue` writers with ones that read back as the same value. `assertLegacySqlText()` refuses, with a 400 on the legacy path only, what has no spelling: a name containing `` ` ``, `[`, `]` or `'`, which backticks and brackets cannot escape, an `@variable` or `$param` name other than letters, digits and `_`, which AlaSQL writes unquoted, a string ending in an odd run of backslashes, or a number that overflows. It then re-parses the backticked text legacy runs and refuses a statement whose text names a column, table, function or variable the statement does not, so an AlaSQL writer not replaced here (the right side of `->`, for one) fails closed. v2 binds the AST directly. +Legacy SQL writes the authorized AST back out with `toString()` and AlaSQL parses that text again (`dataLayer/SQLSearch.ts` `_buildSQL()`, `dataLayer/update.ts`, `deleteTranslator.ts`, and the compiled SET and INSERT expressions), so the permission check holds only if the text parses back to the same statement. AlaSQL writes string and JSON literals unescaped, so `'x'', ssn, ''y'` came back as three columns, and numbers as `1e+25` or `Infinity`, which read back as columns. `installLegacySqlText()`, run by `alasqlFunctionImporter`, replaces the `StringValue`, `Json` and `NumValue` writers with ones that read back as the same value. `assertLegacySqlText()` refuses, with a 400 on the legacy path only, what has no spelling: a name containing `` ` ``, `[`, `]` or `'`, which backticks and brackets cannot escape, an `@variable`, `$param` or `->`/`!` member name other than letters, digits and `_`, which AlaSQL writes unquoted, a string ending in an odd run of backslashes, or a number that overflows. It then re-parses the backticked text legacy runs and refuses it unless it is one statement that reads each table-qualified column, function and variable no more often than the authorized statement does, so an AlaSQL writer not replaced here fails closed. v2 binds the AST directly. Enforced by `unitTests/sqlTranslator/legacySqlText.test.js` and the quoted-string case in `integrationTests/security/sql-attribute-perms.test.ts`. diff --git a/sqlTranslator/legacySqlText.ts b/sqlTranslator/legacySqlText.ts index 7687ce17fb..dec0c906d2 100644 --- a/sqlTranslator/legacySqlText.ts +++ b/sqlTranslator/legacySqlText.ts @@ -13,7 +13,7 @@ const yy = (alasql as any).yy; // Harper writes names inside backticks, and AlaSQL inside brackets; neither can escape these. const UNQUOTABLE_NAME = /[`[\]']/; -// AlaSQL writes `@variable` and `$param` with no quoting at all. +// AlaSQL writes `@variable`, `$param` and the name after `->` or `!` with no quoting at all. const BARE_NAME = /^\w+$/; /** @@ -122,12 +122,13 @@ export function installLegacySqlText() { } // What a statement reads or calls by name. -const NAME_KEYS = ['columnid', 'tableid', 'databaseid', 'funcid', 'aggregatorid', 'variable', 'param']; +const NAME_KEYS = ['databaseid', 'tableid', 'columnid', 'funcid', 'aggregatorid', 'variable', 'param']; -function referencedNames(statement: any): Set { - const names = new Set(); +// How many times each node reads or calls each name; a column counts with its table, as one reference. +function referenceCounts(statement: any): Map { + const counts = new Map(); for (const node of statementNodes(statement)) { - if (node instanceof yy.JavaScript) names.add(`javascript=${node.value}`); + const parts = node instanceof yy.JavaScript ? [`javascript=${node.value}`] : []; for (const key of NAME_KEYS) { if (!Object.hasOwn(node, key)) continue; // `?` is param 0, written back as `$0`. @@ -135,10 +136,13 @@ function referencedNames(statement: any): Set { if (typeof name !== 'string') continue; if (name.length > 1 && name.startsWith('`') && name.endsWith('`')) name = name.slice(1, -1); // AlaSQL writes function names upper-cased. - names.add(`${key}=${key === 'funcid' || key === 'aggregatorid' ? name.toUpperCase() : name}`); + parts.push(`${key}=${key === 'funcid' || key === 'aggregatorid' ? name.toUpperCase() : name}`); } + if (parts.length === 0) continue; + const reference = parts.join('\u0000'); + counts.set(reference, (counts.get(reference) ?? 0) + 1); } - return names; + return counts; } // AST nodes, without the plain objects and arrays that are a JSON literal's data. @@ -165,7 +169,8 @@ export function assertLegacySqlText(statement: any) { throw unwritable('A string literal'); } if (node instanceof yy.Json) jsonText(node.value); - for (const key of ['variable', 'param']) { + const bareKeys = node instanceof yy.Op && (node.op === '->' || node.op === '!') ? ['right'] : ['variable', 'param']; + for (const key of bareKeys) { const value = node[key]; if (Object.hasOwn(node, key) && typeof value === 'string' && !BARE_NAME.test(value)) { throw unwritable(`The name ${JSON.stringify(value)}`); @@ -188,14 +193,15 @@ export function assertLegacySqlText(statement: any) { // `SELECT *` for a role that may read no column leaves no columns, which does not parse. if (written instanceof yy.Select && written.columns?.length === 0) written.columns = [new yy.NumValue({ value: 1 })]; backtickASTSchemaItems(written); - let reparsed: any; + let statements: any[]; try { - reparsed = (alasql as any).parse(written.toString()).statements[0]; + statements = (alasql as any).parse(written.toString()).statements; } catch { throw unwritable('The statement'); } - const names = referencedNames(statement); - for (const name of referencedNames(reparsed)) { - if (!names.has(name)) throw unwritable('The statement'); + if (statements.length !== 1) throw unwritable('The statement'); + const authorized = referenceCounts(statement); + for (const [reference, count] of referenceCounts(statements[0])) { + if (count > (authorized.get(reference) ?? 0)) throw unwritable('The statement'); } } diff --git a/unitTests/sqlTranslator/legacySqlText.test.js b/unitTests/sqlTranslator/legacySqlText.test.js index e18faeedef..bf96950ba5 100644 --- a/unitTests/sqlTranslator/legacySqlText.test.js +++ b/unitTests/sqlTranslator/legacySqlText.test.js @@ -98,6 +98,8 @@ describe('legacy SQL text', function () { "SELECT {'k':1} AS [x`,ssn,`y] FROM dev.t", 'SELECT @`x,ssn` AS v FROM dev.t', 'SELECT $[x,ssn] AS v FROM dev.t', + 'SELECT label->[x, ssn] AS v FROM dev.t', + 'SELECT label![x, ssn] AS v FROM dev.t', ]) { assertUnwritable(sql); } @@ -119,14 +121,20 @@ describe('legacy SQL text', function () { assertUnwritable(`SELECT {'a':${'9'.repeat(400)}} AS v FROM dev.t`); }); - it('refuses a statement whose text reads a name the statement does not', function () { - assertUnwritable('SELECT label->[x, ssn] AS v FROM dev.t'); + it('refuses a statement whose text reads a column the statement does not, or reads it more often', function () { const writer = alasql.yy.StringValue.prototype.toString; alasql.yy.StringValue.prototype.toString = function () { return "'" + this.value + "'"; }; try { assertUnwritable("SELECT 'x'', ssn, ''y' AS v FROM dev.t"); + // `r` and `id` each appear elsewhere; `r.id` does not. + assertUnwritable( + "SELECT l.label, 'x'', r.id, ''y' AS v FROM dev.l AS l INNER JOIN dev.r AS r ON l.label = r.label WHERE l.id = 1" + ); + assertUnwritable("UPDATE dev.t SET ssn = '1', label = CONCAT('x'', ssn, ''y') WHERE id = 1"); + assertUnwritable("SELECT COUNT(*) AS c, 'x'', *, ''y' AS v FROM dev.t"); + assertUnwritable("SELECT 'x''; SELECT ssn FROM dev.t; SELECT ''y' AS v FROM dev.t"); } finally { alasql.yy.StringValue.prototype.toString = writer; } From f9b3d8deb630515c33dd232f954665cd6b739470 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 1 Nov 2026 23:22:08 -0700 Subject: [PATCH 13/19] Fit the SQL design notes under the root index ceiling Main now holds the root DESIGN.md to a byte ceiling it was already over. Shorten this branch's two sqlTranslator headings and index entries, and trim five index descriptions to their invariant; the note bodies are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RGfHqd6v2q9t9jJ26No4Jc Dispatch-Task: pr-maint-abc8bf322b5ccfd671a9379f3776cea4 --- DESIGN.md | 2 +- sqlTranslator/DESIGN.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 93a61eab58..7bd17406fe 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -6,7 +6,7 @@ A note lives in the `DESIGN.md` of the directory that owns the code it describes ## sqlTranslator/ — legacy SQL integration -- [CommonJS date functions](sqlTranslator/DESIGN.md#commonjs-date-functions) — Both runtimes use the same CommonJS helper object, enforced by `alasqlFunctionImporter.test.js`. · [SQL authorization attributes every column with the engine's own resolution rules (`sql_statement_bucket.ts`, `sqlEngine/binder/columnScope.ts`)](sqlTranslator/DESIGN.md#sql-authorization-attributes-every-column-with-the-engines-own-resolution-rules-sql_statement_bucketts-sqlenginebindercolumnscopets) — The bucket and the v2 binder share one column resolver; unresolved qualifiers and whole-row accessors are refused for non-super users. · [Legacy SQL runs the re-parse of the authorized statement's text (`legacySqlText.ts`)](sqlTranslator/DESIGN.md#legacy-sql-runs-the-re-parse-of-the-authorized-statements-text-legacysqltextts) — Literals are written so they decode to the same value; names and strings with no spelling get a 400 on legacy. +- [CommonJS date functions](sqlTranslator/DESIGN.md#commonjs-date-functions) — Both runtimes use the same CommonJS helper object, enforced by `alasqlFunctionImporter.test.js`. · [SQL column attribution](sqlTranslator/DESIGN.md#sql-column-attribution) — Shared column resolver. · [Legacy SQL text round trip](sqlTranslator/DESIGN.md#legacy-sql-text-round-trip) — Re-parsed text reads nothing new. ## .github/ — CI workflows diff --git a/sqlTranslator/DESIGN.md b/sqlTranslator/DESIGN.md index 769691dc9f..89e8c634ea 100644 --- a/sqlTranslator/DESIGN.md +++ b/sqlTranslator/DESIGN.md @@ -4,7 +4,7 @@ `alasqlFunctionImporter.ts` imports the CommonJS date-helper object by default; native namespace imports did not expose its object-literal properties in [#562](https://github.com/HarperFast/harper/pull/562). `unitTests/sqlTranslator/alasqlFunctionImporter.test.js` verifies the registered date functions in both test runtimes. -## SQL authorization attributes every column with the engine's own resolution rules (`sql_statement_bucket.ts`, `sqlEngine/binder/columnScope.ts`) +## SQL column attribution `verifyPermsAST` checks attribute permissions only for the attributes `sqlStatementBucket` records per table, and it runs once, before the router picks legacy or v2, so the bucket alone decides what every engine may read. A column the bucket misses, or records against the wrong table, is read unchecked. @@ -22,7 +22,7 @@ Row accessors that name no column, AlaSQL's `_`, JavaScript expressions and a JS Enforced by `unitTests/sqlTranslator/column-attribution.test.js`, `unitTests/sqlEngine/columnScope.test.js` and `integrationTests/security/sql-attribute-perms.test.ts`. -## Legacy SQL runs the re-parse of the authorized statement's text (`legacySqlText.ts`) +## Legacy SQL text round trip Legacy SQL writes the authorized AST back out with `toString()` and AlaSQL parses that text again (`dataLayer/SQLSearch.ts` `_buildSQL()`, `dataLayer/update.ts`, `deleteTranslator.ts`, and the compiled SET and INSERT expressions), so the permission check holds only if the text parses back to the same statement. AlaSQL writes string and JSON literals unescaped, so `'x'', ssn, ''y'` came back as three columns, and numbers as `1e+25` or `Infinity`, which read back as columns. `installLegacySqlText()`, run by `alasqlFunctionImporter`, replaces the `StringValue`, `Json` and `NumValue` writers with ones that read back as the same value. `assertLegacySqlText()` refuses, with a 400 on the legacy path only, what has no spelling: a name containing `` ` ``, `[`, `]` or `'`, which backticks and brackets cannot escape, an `@variable`, `$param` or `->`/`!` member name other than letters, digits and `_`, which AlaSQL writes unquoted, a string ending in an odd run of backslashes, or a number that overflows. It then re-parses the backticked text legacy runs and refuses it unless it is one statement that reads each table-qualified column, function and variable no more often than the authorized statement does, so an AlaSQL writer not replaced here fails closed. v2 binds the AST directly. From f223898eb0c811e8a4fffc4faf64ca86e0daf7b6 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 2 Nov 2026 00:04:14 -0700 Subject: [PATCH 14/19] Check legacy SQL text without copying it, and only the text legacy writes The round-trip check deep-copied each statement; it now backticks the statement in place and restores it. INSERT takes literal values as data, so only its other values, which legacy writes as SELECT text, are checked: a 10k-row INSERT check drops from seconds to milliseconds, and a literal with no SQL spelling is no longer refused. In a SELECT that reads a table, SQLSearch aliases an unaliased function column as [], so a ] in that text now gets a 400 on legacy. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RGfHqd6v2q9t9jJ26No4Jc Dispatch-Task: pr-maint-abc8bf322b5ccfd671a9379f3776cea4 --- .../security/sql-attribute-perms.test.ts | 8 +++ sqlTranslator/DESIGN.md | 2 +- sqlTranslator/legacySqlText.ts | 63 ++++++++++++++++--- unitTests/sqlTranslator/legacySqlText.test.js | 33 ++++++++++ 4 files changed, 95 insertions(+), 11 deletions(-) diff --git a/integrationTests/security/sql-attribute-perms.test.ts b/integrationTests/security/sql-attribute-perms.test.ts index 77febe29ed..650053881b 100644 --- a/integrationTests/security/sql-attribute-perms.test.ts +++ b/integrationTests/security/sql-attribute-perms.test.ts @@ -283,6 +283,14 @@ for (const sqlEngine of ['legacy', 'auto', 'new']) { const m = await asReader(member); strictEqual(m.status, 400, `${member} -> ${m.status} ${JSON.stringify(m.body).slice(0, 300)}`); ok(!JSON.stringify(m.body).includes(RIGHT_PK), JSON.stringify(m.body)); + + // Legacy names an unaliased function column after its text, in brackets. + const call = `SELECT UPPER('x], id, [y') FROM data.ColPermRight WHERE label = 'shared'`; + const c = await asReader(call); + const callResult = `${call} -> ${c.status} ${JSON.stringify(c.body).slice(0, 300)}`; + if (sqlEngine === 'legacy') strictEqual(c.status, 400, callResult); + else ok(c.status === 200 || c.status === 400, callResult); + ok(!JSON.stringify(c.body).includes(RIGHT_PK), callResult); }); test('super user is unaffected', async () => { diff --git a/sqlTranslator/DESIGN.md b/sqlTranslator/DESIGN.md index 89e8c634ea..ebd3414774 100644 --- a/sqlTranslator/DESIGN.md +++ b/sqlTranslator/DESIGN.md @@ -24,6 +24,6 @@ Enforced by `unitTests/sqlTranslator/column-attribution.test.js`, `unitTests/sql ## Legacy SQL text round trip -Legacy SQL writes the authorized AST back out with `toString()` and AlaSQL parses that text again (`dataLayer/SQLSearch.ts` `_buildSQL()`, `dataLayer/update.ts`, `deleteTranslator.ts`, and the compiled SET and INSERT expressions), so the permission check holds only if the text parses back to the same statement. AlaSQL writes string and JSON literals unescaped, so `'x'', ssn, ''y'` came back as three columns, and numbers as `1e+25` or `Infinity`, which read back as columns. `installLegacySqlText()`, run by `alasqlFunctionImporter`, replaces the `StringValue`, `Json` and `NumValue` writers with ones that read back as the same value. `assertLegacySqlText()` refuses, with a 400 on the legacy path only, what has no spelling: a name containing `` ` ``, `[`, `]` or `'`, which backticks and brackets cannot escape, an `@variable`, `$param` or `->`/`!` member name other than letters, digits and `_`, which AlaSQL writes unquoted, a string ending in an odd run of backslashes, or a number that overflows. It then re-parses the backticked text legacy runs and refuses it unless it is one statement that reads each table-qualified column, function and variable no more often than the authorized statement does, so an AlaSQL writer not replaced here fails closed. v2 binds the AST directly. +Legacy SQL writes the authorized AST back out with `toString()` and AlaSQL parses that text again (`dataLayer/SQLSearch.ts` `_buildSQL()`, `dataLayer/update.ts`, `deleteTranslator.ts`, and the compiled SET and INSERT expressions), so the permission check holds only if the text parses back to the same statement. AlaSQL writes string and JSON literals unescaped, so `'x'', ssn, ''y'` came back as three columns, and numbers as `1e+25` or `Infinity`, which read back as columns. `installLegacySqlText()`, run by `alasqlFunctionImporter`, replaces the `StringValue`, `Json` and `NumValue` writers with ones that read back as the same value. `assertLegacySqlText()` refuses, with a 400 on the legacy path only, what the writers cannot spell: a name containing `` ` ``, `[`, `]` or `'` (brackets cannot hold `]`, and Harper leaves an alias starting with `[` unbackticked), an `@variable`, `$param` or `->`/`!` member name other than letters, digits and `_`, which AlaSQL writes unquoted, a string ending in an odd run of backslashes, which has no single-quoted spelling, a number that overflows, and, in a SELECT that reads a table, an unaliased function column whose text holds `]`, since `SQLSearch` aliases it as `[]`. It then re-parses the backticked statement text and refuses it unless it is one statement that reads each table-qualified column, function and variable no more often than the authorized statement does, so an AlaSQL writer not replaced here fails closed. INSERT takes literal values as data and writes only its other values as text, so only those are checked. v2 binds the AST directly. Enforced by `unitTests/sqlTranslator/legacySqlText.test.js` and the quoted-string case in `integrationTests/security/sql-attribute-perms.test.ts`. diff --git a/sqlTranslator/legacySqlText.ts b/sqlTranslator/legacySqlText.ts index dec0c906d2..028fe388f7 100644 --- a/sqlTranslator/legacySqlText.ts +++ b/sqlTranslator/legacySqlText.ts @@ -4,7 +4,6 @@ * escaping, and numbers in a form it reads back as a column. */ import alasql from 'alasql'; -import clone from 'clone'; import { backtickASTSchemaItems } from '../utility/common_utils.ts'; import { ClientError } from '../utility/errors/hdbError.ts'; import { astNodes } from './sql_statement_bucket.ts'; @@ -162,9 +161,34 @@ function* statementNodes(statement: any): Generator { } } -/** Throws a 400 for a statement whose SQL text would not parse back to what it reads. */ +/** Throws a 400 for a statement whose SQL text, as legacy SQL runs it, would not parse back to what it reads. */ export function assertLegacySqlText(statement: any) { - for (const node of statementNodes(statement)) { + if (statement instanceof yy.Insert) { + // INSERT takes literal values as data, and runs each other value as `SELECT AS [...] FROM ?`. + for (const row of statement.values ?? []) { + for (const value of row) { + if ('value' in value) continue; + assertWritableNodes(value); + const authorized = referenceCounts(value); + authorized.set('param=0', (authorized.get('param=0') ?? 0) + 1); + assertReparse(`SELECT ${value} AS [v] FROM ?`, authorized); + } + } + return; + } + assertWritableNodes(statement); + // SQLSearch names an unaliased function column `[]` when the statement reads a table. + for (const column of statement instanceof yy.Select && statement.from?.length ? statement.columns : []) { + if (column.funcid && column.args && !column.aggregatorid && !column.as && column.toString().includes(']')) { + throw unwritable('A function column holding `]` without an alias'); + } + } + const authorized = referenceCounts(statement); + assertReparse(legacyText(statement), authorized); +} + +function assertWritableNodes(root: any) { + for (const node of statementNodes(root)) { if (node instanceof yy.StringValue && quoteString(String(node.value)) === undefined) { throw unwritable('A string literal'); } @@ -188,20 +212,39 @@ export function assertLegacySqlText(statement: any) { } } } - // Catches any writer not replaced here: the text legacy runs, with Harper's backticks, may read nothing new. - const written = clone(statement); - // `SELECT *` for a role that may read no column leaves no columns, which does not parse. - if (written instanceof yy.Select && written.columns?.length === 0) written.columns = [new yy.NumValue({ value: 1 })]; - backtickASTSchemaItems(written); +} + +// Catches any writer not replaced here: the text legacy runs may read nothing the statement does not. +function assertReparse(text: string, authorized: Map) { let statements: any[]; try { - statements = (alasql as any).parse(written.toString()).statements; + statements = (alasql as any).parse(text).statements; } catch { throw unwritable('The statement'); } if (statements.length !== 1) throw unwritable('The statement'); - const authorized = referenceCounts(statement); for (const [reference, count] of referenceCounts(statements[0])) { if (count > (authorized.get(reference) ?? 0)) throw unwritable('The statement'); } } + +// The statement's text with Harper's backticks, leaving the statement as it was. +function legacyText(statement: any): string { + const columns = statement.columns; + // `SELECT *` for a role that may read no column leaves no columns, which does not parse. + const placeholder = statement instanceof yy.Select && columns?.length === 0; + if (placeholder) statement.columns = [new yy.NumValue({ value: 1 })]; + backtickASTSchemaItems(statement); + try { + return statement.toString(); + } finally { + for (const node of astNodes(statement)) { + for (const key of ['columnid', 'tableid', 'databaseid', 'as']) { + if (!Object.hasOwn(node, `${key}_orig`)) continue; + node[key] = node[`${key}_orig`]; + delete node[`${key}_orig`]; + } + } + if (placeholder) statement.columns = columns; + } +} diff --git a/unitTests/sqlTranslator/legacySqlText.test.js b/unitTests/sqlTranslator/legacySqlText.test.js index bf96950ba5..eb331edd32 100644 --- a/unitTests/sqlTranslator/legacySqlText.test.js +++ b/unitTests/sqlTranslator/legacySqlText.test.js @@ -135,11 +135,44 @@ describe('legacy SQL text', function () { assertUnwritable("UPDATE dev.t SET ssn = '1', label = CONCAT('x'', ssn, ''y') WHERE id = 1"); assertUnwritable("SELECT COUNT(*) AS c, 'x'', *, ''y' AS v FROM dev.t"); assertUnwritable("SELECT 'x''; SELECT ssn FROM dev.t; SELECT ''y' AS v FROM dev.t"); + assertUnwritable("INSERT INTO dev.t (id, label) VALUES (1, CONCAT('x'', ssn, ''y'))"); } finally { alasql.yy.StringValue.prototype.toString = writer; } }); + it('refuses an unaliased function column whose text holds a closing bracket', function () { + assertUnwritable("SELECT UPPER('x], id, [y') FROM dev.t"); + assertLegacySqlText(parse("SELECT UPPER('x], id, [y') AS u FROM dev.t")); + assertLegacySqlText(parse("SELECT geoArea('[[0,0],[1,1]]')")); + }); + + it('checks only the INSERT values legacy SQL writes as text', function () { + // Literal values are data, so one with no SQL spelling is still accepted. + assertLegacySqlText(parse("INSERT INTO dev.t (id, path) VALUES (1, 'a\\')")); + assertLegacySqlText(parse("INSERT INTO dev.t (id, label) VALUES (1, CONCAT('x'', ssn, ''y'))")); + assertUnwritable("INSERT INTO dev.t (id, label) VALUES (1, CONCAT('x', 'a\\'))"); + }); + + it('leaves the statement as it was', function () { + for (const sql of [ + "INSERT INTO dev.t (id, `count`) VALUES (1, {'columnid':'a'})", + 'SELECT `count` AS c FROM dev.t AS d WHERE d.id = 1', + "DELETE FROM dev.t WHERE label = 'a'", + ]) { + const statement = parse(sql); + const before = statement.toString(); + assertLegacySqlText(statement); + assert.strictEqual(statement.toString(), before, sql); + assert.ok(!Object.hasOwn(statement, 'columns') || statement.columns !== undefined, sql); + } + const insert = parse('INSERT INTO dev.t (id) VALUES (1)'); + assertLegacySqlText(insert); + assert.strictEqual(insert.into.tableid, 't'); + assert.strictEqual(insert.columns[0].columnid, 'id'); + assert.ok(!Object.hasOwn(insert.into, 'tableid_orig')); + }); + it('refuses a string literal that has no spelling', function () { assertUnwritable("SELECT 'a\\' AS v FROM dev.t"); assertUnwritable("UPDATE dev.t SET label = {'k':'a\\'} WHERE id = 1"); From 26e9ae0db041cd2bf690eee9a791526389021a99 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 2 Nov 2026 00:23:47 -0700 Subject: [PATCH 15/19] Restore exactly the keys the legacy text check backticks The restore moved any own *_orig key back, so a JSON literal holding a key named columnid_orig was rewritten. It now saves each backticked key and its _orig twin before backticking and puts back exactly those. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RGfHqd6v2q9t9jJ26No4Jc Dispatch-Task: pr-maint-abc8bf322b5ccfd671a9379f3776cea4 --- sqlTranslator/legacySqlText.ts | 26 ++++++++++++------- unitTests/sqlTranslator/legacySqlText.test.js | 1 + 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/sqlTranslator/legacySqlText.ts b/sqlTranslator/legacySqlText.ts index 028fe388f7..723dec6be4 100644 --- a/sqlTranslator/legacySqlText.ts +++ b/sqlTranslator/legacySqlText.ts @@ -120,10 +120,9 @@ export function installLegacySqlText() { }; } -// What a statement reads or calls by name. const NAME_KEYS = ['databaseid', 'tableid', 'columnid', 'funcid', 'aggregatorid', 'variable', 'param']; -// How many times each node reads or calls each name; a column counts with its table, as one reference. +// A column counts together with its table and database, as one reference. function referenceCounts(statement: any): Map { const counts = new Map(); for (const node of statementNodes(statement)) { @@ -144,7 +143,7 @@ function referenceCounts(statement: any): Map { return counts; } -// AST nodes, without the plain objects and arrays that are a JSON literal's data. +// Skips the plain objects and arrays that are a JSON literal's data. function* statementNodes(statement: any): Generator { const literalData = new Set(); for (const node of astNodes(statement)) { @@ -228,22 +227,31 @@ function assertReparse(text: string, authorized: Map) { } } +const BACKTICKED_KEYS = ['columnid', 'tableid', 'databaseid', 'as']; + // The statement's text with Harper's backticks, leaving the statement as it was. function legacyText(statement: any): string { const columns = statement.columns; // `SELECT *` for a role that may read no column leaves no columns, which does not parse. const placeholder = statement instanceof yy.Select && columns?.length === 0; if (placeholder) statement.columns = [new yy.NumValue({ value: 1 })]; + // backtickASTSchemaItems rewrites these keys on every object it reaches, JSON literal data included. + const saved = []; + for (const node of astNodes(statement)) { + for (const key of BACKTICKED_KEYS) { + if (!Object.hasOwn(node, key)) continue; + const orig = `${key}_orig`; + saved.push({ node, key, value: node[key], hadOrig: Object.hasOwn(node, orig), origValue: node[orig] }); + } + } backtickASTSchemaItems(statement); try { return statement.toString(); } finally { - for (const node of astNodes(statement)) { - for (const key of ['columnid', 'tableid', 'databaseid', 'as']) { - if (!Object.hasOwn(node, `${key}_orig`)) continue; - node[key] = node[`${key}_orig`]; - delete node[`${key}_orig`]; - } + for (const { node, key, value, hadOrig, origValue } of saved) { + node[key] = value; + if (hadOrig) node[`${key}_orig`] = origValue; + else delete node[`${key}_orig`]; } if (placeholder) statement.columns = columns; } diff --git a/unitTests/sqlTranslator/legacySqlText.test.js b/unitTests/sqlTranslator/legacySqlText.test.js index eb331edd32..e9ccc2c106 100644 --- a/unitTests/sqlTranslator/legacySqlText.test.js +++ b/unitTests/sqlTranslator/legacySqlText.test.js @@ -159,6 +159,7 @@ describe('legacy SQL text', function () { "INSERT INTO dev.t (id, `count`) VALUES (1, {'columnid':'a'})", 'SELECT `count` AS c FROM dev.t AS d WHERE d.id = 1', "DELETE FROM dev.t WHERE label = 'a'", + "UPDATE dev.t SET label = {'columnid_orig':'v1','columnid':'c','as':'a'} WHERE `count` = 1", ]) { const statement = parse(sql); const before = statement.toString(); From bbe6649bc090179dc4f18526ab9026fc0f6f4d51 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 2 Nov 2026 08:57:18 -0700 Subject: [PATCH 16/19] Name legacy join primary-key results as bracketed aliases SQLSearch wrote each table's primary-key result name between double quotes after the legacy SQL text check had run, so a table alias holding a double quote could end the name early. Every other generated name is already bracketed, and the check refuses brackets in names. Also test that the statement is left as it was when writing it throws. Co-Authored-By: Claude Sonnet 5.5 Dispatch-Task: pr-maint-abc8bf322b5ccfd671a9379f3776cea4 --- dataLayer/SQLSearch.ts | 2 +- sqlTranslator/DESIGN.md | 2 ++ unitTests/sqlTranslator/legacySqlText.test.js | 12 ++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/dataLayer/SQLSearch.ts b/dataLayer/SQLSearch.ts index 7a88a7adc7..e0e2c260f7 100644 --- a/dataLayer/SQLSearch.ts +++ b/dataLayer/SQLSearch.ts @@ -992,7 +992,7 @@ class SQLSearch { table: table.as ? table.as_orig : table.tableid_orig, keys: new Set(), }); - select.push(`${table.as ? table.as : table.tableid}.\`${hash}\` AS "${tableKey}.${hash}"`); + select.push(`${table.as ? table.as : table.tableid}.\`${hash}\` AS [${tableKey}.${hash}]`); existingAttributes[table.as ? table.as_orig : table.tableid_orig] = this.data[`${table.databaseid_orig}_${table.as ? table.as_orig : table.tableid_orig}`].__mergedAttributes; diff --git a/sqlTranslator/DESIGN.md b/sqlTranslator/DESIGN.md index ebd3414774..7fe8207e24 100644 --- a/sqlTranslator/DESIGN.md +++ b/sqlTranslator/DESIGN.md @@ -26,4 +26,6 @@ Enforced by `unitTests/sqlTranslator/column-attribution.test.js`, `unitTests/sql Legacy SQL writes the authorized AST back out with `toString()` and AlaSQL parses that text again (`dataLayer/SQLSearch.ts` `_buildSQL()`, `dataLayer/update.ts`, `deleteTranslator.ts`, and the compiled SET and INSERT expressions), so the permission check holds only if the text parses back to the same statement. AlaSQL writes string and JSON literals unescaped, so `'x'', ssn, ''y'` came back as three columns, and numbers as `1e+25` or `Infinity`, which read back as columns. `installLegacySqlText()`, run by `alasqlFunctionImporter`, replaces the `StringValue`, `Json` and `NumValue` writers with ones that read back as the same value. `assertLegacySqlText()` refuses, with a 400 on the legacy path only, what the writers cannot spell: a name containing `` ` ``, `[`, `]` or `'` (brackets cannot hold `]`, and Harper leaves an alias starting with `[` unbackticked), an `@variable`, `$param` or `->`/`!` member name other than letters, digits and `_`, which AlaSQL writes unquoted, a string ending in an odd run of backslashes, which has no single-quoted spelling, a number that overflows, and, in a SELECT that reads a table, an unaliased function column whose text holds `]`, since `SQLSearch` aliases it as `[]`. It then re-parses the backticked statement text and refuses it unless it is one statement that reads each table-qualified column, function and variable no more often than the authorized statement does, so an AlaSQL writer not replaced here fails closed. INSERT takes literal values as data and writes only its other values as text, so only those are checked. v2 binds the AST directly. +Text `SQLSearch` builds after the check names columns only as `[]`: `_processJoins` writes each table's primary-key result as `[.]`, not `"."`, so an alias holding `"` cannot end the name. + Enforced by `unitTests/sqlTranslator/legacySqlText.test.js` and the quoted-string case in `integrationTests/security/sql-attribute-perms.test.ts`. diff --git a/unitTests/sqlTranslator/legacySqlText.test.js b/unitTests/sqlTranslator/legacySqlText.test.js index e9ccc2c106..50631650fa 100644 --- a/unitTests/sqlTranslator/legacySqlText.test.js +++ b/unitTests/sqlTranslator/legacySqlText.test.js @@ -174,6 +174,18 @@ describe('legacy SQL text', function () { assert.ok(!Object.hasOwn(insert.into, 'tableid_orig')); }); + it('leaves the statement as it was when writing it throws', function () { + const statement = parse('SELECT d.label FROM dev.t AS d WHERE d.`count` = 1 AND d.x = 1e999'); + const names = () => JSON.stringify([statement.columns[0], statement.from[0], statement.where]); + const before = names(); + assert.throws( + () => assertLegacySqlText(statement), + (err) => err.statusCode === 400 + ); + assert.strictEqual(names(), before); + assertLegacySqlText(parse('SELECT d.label FROM dev.t AS d WHERE d.`count` = 1')); + }); + it('refuses a string literal that has no spelling', function () { assertUnwritable("SELECT 'a\\' AS v FROM dev.t"); assertUnwritable("UPDATE dev.t SET label = {'k':'a\\'} WHERE id = 1"); From 1483e5a4fef4d865831eea3bf5bd6c4149a8a8de Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 2 Nov 2026 09:20:25 -0700 Subject: [PATCH 17/19] Keep a join's primary-key result name valid when the key holds a bracket Writing the name only in brackets broke legacy joins on a table whose primary key contains `]`. generatedName() writes brackets, double quotes when the name holds `]`, and refuses a name that holds both. Co-Authored-By: Claude Sonnet 5.5 Dispatch-Task: pr-maint-abc8bf322b5ccfd671a9379f3776cea4 --- dataLayer/SQLSearch.ts | 3 ++- sqlTranslator/DESIGN.md | 2 +- sqlTranslator/legacySqlText.ts | 7 ++++++ unitTests/sqlTranslator/legacySqlText.test.js | 22 ++++++++++++++++++- 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/dataLayer/SQLSearch.ts b/dataLayer/SQLSearch.ts index e0e2c260f7..c21adb59f1 100644 --- a/dataLayer/SQLSearch.ts +++ b/dataLayer/SQLSearch.ts @@ -12,6 +12,7 @@ import uniqBy from 'lodash/uniqBy.js'; import alasql from 'alasql'; alasql.options.cache = false; import alasqlFunctionImporter from '../sqlTranslator/alasqlFunctionImporter.ts'; +import { generatedName } from '../sqlTranslator/legacySqlText.ts'; import clone from 'clone'; import RecursiveIterator from 'recursive-iterator'; import log from '../utility/logging/harper_logger.ts'; @@ -992,7 +993,7 @@ class SQLSearch { table: table.as ? table.as_orig : table.tableid_orig, keys: new Set(), }); - select.push(`${table.as ? table.as : table.tableid}.\`${hash}\` AS [${tableKey}.${hash}]`); + select.push(`${table.as ? table.as : table.tableid}.\`${hash}\` AS ${generatedName(`${tableKey}.${hash}`)}`); existingAttributes[table.as ? table.as_orig : table.tableid_orig] = this.data[`${table.databaseid_orig}_${table.as ? table.as_orig : table.tableid_orig}`].__mergedAttributes; diff --git a/sqlTranslator/DESIGN.md b/sqlTranslator/DESIGN.md index 7fe8207e24..65f42c5fbb 100644 --- a/sqlTranslator/DESIGN.md +++ b/sqlTranslator/DESIGN.md @@ -26,6 +26,6 @@ Enforced by `unitTests/sqlTranslator/column-attribution.test.js`, `unitTests/sql Legacy SQL writes the authorized AST back out with `toString()` and AlaSQL parses that text again (`dataLayer/SQLSearch.ts` `_buildSQL()`, `dataLayer/update.ts`, `deleteTranslator.ts`, and the compiled SET and INSERT expressions), so the permission check holds only if the text parses back to the same statement. AlaSQL writes string and JSON literals unescaped, so `'x'', ssn, ''y'` came back as three columns, and numbers as `1e+25` or `Infinity`, which read back as columns. `installLegacySqlText()`, run by `alasqlFunctionImporter`, replaces the `StringValue`, `Json` and `NumValue` writers with ones that read back as the same value. `assertLegacySqlText()` refuses, with a 400 on the legacy path only, what the writers cannot spell: a name containing `` ` ``, `[`, `]` or `'` (brackets cannot hold `]`, and Harper leaves an alias starting with `[` unbackticked), an `@variable`, `$param` or `->`/`!` member name other than letters, digits and `_`, which AlaSQL writes unquoted, a string ending in an odd run of backslashes, which has no single-quoted spelling, a number that overflows, and, in a SELECT that reads a table, an unaliased function column whose text holds `]`, since `SQLSearch` aliases it as `[]`. It then re-parses the backticked statement text and refuses it unless it is one statement that reads each table-qualified column, function and variable no more often than the authorized statement does, so an AlaSQL writer not replaced here fails closed. INSERT takes literal values as data and writes only its other values as text, so only those are checked. v2 binds the AST directly. -Text `SQLSearch` builds after the check names columns only as `[]`: `_processJoins` writes each table's primary-key result as `[.]`, not `"."`, so an alias holding `"` cannot end the name. +Text `SQLSearch` builds after the check names columns only as `[]`. `_processJoins` writes each table's primary-key result through `generatedName()`: bracketed, double-quoted only when the key holds `]`, refused when it holds both, so an alias holding `"` cannot end the name. Enforced by `unitTests/sqlTranslator/legacySqlText.test.js` and the quoted-string case in `integrationTests/security/sql-attribute-perms.test.ts`. diff --git a/sqlTranslator/legacySqlText.ts b/sqlTranslator/legacySqlText.ts index 723dec6be4..56581bc4bb 100644 --- a/sqlTranslator/legacySqlText.ts +++ b/sqlTranslator/legacySqlText.ts @@ -75,6 +75,13 @@ function numberText(value: number): string { return `${sign}${digits.slice(0, point)}.${digits.slice(point)}`; } +/** A generated result name AlaSQL reads as one name: in brackets, or in double quotes when it holds `]`. */ +export function generatedName(name: string): string { + if (!name.includes(']')) return `[${name}]`; + if (!name.includes('"')) return `"${name}"`; + throw unwritable(`The name ${JSON.stringify(name)}`); +} + function unwritable(what: string) { return new ClientError( `${what} cannot be written back to SQL text for the legacy SQL engine. Remove the quotes, backticks, brackets or backslashes, or use the new SQL engine.` diff --git a/unitTests/sqlTranslator/legacySqlText.test.js b/unitTests/sqlTranslator/legacySqlText.test.js index 50631650fa..d10db86431 100644 --- a/unitTests/sqlTranslator/legacySqlText.test.js +++ b/unitTests/sqlTranslator/legacySqlText.test.js @@ -7,7 +7,12 @@ const assert = require('assert'); const alasql = require('alasql'); -const { assertLegacySqlText, installLegacySqlText, quoteString } = require('#src/sqlTranslator/legacySqlText'); +const { + assertLegacySqlText, + generatedName, + installLegacySqlText, + quoteString, +} = require('#src/sqlTranslator/legacySqlText'); const { backtickASTSchemaItems } = require('#src/utility/common_utils'); installLegacySqlText(); @@ -174,6 +179,21 @@ describe('legacy SQL text', function () { assert.ok(!Object.hasOwn(insert.into, 'tableid_orig')); }); + it('writes a generated result name as one name', function () { + assert.strictEqual(generatedName('r.id'), '[r.id]'); + assert.strictEqual(generatedName('x", r.id, "y.id'), '[x", r.id, "y.id]'); + assert.strictEqual(generatedName('r.key]part'), '"r.key]part"'); + assert.throws( + () => generatedName('x"y.key]part'), + (err) => err.statusCode === 400 + ); + for (const name of ['r.id', 'x", r.id, "y.id', 'r.key]part']) { + const columns = parse(`SELECT t.a AS ${generatedName(name)} FROM dev.t AS t`).columns; + assert.strictEqual(columns.length, 1, name); + assert.strictEqual(columns[0].as, name, name); + } + }); + it('leaves the statement as it was when writing it throws', function () { const statement = parse('SELECT d.label FROM dev.t AS d WHERE d.`count` = 1 AND d.x = 1e999'); const names = () => JSON.stringify([statement.columns[0], statement.from[0], statement.where]); From 9842f1f1fa23e375f9ec6a0de98fd30840caeed6 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 2 Nov 2026 09:57:27 -0700 Subject: [PATCH 18/19] Refuse SQL parameters as whole-row accessors for non-super users The `sql` operation binds no parameters, but legacy SQL runs each statement against its own rows, so `SELECT ?` returned fetched rows including a read-denied primary key, and `SET col = ?` evaluated to the existing record and echoed it in its error. getRowAccessors() now lists `?` and `$n`. Co-Authored-By: Claude Sonnet 5.5 Dispatch-Task: pr-maint-abc8bf322b5ccfd671a9379f3776cea4 --- integrationTests/security/sql-attribute-perms.test.ts | 2 ++ sqlTranslator/DESIGN.md | 2 +- sqlTranslator/sql_statement_bucket.ts | 4 +++- unitTests/sqlTranslator/column-attribution.test.js | 10 ++++++++++ utility/errors/commonErrors.ts | 2 +- 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/integrationTests/security/sql-attribute-perms.test.ts b/integrationTests/security/sql-attribute-perms.test.ts index 650053881b..1749056818 100644 --- a/integrationTests/security/sql-attribute-perms.test.ts +++ b/integrationTests/security/sql-attribute-perms.test.ts @@ -165,6 +165,8 @@ for (const sqlEngine of ['legacy', 'auto', 'new']) { `SELECT _->id AS k FROM data.ColPermRight r WHERE r.label = 'shared'`, "SELECT ``p`` AS rec FROM data.ColPermRight r WHERE r.label = 'shared'", "SELECT {__proto__:{rec:_}} AS obj FROM data.ColPermRight r WHERE r.label LIKE '%shared%'", + `SELECT ? AS v FROM data.ColPermRight r WHERE r.label = 'shared'`, + `UPDATE data.ColPermLeft SET label = ? WHERE id = 'l2'`, ]) { assertRefused(await asReader(sql), sql); } diff --git a/sqlTranslator/DESIGN.md b/sqlTranslator/DESIGN.md index 65f42c5fbb..3e9bb1dc07 100644 --- a/sqlTranslator/DESIGN.md +++ b/sqlTranslator/DESIGN.md @@ -18,7 +18,7 @@ So the bucket attributes columns with the binder's own rules, which live in `col - An unqualified `*` in a join expands to the FROM table's readable attributes qualified by its alias, so each expanded name binds to the table whose permissions chose it rather than to whichever table declares it. - Every walk uses `astNodes()`: iterative, with no depth limit, and visiting shared nodes once. `recursive-iterator` silently stopped at depth 100, so a column nested in 80 function calls went unchecked, and it threw on the subquery node AlaSQL shares between `where` and `exists`. -Row accessors that name no column, AlaSQL's `_`, JavaScript expressions and a JSON literal that sets `__proto__` (every walk here reads own properties, but AlaSQL evaluates the prototype), are refused for non-super users by `getRowAccessors()`. Legacy SQL's row container always carries the primary key (`dataLayer/SQLSearch.ts` keys rows by it), so a whole-row read would expose it whatever the role allows. The JavaScript refusal stays unconditional for non-super users, even if the `_` refusal is ever relaxed for roles without attribute permissions. +Row accessors that name no column, AlaSQL's `_`, JavaScript expressions, `?`/`$n` parameters (the `sql` operation binds none, and legacy SQL fills them with its own rows) and a JSON literal that sets `__proto__` (every walk here reads own properties, but AlaSQL evaluates the prototype), are refused for non-super users by `getRowAccessors()`. Legacy SQL's row container always carries the primary key (`dataLayer/SQLSearch.ts` keys rows by it), so a whole-row read would expose it whatever the role allows. The JavaScript refusal stays unconditional for non-super users, even if the `_` refusal is ever relaxed for roles without attribute permissions. Enforced by `unitTests/sqlTranslator/column-attribution.test.js`, `unitTests/sqlEngine/columnScope.test.js` and `integrationTests/security/sql-attribute-perms.test.ts`. diff --git a/sqlTranslator/sql_statement_bucket.ts b/sqlTranslator/sql_statement_bucket.ts index 8c28f6b30c..69dbe68e4c 100644 --- a/sqlTranslator/sql_statement_bucket.ts +++ b/sqlTranslator/sql_statement_bucket.ts @@ -139,7 +139,7 @@ class sqlStatementBucket { /** * Expressions that read a whole row instead of naming a column: AlaSQL's `_` (`SELECT VALUE`, - * `ROW`, `x._`, `_->attr`) and JavaScript expressions. Attribute permissions cannot apply to them, + * `ROW`, `x._`, `_->attr`), JavaScript expressions and `?`/`$n` parameters. Attribute permissions cannot apply to them, * and legacy SQL's row container always carries the primary key. */ getRowAccessors(): string[] { @@ -147,6 +147,8 @@ class sqlStatementBucket { for (const node of astNodes(this.ast)) { if (node instanceof (alasql as any).yy.Column && node.columnid === '_') accessors.push('_'); else if (node instanceof (alasql as any).yy.JavaScript) accessors.push('JavaScript expression'); + // The `sql` operation binds no parameters, but legacy SQL runs every statement against its own rows. + else if (node instanceof (alasql as any).yy.ParamValue) accessors.push('parameter'); // A JSON literal's `__proto__` key hides an expression in the prototype, where every walk here // (own properties only) misses it but AlaSQL's JSON compiler still evaluates it. else if (!isKnownPrototype(Object.getPrototypeOf(node))) accessors.push('JSON literal with __proto__'); diff --git a/unitTests/sqlTranslator/column-attribution.test.js b/unitTests/sqlTranslator/column-attribution.test.js index 05529717d8..afee17c9c7 100644 --- a/unitTests/sqlTranslator/column-attribution.test.js +++ b/unitTests/sqlTranslator/column-attribution.test.js @@ -257,6 +257,16 @@ describe('SQL column attribution for attribute permissions', () => { ]); }); + it('lists `?` and `$n` parameters, which legacy SQL fills with its own rows', () => { + for (const sql of [ + "SELECT ? AS v FROM data.L WHERE label = 'x'", + "SELECT $1 AS v FROM data.L WHERE label = 'x'", + 'UPDATE data.L SET label = ? WHERE id = 1', + ]) { + assert.deepStrictEqual(bucketFor(sql).getRowAccessors(), ['parameter'], sql); + } + }); + it('refuses a JSON literal that hides an expression behind __proto__', () => { for (const sql of [ "SELECT {__proto__:{rec:_}} AS obj FROM data.L WHERE label = 'x'", diff --git a/utility/errors/commonErrors.ts b/utility/errors/commonErrors.ts index 8ed14337e0..85c67cc337 100644 --- a/utility/errors/commonErrors.ts +++ b/utility/errors/commonErrors.ts @@ -146,7 +146,7 @@ const OPERATION_AUTH_ERROR_MSGS = { UNRESOLVED_SQL_COLUMN: (columns: string[]) => `This operation is not authorized: the statement qualifies ${columns.join(', ')} with a table name or alias that does not identify exactly one table in the statement.`, SQL_ROW_ACCESSOR: - 'This operation is not authorized: the statement reads whole rows through `_`, `SELECT VALUE`/`ROW`, a JavaScript expression or a JSON literal that sets `__proto__`, which attribute permissions cannot be applied to. Name the attributes to read instead.', + 'This operation is not authorized: the statement reads whole rows through `_`, `SELECT VALUE`/`ROW`, a JavaScript expression, a `?` or `$n` parameter or a JSON literal that sets `__proto__`, which attribute permissions cannot be applied to. Name the attributes to read instead.', USER_HAS_NO_PERMS: (user) => `User ${user} has no role or permissions. Please assign the user a valid role.`, DROP_SYSTEM: "The 'system' database, tables and records are used internally by Harper and cannot be updated or removed.", From bab75e35b587d4dfc1078fdc71a71de8a79d5dbe Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 2 Nov 2026 10:42:30 -0700 Subject: [PATCH 19/19] Shorten three root index descriptions so DESIGN.md meets its byte ceiling Main's root DESIGN.md is already 153 bytes over its ceiling, and the design docs check holds any change that touches it to the ceiling. The notes' bodies are unchanged; only their index descriptions are shorter. Co-Authored-By: Claude Sonnet 5.5 Dispatch-Task: pr-maint-abc8bf322b5ccfd671a9379f3776cea4 --- DESIGN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESIGN.md b/DESIGN.md index 7bd17406fe..b412f98cd4 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -158,7 +158,7 @@ A note lives in the `DESIGN.md` of the directory that owns the code it describes - [A restart covers exactly the workers that loaded a component or will load it](components/DESIGN.md#a-restart-covers-exactly-the-workers-that-loaded-a-component-or-will-load-it) — A new or still-isolated application restarts only its dedicated worker; main adds the pool when a pool worker reported loading it, and a named restart reconciles only that application · [`drop_component`: main owns the whole-component drop](components/DESIGN.md#drop_component-main-owns-the-whole-component-drop) — Main drops under the lock (workers hand over and wait); an absent component is a no-op; an isolated worker stops without `restart`; branches go last unless `keepBranches`; each node reports its own outcome. - [Component preparation is serialized across worker threads](components/DESIGN.md#component-preparation-is-serialized-across-worker-threads) — One component transaction at a time across workers; the lifecycle broadcast sits outside the lock; the load lock is keyed by application and plugin. - [Peer-side deploy_component payload read: retryable blob stalls and `Readable.from()` cancellation](components/DESIGN.md#peer-side-deploy_component-payload-read-retryable-blob-stalls-and-readablefrom-cancellation) — Retry a `BLOB_UNAVAILABLE` read only before any byte reached the consumer; use `Readable.from()` for cancellation, not a hand-rolled `ReadableStream`. · [An origin waits for a peer's deploy answer only as long as the peer may take](components/DESIGN.md#an-origin-waits-for-a-peers-deploy-answer-only-as-long-as-the-peer-may-take) — The replicated deploy's per-peer deadline sums the peer's own allowances, so it never undercuts a healthy peer; a missed deadline is a failure with an unknown outcome, not cancellation. -- [A replicated deploy reports each node's install fingerprint, and never acts on it](components/DESIGN.md#a-replicated-deploy-reports-each-nodes-install-fingerprint-and-never-acts-on-it) — Each node records the source identity its resolver already produced and the sha256 of its root lockfiles; the origin compares peers once on the aggregate and reports differences in a warning, the message and `peer_results`, without ever failing the deploy. · [The deployment record (`system.hdb_deployment`)](components/DESIGN.md#the-deployment-record-systemhdb_deployment) — One row per deploy on the receiving node; a live tail ends on a promise settled after the terminal write; a rolling deploy stays `rolling_out` until its activation job records every peer's outcome; the source is OIDC-attested or client-declared; failed deploys' payloads expire by count and age. · [A rolling restart job completes only if every peer restarted (`bin/restart.ts` `restartEachPeer`)](components/DESIGN.md#a-rolling-restart-job-completes-only-if-every-peer-restarted-binrestartts-restarteachpeer) — `ERROR` names each peer that failed or was skipped, with every peer's outcome in `result.replicated`; a failed or possibly-running peer restart halts the roll; only write back-pressure is retried. +- [A replicated deploy reports each node's install fingerprint, and never acts on it](components/DESIGN.md#a-replicated-deploy-reports-each-nodes-install-fingerprint-and-never-acts-on-it) — Peers' install fingerprints are compared once and reported, never acted on. · [The deployment record (`system.hdb_deployment`)](components/DESIGN.md#the-deployment-record-systemhdb_deployment) — One row per deploy on the receiving node; a rolling deploy stays `rolling_out` until every peer's outcome is recorded. · [A rolling restart job completes only if every peer restarted (`bin/restart.ts` `restartEachPeer`)](components/DESIGN.md#a-rolling-restart-job-completes-only-if-every-peer-restarted-binrestartts-restarteachpeer) — `ERROR` names each peer that failed or was skipped, with every outcome in `result.replicated`; a failed peer halts the roll. - [A dangling symlink silently truncates the deploy tarball (`components/packageComponent.ts`)](components/DESIGN.md#a-dangling-symlink-silently-truncates-the-deploy-tarball-componentspackagecomponentts) — tar-fs treats a dangling symlink's ENOENT as end-of-stream, so `scanPackageDirectory()` pre-walks the tree; a deploy also warns when a declared `jsResource`/`graphqlSchema` entry point matched nothing · [The packer never ships the loader's links to the install (`components/packageComponent.ts`)](components/DESIGN.md#the-packer-never-ships-the-loaders-links-to-the-install-componentspackagecomponentts) — The application's `node_modules/harper` is skipped by name; a nested one, or a `harperdb` link, only when it resolves to the running install, and `harperdb` ships as an empty directory the loader relinks. - [Deploy watcher generations preserve logical entry events](components/DESIGN.md#deploy-watcher-generations-preserve-logical-entry-events) — `EntryHandler` snapshots matching paths across a deploy so consumers see logical change and delete events, not raw rescan adds. · [Restart-free deploys require proof of runtime equivalence](components/DESIGN.md#restart-free-deploys-require-proof-of-runtime-equivalence) — A deploy stays restart-free only when declared files, the imported runtime and installed dependencies are all proven equivalent. - [Secret custody starts before boot-time installs](components/DESIGN.md#secret-custody-starts-before-boot-time-installs) — `startSecretCustodyOnMainThread()` runs the custody built-in ahead of `installApplications()`; the root load reuses it.