diff --git a/src/languages/definitions/msdax/msdax.test.ts b/src/languages/definitions/msdax/msdax.test.ts index 7dcac2be99..408cb2a11d 100644 --- a/src/languages/definitions/msdax/msdax.test.ts +++ b/src/languages/definitions/msdax/msdax.test.ts @@ -247,7 +247,7 @@ testTokenization('msdax', [ tokens: [ { startIndex: 0, type: 'keyword.msdax' }, { startIndex: 8, type: 'white.msdax' }, - { startIndex: 9, type: 'keyword.msdax' } + { startIndex: 9, type: 'keyword.function.msdax' } ] } ], @@ -293,7 +293,7 @@ testTokenization('msdax', [ [ { line: 'int', - tokens: [{ startIndex: 0, type: 'keyword.msdax' }] + tokens: [{ startIndex: 0, type: 'keyword.function.msdax' }] } ], @@ -394,6 +394,57 @@ testTokenization('msdax', [ } ], + // Logical operators && and || + [ + { + // [Column] references tokenize as identifier.quote, () merges when adjacent + line: 'IF(Logic[Condition] && "a" = "b", TRUE(), FALSE())', + tokens: [ + { startIndex: 0, type: 'keyword.function.msdax' }, + { startIndex: 2, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 3, type: 'identifier.msdax' }, + { startIndex: 8, type: 'identifier.quote.msdax' }, + { startIndex: 9, type: 'identifier.msdax' }, + { startIndex: 18, type: 'identifier.quote.msdax' }, + { startIndex: 19, type: 'white.msdax' }, + { startIndex: 20, type: 'operator.msdax' }, + { startIndex: 22, type: 'white.msdax' }, + { startIndex: 23, type: 'string.msdax' }, + { startIndex: 26, type: 'white.msdax' }, + { startIndex: 27, type: 'operator.msdax' }, + { startIndex: 28, type: 'white.msdax' }, + { startIndex: 29, type: 'string.msdax' }, + { startIndex: 32, type: 'delimiter.msdax' }, + { startIndex: 33, type: 'white.msdax' }, + { startIndex: 34, type: 'keyword.function.msdax' }, + { startIndex: 38, type: 'delimiter.parenthesis.msdax' }, // () merged + { startIndex: 40, type: 'delimiter.msdax' }, + { startIndex: 41, type: 'white.msdax' }, + { startIndex: 42, type: 'keyword.function.msdax' }, + { startIndex: 47, type: 'delimiter.parenthesis.msdax' } // ()) all merged + ] + } + ], + + [ + { + line: 'Sales[Amount] || Products[Qty]', + tokens: [ + { startIndex: 0, type: 'identifier.msdax' }, + { startIndex: 5, type: 'identifier.quote.msdax' }, + { startIndex: 6, type: 'identifier.msdax' }, + { startIndex: 12, type: 'identifier.quote.msdax' }, + { startIndex: 13, type: 'white.msdax' }, + { startIndex: 14, type: 'operator.msdax' }, + { startIndex: 16, type: 'white.msdax' }, + { startIndex: 17, type: 'identifier.msdax' }, + { startIndex: 25, type: 'identifier.quote.msdax' }, + { startIndex: 26, type: 'identifier.msdax' }, + { startIndex: 29, type: 'identifier.quote.msdax' } + ] + } + ], + // Realistic queries and expressions [ { @@ -420,9 +471,10 @@ testTokenization('msdax', [ [ { + // DATATABLE is a function (datatable-function-dax), emitted as keyword.function. line: 'DATATABLE("Price", STRING, {{"Low"},{"Medium"}})', tokens: [ - { startIndex: 0, type: 'keyword.msdax' }, + { startIndex: 0, type: 'keyword.function.msdax' }, { startIndex: 9, type: 'delimiter.parenthesis.msdax' }, { startIndex: 10, type: 'string.msdax' }, { startIndex: 17, type: 'delimiter.msdax' }, @@ -440,5 +492,482 @@ testTokenization('msdax', [ { startIndex: 47, type: 'delimiter.parenthesis.msdax' } ] } + ], + + // New DAX functions + [ + { + // REMOVEFILTERS = 13 chars (0-12), ( at 13 + // Note: Product matches PRODUCT function (ignoreCase:true), Color is identifier + line: 'REMOVEFILTERS(Product[Color])', + tokens: [ + { startIndex: 0, type: 'keyword.function.msdax' }, + { startIndex: 13, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 14, type: 'keyword.function.msdax' }, + { startIndex: 21, type: 'identifier.quote.msdax' }, + { startIndex: 22, type: 'identifier.msdax' }, + { startIndex: 27, type: 'identifier.quote.msdax' }, + { startIndex: 28, type: 'delimiter.parenthesis.msdax' } + ] + } + ], + + [ + { + // SELECTEDVALUE = 13 chars, Category = 8 chars + // Product matches PRODUCT function (ignoreCase:true) + line: 'SELECTEDVALUE(Product[Category], "All")', + tokens: [ + { startIndex: 0, type: 'keyword.function.msdax' }, + { startIndex: 13, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 14, type: 'keyword.function.msdax' }, + { startIndex: 21, type: 'identifier.quote.msdax' }, + { startIndex: 22, type: 'identifier.msdax' }, + { startIndex: 30, type: 'identifier.quote.msdax' }, + { startIndex: 31, type: 'delimiter.msdax' }, + { startIndex: 32, type: 'white.msdax' }, + { startIndex: 33, type: 'string.msdax' }, + { startIndex: 38, type: 'delimiter.parenthesis.msdax' } + ] + } + ], + + [ + { + // ALLCROSSFILTERED = 16 chars + line: 'ALLCROSSFILTERED(Orders)', + tokens: [ + { startIndex: 0, type: 'keyword.function.msdax' }, + { startIndex: 16, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 17, type: 'identifier.msdax' }, + { startIndex: 23, type: 'delimiter.parenthesis.msdax' } + ] + } + ], + + [ + { + // KEEPFILTERS = 11 chars, Products = 8 chars + line: 'KEEPFILTERS(Products[Color] = "Red")', + tokens: [ + { startIndex: 0, type: 'keyword.function.msdax' }, + { startIndex: 11, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 12, type: 'identifier.msdax' }, + { startIndex: 20, type: 'identifier.quote.msdax' }, + { startIndex: 21, type: 'identifier.msdax' }, + { startIndex: 26, type: 'identifier.quote.msdax' }, + { startIndex: 27, type: 'white.msdax' }, + { startIndex: 28, type: 'operator.msdax' }, + { startIndex: 29, type: 'white.msdax' }, + { startIndex: 30, type: 'string.msdax' }, + { startIndex: 35, type: 'delimiter.parenthesis.msdax' } + ] + } + ], + + // DEFINE with VAR/RETURN + [ + { + // DEFINE(6) MEASURE(7-13) Sales(15-19) [Total](20-26) = VAR x = SUM(Sales[Amount]) RETURN x + 1 + line: 'DEFINE MEASURE Sales[Total] = VAR x = SUM(Sales[Amount]) RETURN x + 1', + tokens: [ + { startIndex: 0, type: 'keyword.msdax' }, + { startIndex: 6, type: 'white.msdax' }, + { startIndex: 7, type: 'keyword.msdax' }, + { startIndex: 14, type: 'white.msdax' }, + { startIndex: 15, type: 'identifier.msdax' }, + { startIndex: 20, type: 'identifier.quote.msdax' }, + { startIndex: 21, type: 'identifier.msdax' }, + { startIndex: 26, type: 'identifier.quote.msdax' }, + { startIndex: 27, type: 'white.msdax' }, + { startIndex: 28, type: 'operator.msdax' }, + { startIndex: 29, type: 'white.msdax' }, + { startIndex: 30, type: 'keyword.msdax' }, + { startIndex: 33, type: 'white.msdax' }, + { startIndex: 34, type: 'identifier.msdax' }, + { startIndex: 35, type: 'white.msdax' }, + { startIndex: 36, type: 'operator.msdax' }, + { startIndex: 37, type: 'white.msdax' }, + { startIndex: 38, type: 'keyword.function.msdax' }, + { startIndex: 41, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 42, type: 'identifier.msdax' }, + { startIndex: 47, type: 'identifier.quote.msdax' }, + { startIndex: 48, type: 'identifier.msdax' }, + { startIndex: 54, type: 'identifier.quote.msdax' }, + { startIndex: 55, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 56, type: 'white.msdax' }, + { startIndex: 57, type: 'keyword.msdax' }, + { startIndex: 63, type: 'white.msdax' }, + { startIndex: 64, type: 'identifier.msdax' }, + { startIndex: 65, type: 'white.msdax' }, + { startIndex: 66, type: 'operator.msdax' }, + { startIndex: 67, type: 'white.msdax' }, + { startIndex: 68, type: 'number.msdax' } + ] + } + ], + + // COLUMN, TABLE and EXPRESSION keywords + [ + { + line: 'COLUMN', + tokens: [{ startIndex: 0, type: 'keyword.msdax' }] + } + ], + + [ + { + line: 'TABLE', + tokens: [{ startIndex: 0, type: 'keyword.msdax' }] + } + ], + + [ + { + line: 'EXPRESSION', + tokens: [{ startIndex: 0, type: 'keyword.msdax' }] + } + ], + + // Window functions — WINDOW(6) 1(7) ,(8) ABS(10-12) ,(13) 0(15) ,(16) PARTITIONBY(18-28) (Product[Category])(29-47) ORDERBY(50-56) (Date[Day], ASC)(57-72) + [ + { + line: 'WINDOW(1, ABS, 0, PARTITIONBY(Product[Category]), ORDERBY(Date[Day], ASC))', + tokens: [ + { startIndex: 0, type: 'keyword.function.msdax' }, + { startIndex: 6, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 7, type: 'number.msdax' }, + { startIndex: 8, type: 'delimiter.msdax' }, + { startIndex: 9, type: 'white.msdax' }, + { startIndex: 10, type: 'keyword.function.msdax' }, + { startIndex: 13, type: 'delimiter.msdax' }, + { startIndex: 14, type: 'white.msdax' }, + { startIndex: 15, type: 'number.msdax' }, + { startIndex: 16, type: 'delimiter.msdax' }, + { startIndex: 17, type: 'white.msdax' }, + { startIndex: 18, type: 'keyword.function.msdax' }, + { startIndex: 29, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 30, type: 'keyword.function.msdax' }, // Product matches PRODUCT function + { startIndex: 37, type: 'identifier.quote.msdax' }, + { startIndex: 38, type: 'identifier.msdax' }, + { startIndex: 46, type: 'identifier.quote.msdax' }, + { startIndex: 47, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 48, type: 'delimiter.msdax' }, + { startIndex: 49, type: 'white.msdax' }, + { startIndex: 50, type: 'keyword.function.msdax' }, + { startIndex: 57, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 58, type: 'keyword.function.msdax' }, // Date matches DATE function + { startIndex: 62, type: 'identifier.quote.msdax' }, + { startIndex: 63, type: 'identifier.msdax' }, + { startIndex: 66, type: 'identifier.quote.msdax' }, + { startIndex: 67, type: 'delimiter.msdax' }, + { startIndex: 68, type: 'white.msdax' }, + { startIndex: 69, type: 'keyword.msdax' }, + { startIndex: 72, type: 'delimiter.parenthesis.msdax' } // )) merged + ] + } + ], + + // Multi-char operators (dax-operator-reference): == <> <= >= != && || + [ + { + // [a] == 1 + line: '[a] == 1', + tokens: [ + { startIndex: 0, type: 'identifier.quote.msdax' }, + { startIndex: 1, type: 'identifier.msdax' }, + { startIndex: 2, type: 'identifier.quote.msdax' }, + { startIndex: 3, type: 'white.msdax' }, + { startIndex: 4, type: 'operator.msdax' }, // == + { startIndex: 6, type: 'white.msdax' }, + { startIndex: 7, type: 'number.msdax' } + ] + } + ], + + [ + { + line: '1 <> 2', + tokens: [ + { startIndex: 0, type: 'number.msdax' }, + { startIndex: 1, type: 'white.msdax' }, + { startIndex: 2, type: 'operator.msdax' }, // <> + { startIndex: 4, type: 'white.msdax' }, + { startIndex: 5, type: 'number.msdax' } + ] + } + ], + + [ + { + line: '1 <= 2', + tokens: [ + { startIndex: 0, type: 'number.msdax' }, + { startIndex: 1, type: 'white.msdax' }, + { startIndex: 2, type: 'operator.msdax' }, // <= + { startIndex: 4, type: 'white.msdax' }, + { startIndex: 5, type: 'number.msdax' } + ] + } + ], + + [ + { + line: '1 >= 2', + tokens: [ + { startIndex: 0, type: 'number.msdax' }, + { startIndex: 1, type: 'white.msdax' }, + { startIndex: 2, type: 'operator.msdax' }, // >= + { startIndex: 4, type: 'white.msdax' }, + { startIndex: 5, type: 'number.msdax' } + ] + } + ], + + // DAX datetime literal dt"..." (dax-syntax-reference) + [ + { + line: 'dt"2015-12-31"', + tokens: [{ startIndex: 0, type: 'string.msdax' }] + } + ], + + [ + { + line: 'FILTER(Sales, [OrderDate] > dt"2015-1-9T02:30:00")', + tokens: [ + { startIndex: 0, type: 'keyword.function.msdax' }, // FILTER + { startIndex: 6, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 7, type: 'identifier.msdax' }, // Sales + { startIndex: 12, type: 'delimiter.msdax' }, + { startIndex: 13, type: 'white.msdax' }, + { startIndex: 14, type: 'identifier.quote.msdax' }, + { startIndex: 15, type: 'identifier.msdax' }, + { startIndex: 24, type: 'identifier.quote.msdax' }, + { startIndex: 25, type: 'white.msdax' }, + { startIndex: 26, type: 'operator.msdax' }, // > + { startIndex: 27, type: 'white.msdax' }, + { startIndex: 28, type: 'string.msdax' }, // dt"..." + { startIndex: 49, type: 'delimiter.parenthesis.msdax' } + ] + } + ], + + // Dotted function names tokenize as a single keyword.function + [ + { + line: 'PERCENTILE.EXC(Sales[Amount], 0.9)', + tokens: [ + { startIndex: 0, type: 'keyword.function.msdax' }, // PERCENTILE.EXC + { startIndex: 14, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 15, type: 'identifier.msdax' }, // Sales + { startIndex: 20, type: 'identifier.quote.msdax' }, + { startIndex: 21, type: 'identifier.msdax' }, + { startIndex: 27, type: 'identifier.quote.msdax' }, + { startIndex: 28, type: 'delimiter.msdax' }, + { startIndex: 29, type: 'white.msdax' }, + { startIndex: 30, type: 'number.msdax' }, // 0.9 + { startIndex: 33, type: 'delimiter.parenthesis.msdax' } + ] + } + ], + + [ + { + line: 'STDEV.P(Sales[Amount])', + tokens: [ + { startIndex: 0, type: 'keyword.function.msdax' }, // STDEV.P + { startIndex: 7, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 8, type: 'identifier.msdax' }, + { startIndex: 13, type: 'identifier.quote.msdax' }, + { startIndex: 14, type: 'identifier.msdax' }, + { startIndex: 20, type: 'identifier.quote.msdax' }, + { startIndex: 21, type: 'delimiter.parenthesis.msdax' } + ] + } + ], + + [ + { + // Three-segment dotted name (INFO.VIEW.MEASURES). `()` merges in output. + line: 'EVALUATE INFO.VIEW.MEASURES()', + tokens: [ + { startIndex: 0, type: 'keyword.msdax' }, + { startIndex: 8, type: 'white.msdax' }, + { startIndex: 9, type: 'keyword.function.msdax' }, // INFO.VIEW.MEASURES + { startIndex: 27, type: 'delimiter.parenthesis.msdax' } + ] + } + ], + + // Newly covered DAX functions (spot-check per category) + [ + { + line: 'COALESCE([a], 0)', + tokens: [ + { startIndex: 0, type: 'keyword.function.msdax' }, // COALESCE + { startIndex: 8, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 9, type: 'identifier.quote.msdax' }, + { startIndex: 10, type: 'identifier.msdax' }, + { startIndex: 11, type: 'identifier.quote.msdax' }, + { startIndex: 12, type: 'delimiter.msdax' }, + { startIndex: 13, type: 'white.msdax' }, + { startIndex: 14, type: 'number.msdax' }, + { startIndex: 15, type: 'delimiter.parenthesis.msdax' } + ] + } + ], + + [ + { + // BITAND from Logical (logical-functions-dax) + line: 'BITAND(13, 11)', + tokens: [ + { startIndex: 0, type: 'keyword.function.msdax' }, + { startIndex: 6, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 7, type: 'number.msdax' }, + { startIndex: 9, type: 'delimiter.msdax' }, + { startIndex: 10, type: 'white.msdax' }, + { startIndex: 11, type: 'number.msdax' }, + { startIndex: 13, type: 'delimiter.parenthesis.msdax' } + ] + } + ], + + [ + { + // COMBINEVALUES from Text + line: 'COMBINEVALUES(",", [a], [b])', + tokens: [ + { startIndex: 0, type: 'keyword.function.msdax' }, + { startIndex: 13, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 14, type: 'string.msdax' }, + { startIndex: 17, type: 'delimiter.msdax' }, + { startIndex: 18, type: 'white.msdax' }, + { startIndex: 19, type: 'identifier.quote.msdax' }, + { startIndex: 20, type: 'identifier.msdax' }, + { startIndex: 21, type: 'identifier.quote.msdax' }, + { startIndex: 22, type: 'delimiter.msdax' }, + { startIndex: 23, type: 'white.msdax' }, + { startIndex: 24, type: 'identifier.quote.msdax' }, + { startIndex: 25, type: 'identifier.msdax' }, + { startIndex: 26, type: 'identifier.quote.msdax' }, + { startIndex: 27, type: 'delimiter.parenthesis.msdax' } + ] + } + ], + + [ + { + // SELECTEDMEASURE (Information / calculation groups). `()` merges. + line: 'SELECTEDMEASURE()', + tokens: [ + { startIndex: 0, type: 'keyword.function.msdax' }, + { startIndex: 15, type: 'delimiter.parenthesis.msdax' } + ] + } + ], + + [ + { + // UTCNOW / UTCTODAY / QUARTER from Date & time. Trailing parens merge. + line: 'QUARTER(UTCTODAY())', + tokens: [ + { startIndex: 0, type: 'keyword.function.msdax' }, + { startIndex: 7, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 8, type: 'keyword.function.msdax' }, + { startIndex: 16, type: 'delimiter.parenthesis.msdax' } + ] + } + ], + + [ + { + // TOCSV and TOJSON from Other + line: 'TOCSV(Sales, 10)', + tokens: [ + { startIndex: 0, type: 'keyword.function.msdax' }, + { startIndex: 5, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 6, type: 'identifier.msdax' }, + { startIndex: 11, type: 'delimiter.msdax' }, + { startIndex: 12, type: 'white.msdax' }, + { startIndex: 13, type: 'number.msdax' }, + { startIndex: 15, type: 'delimiter.parenthesis.msdax' } + ] + } + ], + + [ + { + // DATESWTD (week-to-date; previously missing alongside DATESMTD/QTD/YTD) + line: 'DATESWTD(Date[Date])', + tokens: [ + { startIndex: 0, type: 'keyword.function.msdax' }, + { startIndex: 8, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 9, type: 'keyword.function.msdax' }, // Date matches DATE + { startIndex: 13, type: 'identifier.quote.msdax' }, + { startIndex: 14, type: 'identifier.msdax' }, + { startIndex: 18, type: 'identifier.quote.msdax' }, + { startIndex: 19, type: 'delimiter.parenthesis.msdax' } + ] + } + ], + + // DEFINE FUNCTION user-defined function (function-statement-dax). + // `=>` is a single operator token (dax-operator-reference). + [ + { + line: 'DEFINE FUNCTION MyAdd = (x, y) => x + y', + tokens: [ + { startIndex: 0, type: 'keyword.msdax' }, // DEFINE + { startIndex: 6, type: 'white.msdax' }, + { startIndex: 7, type: 'keyword.msdax' }, // FUNCTION + { startIndex: 15, type: 'white.msdax' }, + { startIndex: 16, type: 'identifier.msdax' }, // MyAdd + { startIndex: 21, type: 'white.msdax' }, + { startIndex: 22, type: 'operator.msdax' }, // = + { startIndex: 23, type: 'white.msdax' }, + { startIndex: 24, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 25, type: 'identifier.msdax' }, + { startIndex: 26, type: 'delimiter.msdax' }, + { startIndex: 27, type: 'white.msdax' }, + { startIndex: 28, type: 'identifier.msdax' }, + { startIndex: 29, type: 'delimiter.parenthesis.msdax' }, + { startIndex: 30, type: 'white.msdax' }, + { startIndex: 31, type: 'operator.msdax' }, // => + { startIndex: 33, type: 'white.msdax' }, + { startIndex: 34, type: 'identifier.msdax' }, + { startIndex: 35, type: 'white.msdax' }, + { startIndex: 36, type: 'operator.msdax' }, // + + { startIndex: 37, type: 'white.msdax' }, + { startIndex: 38, type: 'identifier.msdax' } + ] + } + ], + + // DEFINE TABLE ... WITH VISUAL SHAPE (virtual-table-statement-dax) + [ + { + line: 'DEFINE TABLE T = {1} WITH VISUAL SHAPE', + tokens: [ + { startIndex: 0, type: 'keyword.msdax' }, // DEFINE + { startIndex: 6, type: 'white.msdax' }, + { startIndex: 7, type: 'keyword.msdax' }, // TABLE + { startIndex: 12, type: 'white.msdax' }, + { startIndex: 13, type: 'identifier.msdax' }, // T + { startIndex: 14, type: 'white.msdax' }, + { startIndex: 15, type: 'operator.msdax' }, + { startIndex: 16, type: 'white.msdax' }, + { startIndex: 17, type: 'delimiter.brackets.msdax' }, + { startIndex: 18, type: 'number.msdax' }, + { startIndex: 19, type: 'delimiter.brackets.msdax' }, + { startIndex: 20, type: 'white.msdax' }, + { startIndex: 21, type: 'keyword.msdax' }, // WITH + { startIndex: 25, type: 'white.msdax' }, + { startIndex: 26, type: 'keyword.msdax' }, // VISUAL + { startIndex: 32, type: 'white.msdax' }, + { startIndex: 33, type: 'keyword.msdax' } // SHAPE + ] + } ] ]); diff --git a/src/languages/definitions/msdax/msdax.ts b/src/languages/definitions/msdax/msdax.ts index 0edef52136..b518650ea6 100644 --- a/src/languages/definitions/msdax/msdax.ts +++ b/src/languages/definitions/msdax/msdax.ts @@ -36,21 +36,38 @@ export const language = { ], keywords: [ - // Query keywords + // Query keywords (dax-queries, statements-dax) 'VAR', 'RETURN', 'NOT', 'EVALUATE', - 'DATATABLE', 'ORDER', 'BY', 'START', 'AT', 'DEFINE', 'MEASURE', + 'COLUMN', + 'TABLE', + 'FUNCTION', 'ASC', 'DESC', 'IN', + 'EXPRESSION', + // DEFINE TABLE ... WITH VISUAL SHAPE (...) (virtual-table-statement-dax) + 'WITH', + 'VISUAL', + 'SHAPE', + 'AXIS', + 'GROUP', + 'TOTAL', + 'DENSIFY', + // User-defined function parameter modes/types (function-statement-dax) + 'ANYVAL', + 'ANYREF', + 'SCALAR', + 'VAL', + 'EXPR', // Datatable types 'BOOLEAN', 'DOUBLE', @@ -59,20 +76,36 @@ export const language = { 'CURRENCY', 'STRING' ], + functions: [ - // Relational + // Time intelligence — week/month/quarter/year balances and boundaries 'CLOSINGBALANCEMONTH', 'CLOSINGBALANCEQUARTER', 'CLOSINGBALANCEYEAR', + 'CLOSINGBALANCEWEEK', + 'OPENINGBALANCEMONTH', + 'OPENINGBALANCEQUARTER', + 'OPENINGBALANCEYEAR', + 'OPENINGBALANCEWEEK', + 'ENDOFMONTH', + 'ENDOFQUARTER', + 'ENDOFYEAR', + 'ENDOFWEEK', + 'STARTOFMONTH', + 'STARTOFQUARTER', + 'STARTOFYEAR', + 'STARTOFWEEK', + 'TOTALMTD', + 'TOTALQTD', + 'TOTALYTD', + 'TOTALWTD', 'DATEADD', 'DATESBETWEEN', 'DATESINPERIOD', 'DATESMTD', 'DATESQTD', 'DATESYTD', - 'ENDOFMONTH', - 'ENDOFQUARTER', - 'ENDOFYEAR', + 'DATESWTD', 'FIRSTDATE', 'FIRSTNONBLANK', 'LASTDATE', @@ -80,25 +113,20 @@ export const language = { 'NEXTDAY', 'NEXTMONTH', 'NEXTQUARTER', + 'NEXTWEEK', 'NEXTYEAR', - 'OPENINGBALANCEMONTH', - 'OPENINGBALANCEQUARTER', - 'OPENINGBALANCEYEAR', 'PARALLELPERIOD', 'PREVIOUSDAY', 'PREVIOUSMONTH', 'PREVIOUSQUARTER', + 'PREVIOUSWEEK', 'PREVIOUSYEAR', 'SAMEPERIODLASTYEAR', - 'STARTOFMONTH', - 'STARTOFQUARTER', - 'STARTOFYEAR', - 'TOTALMTD', - 'TOTALQTD', - 'TOTALYTD', + // Table manipulation / filter 'ADDCOLUMNS', 'ADDMISSINGITEMS', 'ALL', + 'ALLCROSSFILTERED', 'ALLEXCEPT', 'ALLNOBLANKROW', 'ALLSELECTED', @@ -129,6 +157,7 @@ export const language = { 'NATURALLEFTOUTERJOIN', 'RELATED', 'RELATEDTABLE', + 'REMOVEFILTERS', 'ROLLUP', 'ROLLUPADDISSUBTOTAL', 'ROLLUPGROUP', @@ -144,6 +173,7 @@ export const language = { 'UNION', 'USERELATIONSHIP', 'VALUES', + // Aggregation / parent-child 'SUM', 'SUMX', 'PATH', @@ -161,6 +191,7 @@ export const language = { 'COUNTROWS', 'COUNTX', 'DISTINCTCOUNT', + 'DISTINCTCOUNTNOBLANK', 'DIVIDE', 'GEOMEAN', 'GEOMEANX', @@ -190,7 +221,32 @@ export const language = { 'VARX.S', 'XIRR', 'XNPV', - // Scalar + // Window / visual calculation functions + 'FIRST', + 'LAST', + 'INDEX', + 'OFFSET', + 'WINDOW', + 'ORDERBY', + 'PARTITIONBY', + 'MATCHBY', + 'RANK', + 'ROWNUMBER', + 'MOVINGAVERAGE', + 'RUNNINGSUM', + 'RANGE', + 'LOOKUP', + 'LOOKUPWITHTOTALS', + 'NEXT', + 'PREVIOUS', + 'LINEST', + 'LINESTX', + 'EXPAND', + 'EXPANDALL', + 'COLLAPSE', + 'COLLAPSEALL', + 'ISATLEVEL', + // Date / time scalar 'DATE', 'DATEDIFF', 'DATEVALUE', @@ -200,43 +256,76 @@ export const language = { 'HOUR', 'MINUTE', 'MONTH', + 'NETWORKDAYS', 'NOW', + 'QUARTER', 'SECOND', 'TIME', 'TIMEVALUE', 'TODAY', + 'UTCNOW', + 'UTCTODAY', 'WEEKDAY', 'WEEKNUM', 'YEAR', 'YEARFRAC', + // Information + 'COLUMNSTATISTICS', 'CONTAINS', 'CONTAINSROW', + 'CONTAINSSTRING', + 'CONTAINSSTRINGEXACT', 'CUSTOMDATA', - 'ERROR', 'HASONEFILTER', 'HASONEVALUE', + 'ISAFTER', 'ISBLANK', + 'ISBOOLEAN', 'ISCROSSFILTERED', + 'ISCURRENCY', + 'ISDATETIME', + 'ISDECIMAL', + 'ISDOUBLE', 'ISEMPTY', 'ISERROR', 'ISEVEN', 'ISFILTERED', + 'ISINSCOPE', 'ISLOGICAL', 'ISNONTEXT', 'ISNUMBER', 'ISODD', + 'ISSELECTEDMEASURE', + 'ISSTRING', 'ISSUBTOTAL', 'ISTEXT', + 'NAMEOF', + 'NONVISUAL', + 'SELECTEDMEASURE', + 'SELECTEDMEASUREFORMATSTRING', + 'SELECTEDMEASURENAME', + 'SELECTEDVALUE', + 'TABLEOF', + 'USERCULTURE', 'USERNAME', + 'USEROBJECTID', 'USERPRINCIPALNAME', + // Logical 'AND', + 'BITAND', + 'BITLSHIFT', + 'BITOR', + 'BITRSHIFT', + 'BITXOR', + 'COALESCE', 'FALSE', 'IF', + 'IF.EAGER', 'IFERROR', - 'NOT', 'OR', 'SWITCH', 'TRUE', + // Math & trig 'ABS', 'ACOS', 'ACOSH', @@ -246,17 +335,8 @@ export const language = { 'ASINH', 'ATAN', 'ATANH', - 'BETA.DIST', - 'BETA.INV', 'CEILING', - 'CHISQ.DIST', - 'CHISQ.DIST.RT', - 'CHISQ.INV', - 'CHISQ.INV.RT', - 'COMBIN', - 'COMBINA', - 'CONFIDENCE.NORM', - 'CONFIDENCE.T', + 'CONVERT', 'COS', 'COSH', 'COT', @@ -265,7 +345,6 @@ export const language = { 'DEGREES', 'EVEN', 'EXP', - 'EXPON.DIST', 'FACT', 'FLOOR', 'GCD', @@ -278,9 +357,7 @@ export const language = { 'MOD', 'MROUND', 'ODD', - 'PERMUT', 'PI', - 'POISSON.DIST', 'POWER', 'QUOTIENT', 'RADIANS', @@ -297,7 +374,32 @@ export const language = { 'TAN', 'TANH', 'TRUNC', + // Statistical + 'BETA.DIST', + 'BETA.INV', + 'CHISQ.DIST', + 'CHISQ.DIST.RT', + 'CHISQ.INV', + 'CHISQ.INV.RT', + 'COMBIN', + 'COMBINA', + 'CONFIDENCE.NORM', + 'CONFIDENCE.T', + 'EXPON.DIST', + 'NORM.DIST', + 'NORM.INV', + 'NORM.S.DIST', + 'NORM.S.INV', + 'PERMUT', + 'POISSON.DIST', + 'T.DIST', + 'T.DIST.2T', + 'T.DIST.RT', + 'T.INV', + 'T.INV.2T', + // Text 'BLANK', + 'COMBINEVALUES', 'CONCATENATE', 'CONCATENATEX', 'EXACT', @@ -317,8 +419,118 @@ export const language = { 'UNICHAR', 'UNICODE', 'UPPER', - 'VALUE' + 'VALUE', + // Financial + 'ACCRINT', + 'ACCRINTM', + 'AMORDEGRC', + 'AMORLINC', + 'COUPDAYBS', + 'COUPDAYS', + 'COUPDAYSNC', + 'COUPNCD', + 'COUPNUM', + 'COUPPCD', + 'CUMIPMT', + 'CUMPRINC', + 'DB', + 'DDB', + 'DISC', + 'DOLLARDE', + 'DOLLARFR', + 'DURATION', + 'EFFECT', + 'FV', + 'INTRATE', + 'IPMT', + 'IRR', + 'ISPMT', + 'MDURATION', + 'MIRR', + 'NOMINAL', + 'NPER', + 'NPV', + 'ODDFPRICE', + 'ODDFYIELD', + 'ODDLPRICE', + 'ODDLYIELD', + 'PDURATION', + 'PMT', + 'PPMT', + 'PRICE', + 'PRICEDISC', + 'PRICEMAT', + 'PV', + 'RATE', + 'RECEIVED', + 'RRI', + 'SLN', + 'SYD', + 'TBILLEQ', + 'TBILLPRICE', + 'TBILLYIELD', + 'VDB', + 'YIELD', + 'YIELDDISC', + 'YIELDMAT', + // Other + 'ERROR', + 'EVALUATEANDLOG', + 'EXTERNALMEASURE', + 'TOCSV', + 'TOJSON', + // INFO DMV-backed family (info-functions-dax) + 'INFO.ALTERNATEOFDEFINITIONS', + 'INFO.ANNOTATIONS', + 'INFO.ATTRIBUTEHIERARCHIES', + 'INFO.ATTRIBUTEHIERARCHYSTORAGES', + 'INFO.CATALOGS', + 'INFO.CHANGEDPROPERTIES', + 'INFO.COLUMNPERMISSIONS', + 'INFO.COLUMNS', + 'INFO.CULTURES', + 'INFO.DATACOVERAGEDEFINITIONS', + 'INFO.DATASOURCES', + 'INFO.DETAILROWSDEFINITIONS', + 'INFO.EXCLUDEDARTIFACTS', + 'INFO.EXPRESSIONS', + 'INFO.EXTENDEDPROPERTIES', + 'INFO.FORMATSTRINGDEFINITIONS', + 'INFO.FUNCTIONS', + 'INFO.GROUPBYCOLUMNS', + 'INFO.HIERARCHIES', + 'INFO.KPIS', + 'INFO.LEVELS', + 'INFO.LINGUISTICMETADATA', + 'INFO.MEASURES', + 'INFO.MODEL', + 'INFO.OBJECTTRANSLATIONS', + 'INFO.PARQUETFILESTATISTICS', + 'INFO.PARTITIONS', + 'INFO.PERSPECTIVECOLUMNS', + 'INFO.PERSPECTIVEHIERARCHIES', + 'INFO.PERSPECTIVEMEASURES', + 'INFO.PERSPECTIVES', + 'INFO.PERSPECTIVETABLES', + 'INFO.QUERYGROUPS', + 'INFO.REFRESHPOLICIES', + 'INFO.RELATEDCOLUMNDETAILS', + 'INFO.RELATIONSHIPS', + 'INFO.ROLEMEMBERSHIPS', + 'INFO.ROLES', + 'INFO.STORAGEFOLDERS', + 'INFO.STORAGETABLECOLUMNS', + 'INFO.STORAGETABLECOLUMNSEGMENTS', + 'INFO.STORAGETABLES', + 'INFO.TABLEPERMISSIONS', + 'INFO.TABLES', + 'INFO.VARIATIONS', + 'INFO.VIEW.COLUMNS', + 'INFO.VIEW.MEASURES', + 'INFO.VIEW.RELATIONSHIPS', + 'INFO.VIEW.TABLES' ], + tokenizer: { root: [ { include: '@comments' }, @@ -326,18 +538,26 @@ export const language = { { include: '@numbers' }, { include: '@strings' }, { include: '@complexIdentifiers' }, - [/[;,.]/, 'delimiter'], + [/[;,]/, 'delimiter'], [/[({})]/, '@brackets'], + // Identifier or dotted identifier (e.g., PERCENTILE.EXC, INFO.VIEW.MEASURES). + // Dotted segments are captured greedily so that keywords/functions with dots + // are matched as a single token against @keywords / @functions. [ - /[a-z_][a-zA-Z0-9_]*/, + /[a-z_][a-zA-Z0-9_]*(?:\.[a-zA-Z][a-zA-Z0-9]*)*/, { cases: { '@keywords': 'keyword', - '@functions': 'keyword', + '@functions': 'keyword.function', '@default': 'identifier' } } ], + // Standalone '.' (e.g., between identifiers that aren't a known dotted function) + [/\./, 'delimiter'], + // Multi-char operators must precede the single-char operator rule. + // Per dax-operator-reference: => (lambda/UDF), == (strict equal), <>, <=, >=, !=, &&, || + [/=>|==|<>|<=|>=|!=|&&|\|\|/, 'operator'], [/[<>=!%&+\-*/|~^]/, 'operator'] ], whitespace: [[/\s+/, 'white']], @@ -356,6 +576,9 @@ export const language = { [/((\d+(\.\d*)?)|(\.\d+))([eE][\-+]?\d+)?/, 'number'] ], strings: [ + // DAX datetime literal: dt"YYYY-MM-DD", dt"YYYY-MM-DDThh:mm:ss" (dax-syntax-reference). + // Must precede the generic string rule so that "dt" isn't first matched as an identifier. + [/dt"/, { token: 'string', next: '@string' }], [/N"/, { token: 'string', next: '@string' }], [/"/, { token: 'string', next: '@string' }] ], diff --git a/src/languages/definitions/powerquery/powerquery.test.ts b/src/languages/definitions/powerquery/powerquery.test.ts index 9066966e08..aa0cc01829 100644 --- a/src/languages/definitions/powerquery/powerquery.test.ts +++ b/src/languages/definitions/powerquery/powerquery.test.ts @@ -228,7 +228,7 @@ testTokenization('powerquery', [ } ], - // built-in keywords/identifiers + // keywords [ { line: 'And as Each each _', @@ -246,19 +246,66 @@ testTokenization('powerquery', [ } ], + // each keyword [ { - line: ' #table({})', + line: 'Table.SelectRows(Products, each [Price] > 100)', tokens: [ - { startIndex: 0, type: 'white.pq' }, - { startIndex: 2, type: 'constructor.pq' }, - { startIndex: 8, type: 'delimiter.parenthesis.pq' }, - { startIndex: 9, type: 'delimiter.brackets.pq' }, - { startIndex: 11, type: 'delimiter.parenthesis.pq' } + { startIndex: 0, type: 'keyword.function.pq' }, + { startIndex: 16, type: 'delimiter.parenthesis.pq' }, + { startIndex: 17, type: 'identifier.pq' }, + { startIndex: 25, type: 'delimiter.pq' }, + { startIndex: 26, type: 'white.pq' }, + { startIndex: 27, type: 'keyword.pq' }, + { startIndex: 31, type: 'white.pq' }, + { startIndex: 32, type: 'delimiter.square.pq' }, + { startIndex: 33, type: 'identifier.pq' }, + { startIndex: 38, type: 'delimiter.square.pq' }, + { startIndex: 39, type: 'white.pq' }, + { startIndex: 40, type: 'operators.pq' }, + { startIndex: 41, type: 'white.pq' }, + { startIndex: 42, type: 'number.pq' }, + { startIndex: 45, type: 'delimiter.parenthesis.pq' } + ] + } + ], + + // section and shared keywords + [ + { + // section(0-6) (7) MySection(8-16) ;(17) + line: 'section MySection;', + tokens: [ + { startIndex: 0, type: 'keyword.pq' }, + { startIndex: 7, type: 'white.pq' }, + { startIndex: 8, type: 'identifier.pq' }, + { startIndex: 17, type: 'delimiter.pq' } + ] + } + ], + + [ + { + // shared(0-5) (6) MyFunction(7-16) (17) =(18) (19) ()(20-21 merged) (22) =>(23-24) (25) 42(26-27) ;(28) + line: 'shared MyFunction = () => 42;', + tokens: [ + { startIndex: 0, type: 'keyword.pq' }, + { startIndex: 6, type: 'white.pq' }, + { startIndex: 7, type: 'identifier.pq' }, + { startIndex: 17, type: 'white.pq' }, + { startIndex: 18, type: 'operators.pq' }, + { startIndex: 19, type: 'white.pq' }, + { startIndex: 20, type: 'delimiter.parenthesis.pq' }, // () merged + { startIndex: 22, type: 'white.pq' }, + { startIndex: 23, type: 'operators.pq' }, + { startIndex: 25, type: 'white.pq' }, + { startIndex: 26, type: 'number.pq' }, + { startIndex: 28, type: 'delimiter.pq' } ] } ], + // as / is type annotations [ { line: 'param as number', @@ -272,6 +319,33 @@ testTokenization('powerquery', [ } ], + [ + { + line: 'value is text', + tokens: [ + { startIndex: 0, type: 'identifier.pq' }, + { startIndex: 5, type: 'white.pq' }, + { startIndex: 6, type: 'keyword.pq' }, + { startIndex: 8, type: 'white.pq' }, + { startIndex: 9, type: 'type.pq' } + ] + } + ], + + // #table constructor + [ + { + line: ' #table({})', + tokens: [ + { startIndex: 0, type: 'white.pq' }, + { startIndex: 2, type: 'constructor.pq' }, + { startIndex: 8, type: 'delimiter.parenthesis.pq' }, + { startIndex: 9, type: 'delimiter.brackets.pq' }, + { startIndex: 11, type: 'delimiter.parenthesis.pq' } + ] + } + ], + [ { line: 'type table', @@ -308,7 +382,43 @@ testTokenization('powerquery', [ } ], - // built-ins + // hash literals / constants + [ + { + line: '#infinity', + tokens: [{ startIndex: 0, type: 'constant.pq' }] + } + ], + + [ + { + line: '#nan', + tokens: [{ startIndex: 0, type: 'constant.pq' }] + } + ], + + [ + { + line: '#sections', + tokens: [{ startIndex: 0, type: 'constant.pq' }] + } + ], + + [ + { + line: '#date', + tokens: [{ startIndex: 0, type: 'constructor.pq' }] + } + ], + + [ + { + line: '#time', + tokens: [{ startIndex: 0, type: 'constructor.pq' }] + } + ], + + // built-in functions [ { line: 'Text.From(1)', @@ -357,7 +467,26 @@ testTokenization('powerquery', [ } ], - // other statements + // try / otherwise — try(0-2) (3) Text.From(4-12) ((13) "invalid"(14-22) )(23) (24) otherwise(25-33) (34) "default"(35-43) + [ + { + line: 'try Text.From("invalid") otherwise "default"', + tokens: [ + { startIndex: 0, type: 'keyword.pq' }, + { startIndex: 3, type: 'white.pq' }, + { startIndex: 4, type: 'keyword.function.pq' }, + { startIndex: 13, type: 'delimiter.parenthesis.pq' }, + { startIndex: 14, type: 'string.pq' }, + { startIndex: 23, type: 'delimiter.parenthesis.pq' }, + { startIndex: 24, type: 'white.pq' }, + { startIndex: 25, type: 'keyword.pq' }, + { startIndex: 34, type: 'white.pq' }, + { startIndex: 35, type: 'string.pq' } + ] + } + ], + + // section with metadata and shared [ { line: '[version="1.0.0.1"] section Foo; shared Member.Name = 1;', @@ -412,5 +541,88 @@ testTokenization('powerquery', [ { startIndex: 30, type: 'delimiter.parenthesis.pq' } ] } + ], + + // try / catch (error-handling, May-2022 addition) + [ + { + // try(0-2) (3) foo(4-6) (7) catch(8-12) (13) ((14)e(15))(16) =>(17-18) (19) e(20) + line: 'try foo catch (e) => e', + tokens: [ + { startIndex: 0, type: 'keyword.pq' }, // try + { startIndex: 3, type: 'white.pq' }, + { startIndex: 4, type: 'identifier.pq' }, + { startIndex: 7, type: 'white.pq' }, + { startIndex: 8, type: 'keyword.pq' }, // catch + { startIndex: 13, type: 'white.pq' }, + { startIndex: 14, type: 'delimiter.parenthesis.pq' }, + { startIndex: 15, type: 'identifier.pq' }, + { startIndex: 16, type: 'delimiter.parenthesis.pq' }, + { startIndex: 17, type: 'white.pq' }, + { startIndex: 18, type: 'operators.pq' }, // => + { startIndex: 20, type: 'white.pq' }, + { startIndex: 21, type: 'identifier.pq' } + ] + } + ], + + [ + { + // Empty parameter list: `()` is tokenized as a single merged parenthesis + // delimiter token. + line: 'try foo catch () => 1', + tokens: [ + { startIndex: 0, type: 'keyword.pq' }, + { startIndex: 3, type: 'white.pq' }, + { startIndex: 4, type: 'identifier.pq' }, + { startIndex: 7, type: 'white.pq' }, + { startIndex: 8, type: 'keyword.pq' }, // catch + { startIndex: 13, type: 'white.pq' }, + { startIndex: 14, type: 'delimiter.parenthesis.pq' }, // () merged + { startIndex: 16, type: 'white.pq' }, + { startIndex: 17, type: 'operators.pq' }, // => + { startIndex: 19, type: 'white.pq' }, + { startIndex: 20, type: 'number.pq' } + ] + } + ], + + // ?? coalesce operator (m-spec-operators) + [ + { + line: 'a ?? b', + tokens: [ + { startIndex: 0, type: 'identifier.pq' }, + { startIndex: 1, type: 'white.pq' }, + { startIndex: 2, type: 'operators.pq' }, // ?? (single multi-char token) + { startIndex: 4, type: 'white.pq' }, + { startIndex: 5, type: 'identifier.pq' } + ] + } + ], + + [ + { + line: 'x = y ?? 0', + tokens: [ + { startIndex: 0, type: 'identifier.pq' }, + { startIndex: 1, type: 'white.pq' }, + { startIndex: 2, type: 'operators.pq' }, // = + { startIndex: 3, type: 'white.pq' }, + { startIndex: 4, type: 'identifier.pq' }, + { startIndex: 5, type: 'white.pq' }, + { startIndex: 6, type: 'operators.pq' }, // ?? + { startIndex: 8, type: 'white.pq' }, + { startIndex: 9, type: 'number.pq' } + ] + } + ], + + // Verbatim text literal #!"..." (m-spec-consolidated-grammar §Literal) + [ + { + line: '#!"raw text"', + tokens: [{ startIndex: 0, type: 'string.pq' }] + } ] ]); diff --git a/src/languages/definitions/powerquery/powerquery.ts b/src/languages/definitions/powerquery/powerquery.ts index fd674fec90..185b626c07 100644 --- a/src/languages/definitions/powerquery/powerquery.ts +++ b/src/languages/definitions/powerquery/powerquery.ts @@ -37,7 +37,10 @@ export const language = { operatorKeywords: ['and', 'not', 'or'], keywords: [ + // Per m-spec-consolidated-grammar §Keywords, and May-2022 addition of `catch` + // (m-spec-error-handling, error-handling). 'as', + 'catch', 'each', 'else', 'error', @@ -131,7 +134,9 @@ export const language = { 'BinaryFormat.UnsignedInteger64', 'Byte.From', 'Character.FromNumber', + 'Character.ToLower', 'Character.ToNumber', + 'Character.ToUpper', 'Combiner.CombineTextByDelimiter', 'Combiner.CombineTextByEachDelimiter', 'Combiner.CombineTextByLengths', @@ -257,6 +262,10 @@ export const language = { 'DateTimeZone.ZoneMinutes', 'Decimal.From', 'Diagnostics.ActivityId', + 'Diagnostics.BeginTrace', + 'Diagnostics.LogFailure', + 'Diagnostics.LogValue', + 'Diagnostics.LogValue2', 'Diagnostics.Trace', 'DirectQueryCapabilities.From', 'Double.From', @@ -295,6 +304,10 @@ export const language = { 'HdInsight.Files', 'Hdfs.Contents', 'Hdfs.Files', + 'Html.Table', + 'Identity.From', + 'Identity.IsMemberOf', + 'IdentityProvider.Default', 'Informix.Database', 'Int16.From', 'Int32.From', @@ -424,15 +437,18 @@ export const language = { 'Number.ToText', 'OData.Feed', 'Odbc.DataSource', + 'Odbc.InferOptions', 'Odbc.Query', 'OleDb.DataSource', 'OleDb.Query', 'Oracle.Database', + 'Pdf.Tables', 'Percentage.From', 'PostgreSQL.Database', 'RData.FromBinary', 'Record.AddField', 'Record.Combine', + 'Record.CombineLists', 'Record.Field', 'Record.FieldCount', 'Record.FieldNames', @@ -440,6 +456,7 @@ export const language = { 'Record.FieldValues', 'Record.FromList', 'Record.FromTable', + 'Record.FromFields', 'Record.HasFields', 'Record.RemoveFields', 'Record.RenameFields', @@ -476,9 +493,11 @@ export const language = { 'SqlExpression.ToExpression', 'Sybase.Database', 'Table.AddColumn', + 'Table.AddFuzzyClusterColumn', 'Table.AddIndexColumn', 'Table.AddJoinColumn', 'Table.AddKey', + 'Table.AddRankColumn', 'Table.AggregateTableColumn', 'Table.AlternateRows', 'Table.Buffer', @@ -488,6 +507,8 @@ export const language = { 'Table.ColumnsOfType', 'Table.Combine', 'Table.CombineColumns', + 'Table.CombineColumnsToRecord', + 'Table.ConformToPageReader', 'Table.Contains', 'Table.ContainsAll', 'Table.ContainsAny', @@ -510,6 +531,9 @@ export const language = { 'Table.FromRecords', 'Table.FromRows', 'Table.FromValue', + 'Table.FuzzyGroup', + 'Table.FuzzyJoin', + 'Table.FuzzyNestedJoin', 'Table.Group', 'Table.HasColumns', 'Table.InsertRows', @@ -560,6 +584,7 @@ export const language = { 'Table.Skip', 'Table.Sort', 'Table.SplitColumn', + 'Table.StopFolding', 'Table.ToColumns', 'Table.ToList', 'Table.ToRecords', @@ -606,6 +631,7 @@ export const language = { 'Text.Repeat', 'Text.Replace', 'Text.ReplaceRange', + 'Text.Reverse', 'Text.Select', 'Text.Split', 'Text.SplitAny', @@ -653,10 +679,12 @@ export const language = { 'Uri.EscapeDataString', 'Uri.Parts', 'Value.Add', + 'Value.Alternates', 'Value.As', 'Value.Compare', 'Value.Divide', 'Value.Equals', + 'Value.Expression', 'Value.Firewall', 'Value.FromText', 'Value.Is', @@ -665,10 +693,14 @@ export const language = { 'Value.NativeQuery', 'Value.NullableEquals', 'Value.RemoveMetadata', + 'Value.ReplaceFields', 'Value.ReplaceMetadata', 'Value.ReplaceType', 'Value.Subtract', + 'Value.Traits', 'Value.Type', + 'Value.VersionIdentity', + 'Value.Versions', 'ValueAction.NativeStatement', 'ValueAction.Replace', 'Variable.Value', @@ -836,15 +868,31 @@ export const language = { tokenizer: { root: [ - // quoted identifier + // Verbatim text literal: #!"..." (m-spec-consolidated-grammar §Literal). + // Must precede the quoted-identifier rule so that #!"..." is not matched + // as a quoted identifier. + [/#!"/, { token: 'string', next: '@string' }], + + // Quoted identifier (e.g., #"My Column") [/#"[\w \.]+"/, 'identifier.quote'], - // numbers + // Hash literals / constructors (#nan, #infinity, #table, etc.) + [/#[\w]+/, { + cases: { + '@constants': 'constant', + '@constructors': 'constructor', + '@default': 'identifier' + } + }], + + // Numbers [/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'], [/0[xX][0-9a-fA-F]+/, 'number.hex'], [/\d+([eE][\-+]?\d+)?/, 'number'], - // keywords + // Keywords, type keywords, constructors, operator keywords. + // `as`, `catch`, `each`, `is`, `section`, `shared`, etc. are all lowercase + // and therefore fall into this rule under `ignoreCase: false`. [ /(#?[a-z]+)\b/, { @@ -859,7 +907,7 @@ export const language = { } ], - // built-in types + // Built-in types: Table.Type, Text.Type, etc. [ /\b([A-Z][a-zA-Z0-9]+\.Type)\b/, { @@ -870,7 +918,7 @@ export const language = { } ], - // other built-ins + // Built-in functions and constants: Table.First, Number.PI, etc. [ /\b([A-Z][a-zA-Z0-9]+\.[A-Z][a-zA-Z0-9]+)\b/, { @@ -882,7 +930,7 @@ export const language = { } ], - // other identifiers + // Other identifiers [/\b([a-zA-Z_][\w\.]*)\b/, 'identifier'], { include: '@whitespace' }, @@ -890,7 +938,11 @@ export const language = { { include: '@strings' }, [/[{}()\[\]]/, '@brackets'], - [/([=\+<>\-\*&@\?\/!])|([<>]=)|(<>)|(=>)|(\.\.\.)|(\.\.)/, 'operators'], + // Operators. Multi-char alternatives come first so `??`, `=>`, `...`, `..`, + // `<=`, `>=`, `<>` are not split into single-char tokens. + // Per m-spec-operators and m-spec-consolidated-grammar §Operators: + // , ; = < <= > >= <> + - * / & ( ) [ ] { } @ ? ?? => .. ... + [/\?\?|=>|\.\.\.|\.\.|<=|>=|<>|[=+<>\-*&@?\/!]/, 'operators'], [/[,;]/, 'delimiter'] ],