From 8dd8ad763e603e37b8b62fb59e2b97819f296c66 Mon Sep 17 00:00:00 2001 From: Oleksandr Pelykh Date: Thu, 11 Jun 2026 16:36:19 +0300 Subject: [PATCH 1/7] feat: add test alias support for playwright --- example/playwright/annotations-status.ts | 27 ++++ .../playwright/custom-fixture-annotations.ts | 29 ++++ .../playwright/sibling-after-skipped-suite.ts | 16 ++ src/analyzer.js | 18 ++- src/lib/frameworks/playwright.js | 147 ++++++++---------- tests/analyzer_test.js | 27 ++++ tests/playwright_test.js | 97 +++++++++++- 7 files changed, 277 insertions(+), 84 deletions(-) create mode 100644 example/playwright/annotations-status.ts create mode 100644 example/playwright/custom-fixture-annotations.ts create mode 100644 example/playwright/sibling-after-skipped-suite.ts diff --git a/example/playwright/annotations-status.ts b/example/playwright/annotations-status.ts new file mode 100644 index 00000000..01cabe55 --- /dev/null +++ b/example/playwright/annotations-status.ts @@ -0,0 +1,27 @@ +import { test, expect } from '@playwright/test'; + +test('plain test', async () => { + await expect(true).toBe(true); +}); + +// .fail marks a test as expected to fail, but it still runs => not skipped +test.fail('expected to fail test', async () => { + await expect(true).toBe(false); +}); + +// .todo => skipped test +test.todo('todo test'); + +// runtime forms without a title declare no separate test +test('runtime annotations have no title', async () => { + test.fail(); + test.skip(); + await expect(true).toBe(true); +}); + +// .fail inside a skipped suite => skipped (suite wins) +test.describe.skip('skipped suite', () => { + test.fail('fail inside skipped suite', async () => { + await expect(true).toBe(false); + }); +}); diff --git a/example/playwright/custom-fixture-annotations.ts b/example/playwright/custom-fixture-annotations.ts new file mode 100644 index 00000000..ba94f8db --- /dev/null +++ b/example/playwright/custom-fixture-annotations.ts @@ -0,0 +1,29 @@ +import { test as base } from '@playwright/test'; + +const testFixture = base.extend<{ someFixture: any }>({ + someFixture: async ({}, use) => { + await use({ name: 'custom fixture name' }); + }, +}); + +testFixture('plain alias test', async ({ someFixture }) => { + console.warn(someFixture.name); +}); + +testFixture.skip('skipped alias test', async ({ someFixture }) => { + console.warn(someFixture.name); +}); + +testFixture.fixme('fixme alias test', async ({ someFixture }) => { + console.warn(someFixture.name); +}); + +testFixture.fail('failing alias test', async ({ someFixture }) => { + console.warn(someFixture.name); +}); + +testFixture.describe('alias suite', () => { + testFixture.fixme('fixme test inside alias suite', async ({ someFixture }) => { + console.warn(someFixture.name); + }); +}); diff --git a/example/playwright/sibling-after-skipped-suite.ts b/example/playwright/sibling-after-skipped-suite.ts new file mode 100644 index 00000000..74e96325 --- /dev/null +++ b/example/playwright/sibling-after-skipped-suite.ts @@ -0,0 +1,16 @@ +import { test, expect } from '@playwright/test'; + +test.describe.skip('skipped suite', () => { + test('inside skipped suite', async () => { + await expect(true).toBe(true); + }); +}); + +// these are siblings declared AFTER the skipped suite closed - they must not inherit skipped +test('sibling after skipped suite', async () => { + await expect(true).toBe(true); +}); + +test.fail('failing sibling after skipped suite', async () => { + await expect(true).toBe(false); +}); diff --git a/src/analyzer.js b/src/analyzer.js index 965296e5..19d8cfbe 100644 --- a/src/analyzer.js +++ b/src/analyzer.js @@ -83,6 +83,20 @@ class Analyzer { // this.addPlugin('@babel/plugin-transform-typescript'); } + // Build the list of glob patterns to scan. When TypeScript support is enabled, a JS-only + // glob (e.g. "**/*.test.js") would silently match nothing in a TS project, so we also scan + // the TypeScript equivalents by swapping the trailing `.js` extension for `.ts`/`.tsx`/etc. + buildPatterns(pattern) { + const patterns = [pattern]; + if (this.typeScript && /\.js$/.test(pattern)) { + const base = pattern.replace(/\.js$/, ''); + for (const ext of ['ts', 'tsx', 'mts', 'cts']) { + patterns.push(`${base}.${ext}`); + } + } + return patterns; + } + analyze(pattern) { if (!this.frameworkParser) throw new Error("No test framework specified. Can't analyze"); @@ -93,7 +107,9 @@ class Analyzer { const originalCwd = process.cwd(); process.chdir(this.workDir); - let files = glob.sync(pattern, { windowsPathsNoEscape: true }); + const patterns = this.buildPatterns(pattern); + debug('Patterns:', patterns); + let files = [...new Set(patterns.flatMap(p => glob.sync(p, { windowsPathsNoEscape: true })))]; // Exclude files matching the exclude pattern if provided if (this.opts.exclude) { diff --git a/src/lib/frameworks/playwright.js b/src/lib/frameworks/playwright.js index e7a6782e..0f9d0262 100644 --- a/src/lib/frameworks/playwright.js +++ b/src/lib/frameworks/playwright.js @@ -23,12 +23,52 @@ module.exports = (ast, file = '', source = '', opts = {}) => { let beforeEachCode = ''; let afterCode = ''; + // valid test identifiers: built-in `test`/`it` plus any custom fixtures/aliases + const testNames = ['test', 'it', ...(opts?.testAlias || [])]; + function addSuite(path) { currentSuite = currentSuite.filter(s => s.loc.end.line > path.loc.start.line); path.tags = playwright.getTestProps({ parent: { expression: path } }).tags; currentSuite.push(path); } + // suites that actually enclose the call at `path`. `currentSuite` is only pruned when a + // new suite is added, so it can still hold sibling suites that already closed above this + // line — those must not leak their name or `skipped` flag onto a test declared after them. + function getEnclosingSuites(path) { + return currentSuite.filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)); + } + + // resolve the name of the test object an annotation (`.skip`, `.fixme`, `.fail`, `.todo`) + // is called on, e.g. `test`, `it`, `describe` or a custom test alias / fixture name + function getTestObjectName(path) { + if (!path.parent || !path.parent.object) return null; + return ( + path.parent.object.name || path.parent.object.property?.name || path.parent.object.callee?.object?.name || null + ); + } + + // Register a single named test declared with an annotation call + // (`test.skip` / `test.fixme` / `test.fail` / `test.todo`, or the alias equivalents). + // `path` is the annotation identifier node; its enclosing call holds the test title. + // Calls without a string title (the runtime form `test.skip()` used inside a test body) + // declare no test and are ignored. The caller decides `skipped`. + function registerAnnotatedTest(path, { skipped }) { + if (!hasStringOrTemplateArgument(path.parentPath.container)) return; + + tests.push({ + name: getStringValue(path.parentPath.container), + suites: getEnclosingSuites(path).map(s => getStringValue(s)), + line: getLineNumber(path), + // `path` is the annotation identifier (`fixme`/`skip`/...); its container ends on the + // member-expression line only. The full call (and its body) is `path.parentPath.container`, + // so take the end line from there to capture the complete test code. + code: getCode(source, getLineNumber(path), getEndLineNumber(path.parentPath), isLineNumber), + file, + skipped, + }); + } + traverse(ast, { enter(path) { if (path.isIdentifier({ name: 'describe' })) { @@ -90,31 +130,16 @@ module.exports = (ast, file = '', source = '', opts = {}) => { } } - if (path.isIdentifier({ name: 'skip' })) { - if (!path.parent || !path.parent.object) { - return; - } - const name = - path.parent.object.name || path.parent.object.property.name || path.parent.object.callee.object.name; - - if (name === 'test' || name === 'it') { - // test or it - if (!hasStringOrTemplateArgument(path.parentPath.container)) return; - - const testName = getStringValue(path.parentPath.container); - tests.push({ - name: testName, - suites: currentSuite - .filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)) - .map(s => getStringValue(s)), - line: getLineNumber(path), - code: getCode(source, getLineNumber(path), getEndLineNumber(path), isLineNumber), - file, - skipped: true, - }); - } + // `.skip` / `.fixme` mark a test (or every test in a suite) as skipped, + // supporting `test`, `it`, `describe` and any custom test alias / fixture + if (path.isIdentifier({ name: 'skip' }) || path.isIdentifier({ name: 'fixme' })) { + const name = getTestObjectName(path); + if (!name) return; - if (name === 'describe') { + if (testNames.includes(name)) { + // test or it (or alias), e.g. `myFixture.fixme('...', ...)` + registerAnnotatedTest(path, { skipped: true }); + } else if (name === 'describe') { // suite if (!hasStringOrTemplateArgument(path.parentPath.container)) return; const suite = path.parentPath.container; @@ -125,61 +150,23 @@ module.exports = (ast, file = '', source = '', opts = {}) => { // todo: handle "context" } - if (path.isIdentifier({ name: 'fixme' })) { - if (!path.parent || !path.parent.object) { - return; - } - const name = - path.parent.object.name || path.parent.object.property.name || path.parent.object.callee.object.name; - - if (name === 'test' || name === 'it') { - // test or it - if (!hasStringOrTemplateArgument(path.parentPath.container)) return; + // `.fail` marks a test as expected to fail; it still runs, so it is not skipped + if (path.isIdentifier({ name: 'fail' })) { + const name = getTestObjectName(path); + if (!name) return; - const testName = getStringValue(path.parentPath.container); - tests.push({ - name: testName, - suites: currentSuite - .filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)) - .map(s => getStringValue(s)), - line: getLineNumber(path), - code: getCode(source, getLineNumber(path), getEndLineNumber(path), isLineNumber), - file, - skipped: true, - }); + if (testNames.includes(name)) { + registerAnnotatedTest(path, { skipped: getEnclosingSuites(path).some(s => s.skipped) }); } - - if (name === 'describe') { - // suite - if (!hasStringOrTemplateArgument(path.parentPath.container)) return; - const suite = path.parentPath.container; - suite.skipped = true; - addSuite(suite); - } - - // todo: handle "context" } if (path.isIdentifier({ name: 'todo' })) { - if (!path.parent || !path.parent.object) { - return; - } - // todo tests => skipped tests - if (path.parent.object.name === 'test') { - // test - if (!hasStringOrTemplateArgument(path.parentPath.container)) return; + const name = getTestObjectName(path); + if (!name) return; - const testName = getStringValue(path.parentPath.container); - tests.push({ - name: testName, - suites: currentSuite - .filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)) - .map(s => getStringValue(s)), - line: getLineNumber(path), - code: getCode(source, getLineNumber(path), getEndLineNumber(path), isLineNumber), - file, - skipped: true, - }); + // todo tests => skipped tests + if (testNames.includes(name)) { + registerAnnotatedTest(path, { skipped: true }); } } @@ -202,19 +189,18 @@ module.exports = (ast, file = '', source = '', opts = {}) => { afterCode; const testName = getStringValue(path.parent); + const enclosingSuites = getEnclosingSuites(path); tests.push({ name: testName, - suites: currentSuite - .filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)) - .map(s => getStringValue(s)), + suites: enclosingSuites.map(s => getStringValue(s)), updatePoint: getUpdatePoint(path.parent), line: getLineNumber(path), code, file, tags: [...getAllSuiteTags(currentSuite), ...playwright.getTestProps(path.parentPath).tags], annotations: playwright.getTestProps(path.parentPath).annotations, - skipped: !!currentSuite.filter(s => s.skipped).length, + skipped: enclosingSuites.some(s => s.skipped), }); // stop the loop if the test is found @@ -227,16 +213,15 @@ module.exports = (ast, file = '', source = '', opts = {}) => { if (!hasStringOrTemplateArgument(currentPath.parent)) return; const testName = getStringValue(currentPath.parent); + const enclosingSuites = getEnclosingSuites(path); tests.push({ name: testName, - suites: currentSuite - .filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)) - .map(s => getStringValue(s)), + suites: enclosingSuites.map(s => getStringValue(s)), updatePoint: getUpdatePoint(path.parent), line: getLineNumber(currentPath), code: getCode(source, getLineNumber(currentPath), getEndLineNumber(currentPath), isLineNumber), file, - skipped: !!currentSuite.filter(s => s.skipped).length, + skipped: enclosingSuites.some(s => s.skipped), }); } }, diff --git a/tests/analyzer_test.js b/tests/analyzer_test.js index 31fc7a07..06ae8fc7 100644 --- a/tests/analyzer_test.js +++ b/tests/analyzer_test.js @@ -42,6 +42,33 @@ describe('analyzer', () => { expect(decorator.getSuiteNames()).to.include('Login - Global Header: Institutional Sign In Modal'); }); + it('should also scan TypeScript files when given a JS-only glob and TypeScript is enabled', () => { + analyzer = new Analyzer('mocha', path.join(__dirname, '..')); + analyzer.withTypeScript(); + // a `.js` pattern would match nothing in this TS-only dir; buildPatterns adds the `.ts` variant + analyzer.analyze('./example/protractor/**.js'); + const decorator = analyzer.getDecorator(); + expect(decorator.getSuiteNames()).to.include('Login - Global Header: Institutional Sign In Modal'); + }); + + it('buildPatterns expands a trailing .js extension to TS variants only when TypeScript is enabled', () => { + analyzer = new Analyzer('mocha', path.join(__dirname, '..')); + + // without TypeScript the pattern is left untouched + expect(analyzer.buildPatterns('./example/foo/**.js')).to.deep.equal(['./example/foo/**.js']); + // non-.js patterns are never expanded, even with TypeScript on + analyzer.withTypeScript(); + expect(analyzer.buildPatterns('./example/foo/**.ts')).to.deep.equal(['./example/foo/**.ts']); + // .js patterns gain the TS equivalents + expect(analyzer.buildPatterns('./example/foo/**.js')).to.deep.equal([ + './example/foo/**.js', + './example/foo/**.ts', + './example/foo/**.tsx', + './example/foo/**.mts', + './example/foo/**.cts', + ]); + }); + it('should exclude dir in file name if dir specified', () => { analyzer = new Analyzer('mocha', 'example'); analyzer.analyze('mocha/**_test.js'); diff --git a/tests/playwright_test.js b/tests/playwright_test.js index 42bcb5ed..474aa5e4 100644 --- a/tests/playwright_test.js +++ b/tests/playwright_test.js @@ -248,9 +248,11 @@ test.describe.only('my test', () => { ast = jsParser.parse(source, { sourceType: 'unambiguous' }); const tests = playwrightParser(ast, '', source); - expect(tests[1].code.trim()).to.equal("test.skip('my skip test @first', async ({ page }) => {".trim()); expect(tests[1].name).to.equal('my skip test @first'); expect(tests[1].suites.length).to.eql(1); + // code captures the full test body (signature + assertions), not just the signature line + expect(tests[1].code).to.include("test.skip('my skip test @first', async ({ page }) => {"); + expect(tests[1].code).to.include("await expect(page).toHaveURL('https://my.start.url/');"); }); it('should parse playwright-js tests with annotation including fixme', () => { @@ -258,9 +260,11 @@ test.describe.only('my test', () => { ast = jsParser.parse(source, { sourceType: 'unambiguous' }); const tests = playwrightParser(ast, '', source); - expect(tests[2].code.trim()).to.equal("test.fixme('my fixme test @third', async ({ page }) => {".trim()); expect(tests[2].name).to.equal('my fixme test @third'); expect(tests[2].suites.length).to.eql(1); + // code captures the full test body (signature + assertions), not just the signature line + expect(tests[2].code).to.include("test.fixme('my fixme test @third', async ({ page }) => {"); + expect(tests[2].code).to.include("await expect(page).toHaveURL('https://my.start.url/');"); }); it('should parse playwright-ts tests with annotations', () => { @@ -505,6 +509,95 @@ test.describe.only('my test', () => { expect(tests.length).to.equal(1); }); + it('should parse annotations (.skip/.fixme/.fail) on a custom test alias', () => { + source = fs.readFileSync('./example/playwright/custom-fixture-annotations.ts').toString(); + ast = jsParser.parse(source, { sourceType: 'unambiguous', plugins: ['typescript'] }); + const tests = playwrightParser(ast, '', source, { testAlias: ['testFixture'] }); + + const byName = name => tests.find(t => t.name === name); + + expect(tests.length).to.equal(5); + + expect(byName('plain alias test').skipped).to.be.false; + // .skip and .fixme on the alias mark the test as skipped + expect(byName('skipped alias test').skipped).to.be.true; + expect(byName('fixme alias test').skipped).to.be.true; + // .fail still runs, so it is not skipped + expect(byName('failing alias test').skipped).to.be.false; + // annotations work for aliases nested inside an alias suite + expect(byName('fixme test inside alias suite').skipped).to.be.true; + expect(byName('fixme test inside alias suite').suites).to.deep.equal(['alias suite']); + }); + + it('should not parse custom alias annotations when the alias is not configured', () => { + source = fs.readFileSync('./example/playwright/custom-fixture-annotations.ts').toString(); + ast = jsParser.parse(source, { sourceType: 'unambiguous', plugins: ['typescript'] }); + const tests = playwrightParser(ast, '', source); + + expect(tests.length).to.equal(0); + }); + + describe('annotations status (.skip/.fixme/.fail/.todo)', () => { + let tests; + + beforeEach(() => { + source = fs.readFileSync('./example/playwright/annotations-status.ts').toString(); + ast = jsParser.parse(source, { sourceType: 'unambiguous', plugins: ['typescript'] }); + tests = playwrightParser(ast, '', source); + }); + + const byName = name => tests.find(t => t.name === name); + + it('registers every named test exactly once (runtime no-title forms excluded)', () => { + // 5 named tests; inline `test.fail()` / `test.skip()` without a title add nothing + expect(tests.length).to.equal(5); + expect(tests.map(t => t.name)).to.deep.equal([ + 'plain test', + 'expected to fail test', + 'todo test', + 'runtime annotations have no title', + 'fail inside skipped suite', + ]); + }); + + it('marks .todo as skipped', () => { + expect(byName('todo test').skipped).to.be.true; + }); + + it('keeps .fail tests runnable (not skipped)', () => { + expect(byName('expected to fail test').skipped).to.be.false; + }); + + it('treats .fail inside a skipped suite as skipped', () => { + const test = byName('fail inside skipped suite'); + expect(test.skipped).to.be.true; + expect(test.suites).to.deep.equal(['skipped suite']); + }); + + it('ignores runtime `test.fail()` / `test.skip()` calls without a title', () => { + const test = byName('runtime annotations have no title'); + expect(test).to.not.be.undefined; + expect(test.skipped).to.be.false; + }); + }); + + it('should not leak a skipped suite onto sibling tests declared after it', () => { + source = fs.readFileSync('./example/playwright/sibling-after-skipped-suite.ts').toString(); + ast = jsParser.parse(source, { sourceType: 'unambiguous', plugins: ['typescript'] }); + const tests = playwrightParser(ast, '', source); + + const byName = name => tests.find(t => t.name === name); + + // nested test inherits the skipped suite + expect(byName('inside skipped suite').skipped).to.be.true; + expect(byName('inside skipped suite').suites).to.deep.equal(['skipped suite']); + // siblings declared after the suite closed must not inherit it (skipped or suite name) + expect(byName('sibling after skipped suite').skipped).to.be.false; + expect(byName('sibling after skipped suite').suites).to.deep.equal([]); + expect(byName('failing sibling after skipped suite').skipped).to.be.false; + expect(byName('failing sibling after skipped suite').suites).to.deep.equal([]); + }); + it('should not crash when test is assigned to a variable or inside an array (regression for issue #1)', () => { const source = ` // This works normally From 72855353261a077203245a405561ebfbfaf5c2c6 Mon Sep 17 00:00:00 2001 From: Oleksandr Pelykh Date: Sat, 13 Jun 2026 09:09:15 +0300 Subject: [PATCH 2/7] add .slow processing; update docs --- README.md | 23 +++++++++++++++++++++++ example/playwright/annotations-status.ts | 6 ++++++ src/lib/frameworks/playwright.js | 7 ++++--- tests/playwright_test.js | 13 +++++++++---- 4 files changed, 42 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 130cf961..0649522a 100644 --- a/README.md +++ b/README.md @@ -855,6 +855,29 @@ Test aliases are used to map tests in source code to tests in Testomat.io. By de TESTOMATIO=11111111 npx check-tests Playwright "**/*{.,_}{test,spec}.ts" --test-alias myTest,myCustomFunction ``` +For Playwright, aliases are also recognized on test annotations, so `myTest.skip(...)`, `myTest.fixme(...)`, `myTest.fail(...)`, `myTest.slow(...)` and `myTest.todo(...)` are parsed the same way as the built-in `test`/`it`: + +```js +import { test as base } from '@playwright/test'; + +const myTest = base.extend({ + /* ... */ +}); + +myTest.skip('skipped alias test', async () => { + /* ... */ +}); +myTest.fixme('fixme alias test', async () => { + /* ... */ +}); +``` + +> **Important:** annotated tests declared on a custom object (e.g. `myTest.skip(...)`, `myTest.fail(...)`, `myTest.fixme(...)`, `myTest.slow(...)`) are **only** parsed when that object is passed via `--test-alias`. Without it, only the built-in `test`/`it` annotations are detected and these tests are silently skipped: +> +> ``` +> TESTOMATIO={token} npx check-tests Playwright "**/*{.,_}{test,spec}.ts" --test-alias myTest,myTest2 +> ``` + ## Programmatic API Import Analyzer from module: diff --git a/example/playwright/annotations-status.ts b/example/playwright/annotations-status.ts index 01cabe55..fc01c53b 100644 --- a/example/playwright/annotations-status.ts +++ b/example/playwright/annotations-status.ts @@ -9,6 +9,11 @@ test.fail('expected to fail test', async () => { await expect(true).toBe(false); }); +// .slow triples the timeout, but the test still runs => not skipped +test.slow('slow test', async () => { + await expect(true).toBe(true); +}); + // .todo => skipped test test.todo('todo test'); @@ -16,6 +21,7 @@ test.todo('todo test'); test('runtime annotations have no title', async () => { test.fail(); test.skip(); + test.slow(); await expect(true).toBe(true); }); diff --git a/src/lib/frameworks/playwright.js b/src/lib/frameworks/playwright.js index 0f9d0262..63d3144f 100644 --- a/src/lib/frameworks/playwright.js +++ b/src/lib/frameworks/playwright.js @@ -39,7 +39,7 @@ module.exports = (ast, file = '', source = '', opts = {}) => { return currentSuite.filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)); } - // resolve the name of the test object an annotation (`.skip`, `.fixme`, `.fail`, `.todo`) + // resolve the name of the test object an annotation (`.skip`, `.fixme`, `.fail`, `.slow`) // is called on, e.g. `test`, `it`, `describe` or a custom test alias / fixture name function getTestObjectName(path) { if (!path.parent || !path.parent.object) return null; @@ -150,8 +150,9 @@ module.exports = (ast, file = '', source = '', opts = {}) => { // todo: handle "context" } - // `.fail` marks a test as expected to fail; it still runs, so it is not skipped - if (path.isIdentifier({ name: 'fail' })) { + // `.fail` (expected to fail) and `.slow` (extended timeout) still run, so they are not + // skipped on their own — only inherited skip from an enclosing suite applies + if (path.isIdentifier({ name: 'fail' }) || path.isIdentifier({ name: 'slow' })) { const name = getTestObjectName(path); if (!name) return; diff --git a/tests/playwright_test.js b/tests/playwright_test.js index 474aa5e4..728d8b67 100644 --- a/tests/playwright_test.js +++ b/tests/playwright_test.js @@ -537,7 +537,7 @@ test.describe.only('my test', () => { expect(tests.length).to.equal(0); }); - describe('annotations status (.skip/.fixme/.fail/.todo)', () => { + describe('annotations status (.skip/.fixme/.fail/.slow/.todo)', () => { let tests; beforeEach(() => { @@ -549,11 +549,12 @@ test.describe.only('my test', () => { const byName = name => tests.find(t => t.name === name); it('registers every named test exactly once (runtime no-title forms excluded)', () => { - // 5 named tests; inline `test.fail()` / `test.skip()` without a title add nothing - expect(tests.length).to.equal(5); + // 6 named tests; inline `test.fail()` / `test.skip()` / `test.slow()` without a title add nothing + expect(tests.length).to.equal(6); expect(tests.map(t => t.name)).to.deep.equal([ 'plain test', 'expected to fail test', + 'slow test', 'todo test', 'runtime annotations have no title', 'fail inside skipped suite', @@ -568,13 +569,17 @@ test.describe.only('my test', () => { expect(byName('expected to fail test').skipped).to.be.false; }); + it('keeps .slow tests runnable (not skipped)', () => { + expect(byName('slow test').skipped).to.be.false; + }); + it('treats .fail inside a skipped suite as skipped', () => { const test = byName('fail inside skipped suite'); expect(test.skipped).to.be.true; expect(test.suites).to.deep.equal(['skipped suite']); }); - it('ignores runtime `test.fail()` / `test.skip()` calls without a title', () => { + it('ignores runtime `test.fail()` / `test.skip()` / `test.slow()` calls without a title', () => { const test = byName('runtime annotations have no title'); expect(test).to.not.be.undefined; expect(test.skipped).to.be.false; From fc4de376402afb2a730b44290142985c2b4f4039 Mon Sep 17 00:00:00 2001 From: Oleksandr Pelykh Date: Sat, 4 Jul 2026 08:49:44 +0300 Subject: [PATCH 3/7] refactor; update docs --- README.md | 31 ++++---- example/playwright/annotations-status.ts | 3 - src/analyzer.js | 18 +---- src/lib/frameworks/playwright.js | 92 ++++++++---------------- tests/analyzer_test.js | 27 ------- tests/playwright_test.js | 13 ++-- 6 files changed, 52 insertions(+), 132 deletions(-) diff --git a/README.md b/README.md index 0649522a..0d82ff50 100644 --- a/README.md +++ b/README.md @@ -855,28 +855,31 @@ Test aliases are used to map tests in source code to tests in Testomat.io. By de TESTOMATIO=11111111 npx check-tests Playwright "**/*{.,_}{test,spec}.ts" --test-alias myTest,myCustomFunction ``` -For Playwright, aliases are also recognized on test annotations, so `myTest.skip(...)`, `myTest.fixme(...)`, `myTest.fail(...)`, `myTest.slow(...)` and `myTest.todo(...)` are parsed the same way as the built-in `test`/`it`: +For Playwright, aliases are also recognized on test annotations: `myTest.skip()`, `myTest.fixme()`, `myTest.fail()` and `myTest.slow()` are parsed the same way as annotations on the built-in `test` / `it`. Annotated tests defined on a custom test object (fixture) are only detected when its name is passed via `--test-alias`. -```js -import { test as base } from '@playwright/test'; +Example of what you may have in your code: + +```ts +import { myFixture } from './fixtures'; -const myTest = base.extend({ - /* ... */ +myFixture('regular test', async () => { + // ... }); -myTest.skip('skipped alias test', async () => { - /* ... */ +myFixture.skip('skipped test', async () => { + // ... }); -myTest.fixme('fixme alias test', async () => { - /* ... */ + +myFixture.fixme('broken test', async () => { + // ... }); ``` -> **Important:** annotated tests declared on a custom object (e.g. `myTest.skip(...)`, `myTest.fail(...)`, `myTest.fixme(...)`, `myTest.slow(...)`) are **only** parsed when that object is passed via `--test-alias`. Without it, only the built-in `test`/`it` annotations are detected and these tests are silently skipped: -> -> ``` -> TESTOMATIO={token} npx check-tests Playwright "**/*{.,_}{test,spec}.ts" --test-alias myTest,myTest2 -> ``` +To import tests defined on a custom test object (fixture), pass its name via `--test-alias` option: + +``` +TESTOMATIO={API_KEY} npx check-tests Playwright "**/*{.,_}{test,spec}.ts" --test-alias myFixture +``` ## Programmatic API diff --git a/example/playwright/annotations-status.ts b/example/playwright/annotations-status.ts index fc01c53b..3e4e7212 100644 --- a/example/playwright/annotations-status.ts +++ b/example/playwright/annotations-status.ts @@ -14,9 +14,6 @@ test.slow('slow test', async () => { await expect(true).toBe(true); }); -// .todo => skipped test -test.todo('todo test'); - // runtime forms without a title declare no separate test test('runtime annotations have no title', async () => { test.fail(); diff --git a/src/analyzer.js b/src/analyzer.js index 19d8cfbe..965296e5 100644 --- a/src/analyzer.js +++ b/src/analyzer.js @@ -83,20 +83,6 @@ class Analyzer { // this.addPlugin('@babel/plugin-transform-typescript'); } - // Build the list of glob patterns to scan. When TypeScript support is enabled, a JS-only - // glob (e.g. "**/*.test.js") would silently match nothing in a TS project, so we also scan - // the TypeScript equivalents by swapping the trailing `.js` extension for `.ts`/`.tsx`/etc. - buildPatterns(pattern) { - const patterns = [pattern]; - if (this.typeScript && /\.js$/.test(pattern)) { - const base = pattern.replace(/\.js$/, ''); - for (const ext of ['ts', 'tsx', 'mts', 'cts']) { - patterns.push(`${base}.${ext}`); - } - } - return patterns; - } - analyze(pattern) { if (!this.frameworkParser) throw new Error("No test framework specified. Can't analyze"); @@ -107,9 +93,7 @@ class Analyzer { const originalCwd = process.cwd(); process.chdir(this.workDir); - const patterns = this.buildPatterns(pattern); - debug('Patterns:', patterns); - let files = [...new Set(patterns.flatMap(p => glob.sync(p, { windowsPathsNoEscape: true })))]; + let files = glob.sync(pattern, { windowsPathsNoEscape: true }); // Exclude files matching the exclude pattern if provided if (this.opts.exclude) { diff --git a/src/lib/frameworks/playwright.js b/src/lib/frameworks/playwright.js index 63d3144f..cfd403fe 100644 --- a/src/lib/frameworks/playwright.js +++ b/src/lib/frameworks/playwright.js @@ -23,7 +23,7 @@ module.exports = (ast, file = '', source = '', opts = {}) => { let beforeEachCode = ''; let afterCode = ''; - // valid test identifiers: built-in `test`/`it` plus any custom fixtures/aliases + // built-in `test`/`it` plus any custom fixtures/aliases passed via --test-alias const testNames = ['test', 'it', ...(opts?.testAlias || [])]; function addSuite(path) { @@ -32,40 +32,31 @@ module.exports = (ast, file = '', source = '', opts = {}) => { currentSuite.push(path); } - // suites that actually enclose the call at `path`. `currentSuite` is only pruned when a - // new suite is added, so it can still hold sibling suites that already closed above this - // line — those must not leak their name or `skipped` flag onto a test declared after them. - function getEnclosingSuites(path) { + // suites that enclose the call at `path` (ignoring sibling suites already closed above it) + function getSuites(path) { return currentSuite.filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)); } - // resolve the name of the test object an annotation (`.skip`, `.fixme`, `.fail`, `.slow`) - // is called on, e.g. `test`, `it`, `describe` or a custom test alias / fixture name - function getTestObjectName(path) { + // name of the object an annotation is called on, e.g. `test`/`it`/`describe` or a custom alias + function getAnnotatedObjectName(path) { if (!path.parent || !path.parent.object) return null; - return ( - path.parent.object.name || path.parent.object.property?.name || path.parent.object.callee?.object?.name || null - ); + return path.parent.object.name || path.parent.object.property?.name || path.parent.object.callee?.object?.name; } - // Register a single named test declared with an annotation call - // (`test.skip` / `test.fixme` / `test.fail` / `test.todo`, or the alias equivalents). - // `path` is the annotation identifier node; its enclosing call holds the test title. - // Calls without a string title (the runtime form `test.skip()` used inside a test body) - // declare no test and are ignored. The caller decides `skipped`. - function registerAnnotatedTest(path, { skipped }) { + // register a test declared with an annotation (`.skip`/`.fixme`/`.fail`/`.slow`/`.todo`); + // runtime forms without a title (`test.skip()` inside a body) declare no test and are ignored + function addAnnotatedTest(path, skipped) { if (!hasStringOrTemplateArgument(path.parentPath.container)) return; + const suites = getSuites(path); tests.push({ name: getStringValue(path.parentPath.container), - suites: getEnclosingSuites(path).map(s => getStringValue(s)), + suites: suites.map(s => getStringValue(s)), line: getLineNumber(path), - // `path` is the annotation identifier (`fixme`/`skip`/...); its container ends on the - // member-expression line only. The full call (and its body) is `path.parentPath.container`, - // so take the end line from there to capture the complete test code. + // end line comes from the enclosing call (`path` is just the annotation identifier) to capture the full body code: getCode(source, getLineNumber(path), getEndLineNumber(path.parentPath), isLineNumber), file, - skipped, + skipped: skipped || suites.some(s => s.skipped), }); } @@ -130,49 +121,28 @@ module.exports = (ast, file = '', source = '', opts = {}) => { } } - // `.skip` / `.fixme` mark a test (or every test in a suite) as skipped, - // supporting `test`, `it`, `describe` and any custom test alias / fixture - if (path.isIdentifier({ name: 'skip' }) || path.isIdentifier({ name: 'fixme' })) { - const name = getTestObjectName(path); + // `.skip`/`.fixme` skip the test (or whole suite); `.fail`/`.slow` still run but inherit + // a skip from an enclosing suite + if (['skip', 'fixme', 'fail', 'slow'].includes(path.node.name)) { + const name = getAnnotatedObjectName(path); if (!name) return; if (testNames.includes(name)) { - // test or it (or alias), e.g. `myFixture.fixme('...', ...)` - registerAnnotatedTest(path, { skipped: true }); - } else if (name === 'describe') { - // suite + addAnnotatedTest(path, path.node.name === 'skip' || path.node.name === 'fixme'); + } else if ((path.node.name === 'skip' || path.node.name === 'fixme') && name === 'describe') { if (!hasStringOrTemplateArgument(path.parentPath.container)) return; const suite = path.parentPath.container; suite.skipped = true; addSuite(suite); } - - // todo: handle "context" - } - - // `.fail` (expected to fail) and `.slow` (extended timeout) still run, so they are not - // skipped on their own — only inherited skip from an enclosing suite applies - if (path.isIdentifier({ name: 'fail' }) || path.isIdentifier({ name: 'slow' })) { - const name = getTestObjectName(path); - if (!name) return; - - if (testNames.includes(name)) { - registerAnnotatedTest(path, { skipped: getEnclosingSuites(path).some(s => s.skipped) }); - } } + // `.todo` tests are always skipped if (path.isIdentifier({ name: 'todo' })) { - const name = getTestObjectName(path); - if (!name) return; - - // todo tests => skipped tests - if (testNames.includes(name)) { - registerAnnotatedTest(path, { skipped: true }); - } + if (testNames.includes(getAnnotatedObjectName(path))) addAnnotatedTest(path, true); } - const fixtureNames = [...['test', 'it'], ...(opts?.testAlias || [])]; - for (const fiixtureName of fixtureNames || []) { + for (const fiixtureName of testNames) { if (path.isIdentifier({ name: fiixtureName })) { if (!hasStringOrTemplateArgument(path.parent)) return; @@ -189,19 +159,18 @@ module.exports = (ast, file = '', source = '', opts = {}) => { getCode(source, getLineNumber(path), getEndLineNumber(path), isLineNumber) + afterCode; - const testName = getStringValue(path.parent); - const enclosingSuites = getEnclosingSuites(path); + const suites = getSuites(path); tests.push({ - name: testName, - suites: enclosingSuites.map(s => getStringValue(s)), + name: getStringValue(path.parent), + suites: suites.map(s => getStringValue(s)), updatePoint: getUpdatePoint(path.parent), line: getLineNumber(path), code, file, tags: [...getAllSuiteTags(currentSuite), ...playwright.getTestProps(path.parentPath).tags], annotations: playwright.getTestProps(path.parentPath).annotations, - skipped: enclosingSuites.some(s => s.skipped), + skipped: suites.some(s => s.skipped), }); // stop the loop if the test is found @@ -213,16 +182,15 @@ module.exports = (ast, file = '', source = '', opts = {}) => { const currentPath = path.parentPath.parentPath; if (!hasStringOrTemplateArgument(currentPath.parent)) return; - const testName = getStringValue(currentPath.parent); - const enclosingSuites = getEnclosingSuites(path); + const suites = getSuites(path); tests.push({ - name: testName, - suites: enclosingSuites.map(s => getStringValue(s)), + name: getStringValue(currentPath.parent), + suites: suites.map(s => getStringValue(s)), updatePoint: getUpdatePoint(path.parent), line: getLineNumber(currentPath), code: getCode(source, getLineNumber(currentPath), getEndLineNumber(currentPath), isLineNumber), file, - skipped: enclosingSuites.some(s => s.skipped), + skipped: suites.some(s => s.skipped), }); } }, diff --git a/tests/analyzer_test.js b/tests/analyzer_test.js index 06ae8fc7..31fc7a07 100644 --- a/tests/analyzer_test.js +++ b/tests/analyzer_test.js @@ -42,33 +42,6 @@ describe('analyzer', () => { expect(decorator.getSuiteNames()).to.include('Login - Global Header: Institutional Sign In Modal'); }); - it('should also scan TypeScript files when given a JS-only glob and TypeScript is enabled', () => { - analyzer = new Analyzer('mocha', path.join(__dirname, '..')); - analyzer.withTypeScript(); - // a `.js` pattern would match nothing in this TS-only dir; buildPatterns adds the `.ts` variant - analyzer.analyze('./example/protractor/**.js'); - const decorator = analyzer.getDecorator(); - expect(decorator.getSuiteNames()).to.include('Login - Global Header: Institutional Sign In Modal'); - }); - - it('buildPatterns expands a trailing .js extension to TS variants only when TypeScript is enabled', () => { - analyzer = new Analyzer('mocha', path.join(__dirname, '..')); - - // without TypeScript the pattern is left untouched - expect(analyzer.buildPatterns('./example/foo/**.js')).to.deep.equal(['./example/foo/**.js']); - // non-.js patterns are never expanded, even with TypeScript on - analyzer.withTypeScript(); - expect(analyzer.buildPatterns('./example/foo/**.ts')).to.deep.equal(['./example/foo/**.ts']); - // .js patterns gain the TS equivalents - expect(analyzer.buildPatterns('./example/foo/**.js')).to.deep.equal([ - './example/foo/**.js', - './example/foo/**.ts', - './example/foo/**.tsx', - './example/foo/**.mts', - './example/foo/**.cts', - ]); - }); - it('should exclude dir in file name if dir specified', () => { analyzer = new Analyzer('mocha', 'example'); analyzer.analyze('mocha/**_test.js'); diff --git a/tests/playwright_test.js b/tests/playwright_test.js index 728d8b67..92daf216 100644 --- a/tests/playwright_test.js +++ b/tests/playwright_test.js @@ -250,7 +250,7 @@ test.describe.only('my test', () => { expect(tests[1].name).to.equal('my skip test @first'); expect(tests[1].suites.length).to.eql(1); - // code captures the full test body (signature + assertions), not just the signature line + // code captures the full test body, not just the signature line expect(tests[1].code).to.include("test.skip('my skip test @first', async ({ page }) => {"); expect(tests[1].code).to.include("await expect(page).toHaveURL('https://my.start.url/');"); }); @@ -262,7 +262,7 @@ test.describe.only('my test', () => { expect(tests[2].name).to.equal('my fixme test @third'); expect(tests[2].suites.length).to.eql(1); - // code captures the full test body (signature + assertions), not just the signature line + // code captures the full test body, not just the signature line expect(tests[2].code).to.include("test.fixme('my fixme test @third', async ({ page }) => {"); expect(tests[2].code).to.include("await expect(page).toHaveURL('https://my.start.url/');"); }); @@ -517,14 +517,10 @@ test.describe.only('my test', () => { const byName = name => tests.find(t => t.name === name); expect(tests.length).to.equal(5); - expect(byName('plain alias test').skipped).to.be.false; - // .skip and .fixme on the alias mark the test as skipped expect(byName('skipped alias test').skipped).to.be.true; expect(byName('fixme alias test').skipped).to.be.true; - // .fail still runs, so it is not skipped - expect(byName('failing alias test').skipped).to.be.false; - // annotations work for aliases nested inside an alias suite + expect(byName('failing alias test').skipped).to.be.false; // .fail still runs expect(byName('fixme test inside alias suite').skipped).to.be.true; expect(byName('fixme test inside alias suite').suites).to.deep.equal(['alias suite']); }); @@ -593,10 +589,9 @@ test.describe.only('my test', () => { const byName = name => tests.find(t => t.name === name); - // nested test inherits the skipped suite expect(byName('inside skipped suite').skipped).to.be.true; expect(byName('inside skipped suite').suites).to.deep.equal(['skipped suite']); - // siblings declared after the suite closed must not inherit it (skipped or suite name) + // siblings declared after the suite closed must not inherit it expect(byName('sibling after skipped suite').skipped).to.be.false; expect(byName('sibling after skipped suite').suites).to.deep.equal([]); expect(byName('failing sibling after skipped suite').skipped).to.be.false; From d09d231a038bf819e73f3d90313b459c2de2e58f Mon Sep 17 00:00:00 2001 From: Oleksandr Pelykh Date: Sat, 4 Jul 2026 08:53:28 +0300 Subject: [PATCH 4/7] upd test --- README.md | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 0d82ff50..c75e4ea1 100644 --- a/README.md +++ b/README.md @@ -849,36 +849,10 @@ test('resource management', () => { ## Test aliases -Test aliases are used to map tests in source code to tests in Testomat.io. By default `test` and `it` are parsed. But if you rename them or use another function to define tests (e.g. created/extended test object in Playwright), you can add alias (or multiple aliases, separated by comma) via `--test-alias` option: +Test aliases (`test.skip()`, `test.fixme()`, `test.fail()`, `test.slow()`) are used to map tests in source code to tests in Testomat.io. By default `test` and `it` are parsed. But if you rename them or use another function to define tests (e.g. created/extended test object in Playwright), you can add alias (or multiple aliases, separated by comma) via `--test-alias` option: ``` -TESTOMATIO=11111111 npx check-tests Playwright "**/*{.,_}{test,spec}.ts" --test-alias myTest,myCustomFunction -``` - -For Playwright, aliases are also recognized on test annotations: `myTest.skip()`, `myTest.fixme()`, `myTest.fail()` and `myTest.slow()` are parsed the same way as annotations on the built-in `test` / `it`. Annotated tests defined on a custom test object (fixture) are only detected when its name is passed via `--test-alias`. - -Example of what you may have in your code: - -```ts -import { myFixture } from './fixtures'; - -myFixture('regular test', async () => { - // ... -}); - -myFixture.skip('skipped test', async () => { - // ... -}); - -myFixture.fixme('broken test', async () => { - // ... -}); -``` - -To import tests defined on a custom test object (fixture), pass its name via `--test-alias` option: - -``` -TESTOMATIO={API_KEY} npx check-tests Playwright "**/*{.,_}{test,spec}.ts" --test-alias myFixture +TESTOMATIO={API_KEY} npx check-tests Playwright "**/*{.,_}{test,spec}.ts" --test-alias myTest,myFixture ``` ## Programmatic API From ff69bb724b42d76762881206dcf29a8bd6132399 Mon Sep 17 00:00:00 2001 From: Oleksandr Pelykh Date: Sat, 4 Jul 2026 09:07:24 +0300 Subject: [PATCH 5/7] refactor --- src/lib/frameworks/playwright.js | 77 ++++++++++++++------------------ 1 file changed, 34 insertions(+), 43 deletions(-) diff --git a/src/lib/frameworks/playwright.js b/src/lib/frameworks/playwright.js index cfd403fe..602923c3 100644 --- a/src/lib/frameworks/playwright.js +++ b/src/lib/frameworks/playwright.js @@ -32,34 +32,6 @@ module.exports = (ast, file = '', source = '', opts = {}) => { currentSuite.push(path); } - // suites that enclose the call at `path` (ignoring sibling suites already closed above it) - function getSuites(path) { - return currentSuite.filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)); - } - - // name of the object an annotation is called on, e.g. `test`/`it`/`describe` or a custom alias - function getAnnotatedObjectName(path) { - if (!path.parent || !path.parent.object) return null; - return path.parent.object.name || path.parent.object.property?.name || path.parent.object.callee?.object?.name; - } - - // register a test declared with an annotation (`.skip`/`.fixme`/`.fail`/`.slow`/`.todo`); - // runtime forms without a title (`test.skip()` inside a body) declare no test and are ignored - function addAnnotatedTest(path, skipped) { - if (!hasStringOrTemplateArgument(path.parentPath.container)) return; - - const suites = getSuites(path); - tests.push({ - name: getStringValue(path.parentPath.container), - suites: suites.map(s => getStringValue(s)), - line: getLineNumber(path), - // end line comes from the enclosing call (`path` is just the annotation identifier) to capture the full body - code: getCode(source, getLineNumber(path), getEndLineNumber(path.parentPath), isLineNumber), - file, - skipped: skipped || suites.some(s => s.skipped), - }); - } - traverse(ast, { enter(path) { if (path.isIdentifier({ name: 'describe' })) { @@ -121,25 +93,41 @@ module.exports = (ast, file = '', source = '', opts = {}) => { } } - // `.skip`/`.fixme` skip the test (or whole suite); `.fail`/`.slow` still run but inherit - // a skip from an enclosing suite - if (['skip', 'fixme', 'fail', 'slow'].includes(path.node.name)) { - const name = getAnnotatedObjectName(path); - if (!name) return; + // `.skip`/`.fixme`/`.todo` tests are skipped; `.fail`/`.slow` tests still run; + // runtime forms without a title (e.g. `test.skip()` inside a body) declare no test + if (path.isIdentifier() && ['skip', 'fixme', 'fail', 'slow', 'todo'].includes(path.node.name)) { + if (!path.parent || !path.parent.object) { + return; + } + const name = + path.parent.object.name || path.parent.object.property?.name || path.parent.object.callee?.object?.name; if (testNames.includes(name)) { - addAnnotatedTest(path, path.node.name === 'skip' || path.node.name === 'fixme'); - } else if ((path.node.name === 'skip' || path.node.name === 'fixme') && name === 'describe') { + // test or it + if (!hasStringOrTemplateArgument(path.parentPath.container)) return; + + const testName = getStringValue(path.parentPath.container); + const suites = currentSuite.filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)); + tests.push({ + name: testName, + suites: suites.map(s => getStringValue(s)), + line: getLineNumber(path), + // end line comes from the enclosing call to capture the full test body + code: getCode(source, getLineNumber(path), getEndLineNumber(path.parentPath), isLineNumber), + file, + skipped: ['skip', 'fixme', 'todo'].includes(path.node.name) || suites.some(s => s.skipped), + }); + } + + if (name === 'describe' && (path.node.name === 'skip' || path.node.name === 'fixme')) { + // suite if (!hasStringOrTemplateArgument(path.parentPath.container)) return; const suite = path.parentPath.container; suite.skipped = true; addSuite(suite); } - } - // `.todo` tests are always skipped - if (path.isIdentifier({ name: 'todo' })) { - if (testNames.includes(getAnnotatedObjectName(path))) addAnnotatedTest(path, true); + // todo: handle "context" } for (const fiixtureName of testNames) { @@ -159,10 +147,11 @@ module.exports = (ast, file = '', source = '', opts = {}) => { getCode(source, getLineNumber(path), getEndLineNumber(path), isLineNumber) + afterCode; - const suites = getSuites(path); + const testName = getStringValue(path.parent); + const suites = currentSuite.filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)); tests.push({ - name: getStringValue(path.parent), + name: testName, suites: suites.map(s => getStringValue(s)), updatePoint: getUpdatePoint(path.parent), line: getLineNumber(path), @@ -170,6 +159,7 @@ module.exports = (ast, file = '', source = '', opts = {}) => { file, tags: [...getAllSuiteTags(currentSuite), ...playwright.getTestProps(path.parentPath).tags], annotations: playwright.getTestProps(path.parentPath).annotations, + // only suites still enclosing this line can mark it skipped (not closed siblings) skipped: suites.some(s => s.skipped), }); @@ -182,9 +172,10 @@ module.exports = (ast, file = '', source = '', opts = {}) => { const currentPath = path.parentPath.parentPath; if (!hasStringOrTemplateArgument(currentPath.parent)) return; - const suites = getSuites(path); + const testName = getStringValue(currentPath.parent); + const suites = currentSuite.filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path)); tests.push({ - name: getStringValue(currentPath.parent), + name: testName, suites: suites.map(s => getStringValue(s)), updatePoint: getUpdatePoint(path.parent), line: getLineNumber(currentPath), From 545ba61edda23a04d65bee4913a4151400390668 Mon Sep 17 00:00:00 2001 From: opelykh Date: Mon, 13 Jul 2026 12:25:49 +0300 Subject: [PATCH 6/7] remove "todo" annotation and fix unit tests --- src/lib/frameworks/playwright.js | 6 +++--- tests/playwright_test.js | 11 +++-------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/lib/frameworks/playwright.js b/src/lib/frameworks/playwright.js index 602923c3..6dfcc182 100644 --- a/src/lib/frameworks/playwright.js +++ b/src/lib/frameworks/playwright.js @@ -93,9 +93,9 @@ module.exports = (ast, file = '', source = '', opts = {}) => { } } - // `.skip`/`.fixme`/`.todo` tests are skipped; `.fail`/`.slow` tests still run; + // `.skip`/`.fixme` tests are skipped; `.fail`/`.slow` tests still run; // runtime forms without a title (e.g. `test.skip()` inside a body) declare no test - if (path.isIdentifier() && ['skip', 'fixme', 'fail', 'slow', 'todo'].includes(path.node.name)) { + if (path.isIdentifier() && ['skip', 'fixme', 'fail', 'slow'].includes(path.node.name)) { if (!path.parent || !path.parent.object) { return; } @@ -115,7 +115,7 @@ module.exports = (ast, file = '', source = '', opts = {}) => { // end line comes from the enclosing call to capture the full test body code: getCode(source, getLineNumber(path), getEndLineNumber(path.parentPath), isLineNumber), file, - skipped: ['skip', 'fixme', 'todo'].includes(path.node.name) || suites.some(s => s.skipped), + skipped: ['skip', 'fixme'].includes(path.node.name) || suites.some(s => s.skipped), }); } diff --git a/tests/playwright_test.js b/tests/playwright_test.js index 92daf216..a734b9ec 100644 --- a/tests/playwright_test.js +++ b/tests/playwright_test.js @@ -533,7 +533,7 @@ test.describe.only('my test', () => { expect(tests.length).to.equal(0); }); - describe('annotations status (.skip/.fixme/.fail/.slow/.todo)', () => { + describe('annotations status (.skip/.fixme/.fail/.slow)', () => { let tests; beforeEach(() => { @@ -545,22 +545,17 @@ test.describe.only('my test', () => { const byName = name => tests.find(t => t.name === name); it('registers every named test exactly once (runtime no-title forms excluded)', () => { - // 6 named tests; inline `test.fail()` / `test.skip()` / `test.slow()` without a title add nothing - expect(tests.length).to.equal(6); + // 5 named tests; inline `test.fail()` / `test.skip()` / `test.slow()` without a title add nothing + expect(tests.length).to.equal(5); expect(tests.map(t => t.name)).to.deep.equal([ 'plain test', 'expected to fail test', 'slow test', - 'todo test', 'runtime annotations have no title', 'fail inside skipped suite', ]); }); - it('marks .todo as skipped', () => { - expect(byName('todo test').skipped).to.be.true; - }); - it('keeps .fail tests runnable (not skipped)', () => { expect(byName('expected to fail test').skipped).to.be.false; }); From f37a464b49f50c10164fe6d25f4356a643725ad5 Mon Sep 17 00:00:00 2001 From: opelykh Date: Wed, 15 Jul 2026 10:22:26 +0300 Subject: [PATCH 7/7] process cypress tags --- example/mocha/cypress_tags_spec.js | 46 ++++++++++++++++++++++++++++++ src/lib/frameworks/mocha.js | 6 ++++ src/lib/utils.js | 33 +++++++++++++++++++++ tests/mocha_test.js | 36 +++++++++++++++++++++++ 4 files changed, 121 insertions(+) create mode 100644 example/mocha/cypress_tags_spec.js diff --git a/example/mocha/cypress_tags_spec.js b/example/mocha/cypress_tags_spec.js new file mode 100644 index 00000000..cda29045 --- /dev/null +++ b/example/mocha/cypress_tags_spec.js @@ -0,0 +1,46 @@ +/// + +const TestTypes = { + regression: '@regression', + smoke: '@smoke', +}; + +const Services = { + integrations: 'integrations', +}; + +describe( + 'Ability to Edit and Delete Assets', + { + tags: [TestTypes.regression, Services.integrations, '@multiTenant', 'v8'], + }, + () => { + beforeEach(() => { + cy.visit('http://localhost:8080/assets'); + }); + + it('edits an asset', () => { + cy.get('.edit').click(); + }); + + it('deletes an asset', { tags: ['@slow', `nightly`] }, () => { + cy.get('.delete').click(); + }); + + describe('nested suite', { tags: 'wip' }, () => { + it('inherits tags from all parent suites', () => { + cy.get('.nested').click(); + }); + }); + }, +); + +describe('Suite without tags', () => { + it('has no tags', () => { + cy.visit('http://localhost:8080'); + }); + + it.skip('skipped test with own tags', { tags: '@quarantine' }, () => { + cy.visit('http://localhost:8080'); + }); +}); diff --git a/src/lib/frameworks/mocha.js b/src/lib/frameworks/mocha.js index 67cba213..536c1172 100644 --- a/src/lib/frameworks/mocha.js +++ b/src/lib/frameworks/mocha.js @@ -7,6 +7,8 @@ const { getLineNumber, getEndLineNumber, getCode, + cypress, + getAllSuiteTags, } = require('../utils'); module.exports = (ast, file = '', source = '', opts = {}) => { @@ -23,6 +25,7 @@ module.exports = (ast, file = '', source = '', opts = {}) => { function addSuite(path) { currentSuite = currentSuite.filter(s => s.loc.end.line > path.loc.start.line); + path.tags = cypress.getTags(path); currentSuite.push(path); } @@ -94,6 +97,7 @@ module.exports = (ast, file = '', source = '', opts = {}) => { line: getLineNumber(path), code: getCode(source, getLineNumber(path), getEndLineNumber(path), isLineNumber), file, + tags: [...getAllSuiteTags(currentSuite), ...cypress.getTags(path.parentPath.container)], skipped: true, }); } @@ -117,6 +121,7 @@ module.exports = (ast, file = '', source = '', opts = {}) => { updatePoint: getUpdatePoint(path.parent), line: getLineNumber(path), code: getCode(source, getLineNumber(path), getEndLineNumber(path), isLineNumber), + tags: [...getAllSuiteTags(currentSuite), ...cypress.getTags(path.parent)], skipped: true, file, }); @@ -147,6 +152,7 @@ module.exports = (ast, file = '', source = '', opts = {}) => { line: getLineNumber(path), code, file, + tags: [...getAllSuiteTags(currentSuite), ...cypress.getTags(path.parent)], skipped: !!currentSuite.filter(s => s.skipped).length, }); } diff --git a/src/lib/utils.js b/src/lib/utils.js index 4c6d3e1f..461d18a3 100644 --- a/src/lib/utils.js +++ b/src/lib/utils.js @@ -196,6 +196,38 @@ const playwright = { }, }; +const cypress = { + // extracts tags from Cypress test config object: describe('title', { tags: [...] }, fn) or it('title', { tags: '@smoke' }, fn) + getTags: node => { + const args = node?.arguments; + if (!args?.length) return []; + const configArg = args.find(arg => arg.type === 'ObjectExpression'); + if (!configArg) return []; + + const tagsProp = configArg.properties.find(prop => prop.key?.name === 'tags' || prop.key?.value === 'tags'); + if (!tagsProp || !tagsProp.value) return []; + + // tags value could be a single tag or an array of tags + const elements = tagsProp.value.type === 'ArrayExpression' ? tagsProp.value.elements : [tagsProp.value]; + + const getTagName = el => { + if (!el) return; + if (el.type === 'StringLiteral' || el.type === 'Literal') return el.value; + if (el.type === 'TemplateLiteral' && !el.expressions.length && el.quasis.length === 1) { + return el.quasis[0].value.cooked; + } + // enum-style tags like TestTypes.regression can't be resolved statically; use the property name + if (el.type === 'MemberExpression' && el.property?.type === 'Identifier') return el.property.name; + if (el.type === 'Identifier') return el.name; + }; + + return elements + .map(getTagName) + .filter(tag => typeof tag === 'string' && tag.length) + .map(tag => (tag.startsWith('@') ? tag.substring(1) : tag)); + }, +}; + const arrayCompare = function (a, b, id) { const missing = []; const found = []; @@ -276,6 +308,7 @@ module.exports = { replaceAtPoint, cleanAtPoint, playwright, + cypress, arrayCompare, getAllSuiteTags, }; diff --git a/tests/mocha_test.js b/tests/mocha_test.js index 84fd774e..e6a68c18 100644 --- a/tests/mocha_test.js +++ b/tests/mocha_test.js @@ -51,6 +51,42 @@ describe('mocha parser', () => { }); }); + context('cypress tags', () => { + let tests; + + before(() => { + source = fs.readFileSync('./example/mocha/cypress_tags_spec.js').toString(); + ast = parser.parse(source); + tests = mochaParser(ast, '', source); + }); + + it('should sync tags from describe config to tests', () => { + const test = tests.find(t => t.name === 'edits an asset'); + expect(test.tags).to.eql(['regression', 'integrations', 'multiTenant', 'v8']); + }); + + it('should combine suite tags with test tags', () => { + const test = tests.find(t => t.name === 'deletes an asset'); + expect(test.tags).to.eql(['regression', 'integrations', 'multiTenant', 'v8', 'slow', 'nightly']); + }); + + it('should inherit tags from all parent suites', () => { + const test = tests.find(t => t.name === 'inherits tags from all parent suites'); + expect(test.tags).to.eql(['regression', 'integrations', 'multiTenant', 'v8', 'wip']); + }); + + it('should not add tags to tests without them', () => { + const test = tests.find(t => t.name === 'has no tags'); + expect(test.tags).to.eql([]); + }); + + it('should parse tags of skipped tests', () => { + const test = tests.find(t => t.name === 'skipped test with own tags'); + expect(test.skipped).to.eql(true); + expect(test.tags).to.eql(['quarantine']); + }); + }); + context('graphql tests', () => { before(() => { source = fs.readFileSync('./example/mocha/graphql_test.js').toString();