diff --git a/examples/arithmetics/benchmark_test.go b/examples/arithmetics/benchmark_test.go index e2d8ea45..2d2f0730 100644 --- a/examples/arithmetics/benchmark_test.go +++ b/examples/arithmetics/benchmark_test.go @@ -30,7 +30,8 @@ func BenchmarkParser(b *testing.B) { srv := CreateServices() lexerService := service.MustGet[lexer.Lexer](srv) parserService := service.MustGet[parser.Parser](srv) - tokens := lexerService.Exec(content).Tokens + lexerResult, _ := lexerService.Exec(b.Context(), content) + tokens := lexerResult.Tokens doc, err := fastbelt.NewDocumentFromString("file:///workspace/bench.calc", "arithmetics", content) if err != nil { b.Fatal(err) @@ -39,7 +40,7 @@ func BenchmarkParser(b *testing.B) { b.SetBytes(int64(len(content))) b.ResetTimer() for b.Loop() { - result := parserService.Parse(doc) + result, _ := parserService.Parse(b.Context(), doc) doc.Root = result.Node } } @@ -57,8 +58,9 @@ func BenchmarkLexerAndParser(b *testing.B) { b.SetBytes(int64(len(content))) b.ResetTimer() for b.Loop() { - doc.Tokens = lexerService.Exec(content).Tokens - result := parserService.Parse(doc) + lexerResult, _ := lexerService.Exec(b.Context(), content) + doc.Tokens = lexerResult.Tokens + result, _ := parserService.Parse(b.Context(), doc) doc.Root = result.Node } } diff --git a/examples/arithmetics/completion_gen.go b/examples/arithmetics/completion_gen.go index 26a7bbdb..1395eaae 100644 --- a/examples/arithmetics/completion_gen.go +++ b/examples/arithmetics/completion_gen.go @@ -57,8 +57,8 @@ func NewArithmeticsCompletionAdapter(sc *service.Container) *ArithmeticsCompleti return &ArithmeticsCompletionAdapter{sc: sc} } -func (a *ArithmeticsCompletionAdapter) Parse(tokens []core.Token) *parser.CompletionParseResult { - return NewCompletionParser(a.sc).Parse(tokens) +func (a *ArithmeticsCompletionAdapter) Parse(ctx context.Context, tokens []core.Token) *parser.CompletionParseResult { + return NewCompletionParser(a.sc).Parse(ctx, tokens) } func (a *ArithmeticsCompletionAdapter) ATN() *parser.RuntimeATN { diff --git a/examples/arithmetics/completion_parser_gen.go b/examples/arithmetics/completion_parser_gen.go index 56316c26..e4ca3fa3 100644 --- a/examples/arithmetics/completion_parser_gen.go +++ b/examples/arithmetics/completion_parser_gen.go @@ -3,6 +3,7 @@ package arithmetics import ( + "context" core "typefox.dev/fastbelt" "typefox.dev/fastbelt/parser" "typefox.dev/fastbelt/util/service" @@ -27,12 +28,12 @@ func NewCompletionParser(sc *service.Container) *CompletionParser { // the document's tokens up to the cursor) and returns the recorded // snapshots and rule stack. The completion provider feeds that result into // the ATN simulator. -func (p *CompletionParser) Parse(tokens []core.Token) *parser.CompletionParseResult { +func (p *CompletionParser) Parse(ctx context.Context, tokens []core.Token) *parser.CompletionParseResult { messages := service.MustGet[parser.ErrorMessageProvider](p.sc) recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) lookahead := service.MustGet[ArithmeticsParserLookahead](p.sc) cp := &CompletionParser{sc: p.sc, atn: p.atn, lookahead: lookahead} - cp.state = parser.NewParserState(tokens, p.atn(), recovery, messages) + cp.state = parser.NewParserState(ctx, tokens, p.atn(), recovery, messages) cp.cp = parser.NewCompletionParserState(cp.state) cp.ParseModule() return cp.cp.Result(tokens) diff --git a/examples/arithmetics/parser_gen.go b/examples/arithmetics/parser_gen.go index 9879c5d5..639d70bf 100644 --- a/examples/arithmetics/parser_gen.go +++ b/examples/arithmetics/parser_gen.go @@ -3,6 +3,8 @@ package arithmetics import ( + "context" + core "typefox.dev/fastbelt" "typefox.dev/fastbelt/parser" "typefox.dev/fastbelt/util/service" @@ -15,15 +17,23 @@ type Parser struct { lookahead ArithmeticsParserLookahead } -func (p *Parser) Parse(document *core.Document) *parser.ParseResult { +func (p *Parser) Parse(ctx context.Context, document *core.Document) (*parser.ParseResult, error) { recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) messages := service.MustGet[parser.ErrorMessageProvider](p.sc) referencesConstructor := service.MustGet[ArithmeticsReferencesConstructor](p.sc) lookahead := service.MustGet[ArithmeticsParserLookahead](p.sc) - cp := &Parser{sc: p.sc, referencesConstructor: referencesConstructor, lookahead: lookahead, state: parser.NewParserState(document.Tokens, ATN(), recovery, messages)} + cp := &Parser{ + sc: p.sc, + referencesConstructor: referencesConstructor, + lookahead: lookahead, + state: parser.NewParserState(ctx, document.Tokens, ATN(), recovery, messages), + } result := cp.ParseModule() + if ctx.Err() != nil { + return nil, ctx.Err() + } cp.state.ExpectEndOfInput() - return &parser.ParseResult{Node: result, Errors: cp.state.Errors()} + return &parser.ParseResult{Node: result, Errors: cp.state.Errors()}, nil } func NewParser(sc *service.Container) *Parser { diff --git a/examples/statemachine/benchmark_test.go b/examples/statemachine/benchmark_test.go index bda1da42..8b7c1795 100644 --- a/examples/statemachine/benchmark_test.go +++ b/examples/statemachine/benchmark_test.go @@ -11,7 +11,6 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" "typefox.dev/fastbelt" "typefox.dev/fastbelt/lexer" "typefox.dev/fastbelt/parser" @@ -68,84 +67,6 @@ func BenchmarkWorkspaceCycle(b *testing.B) { b.ReportMetric(msPerResource, "ms/resource") } -func BenchmarkTraverseContentSeq(b *testing.B) { - content, _ := generateStatemachineContent(0) - srv := CreateServices() - documentParser := service.MustGet[workspace.DocumentParser](srv) - doc, err := fastbelt.NewDocumentFromString("file:///workspace/statemachine_0.statemachine", "statemachine", content) - if err != nil { - b.Fatal(err) - } - documentParser.Parse(doc) - - for b.Loop() { - count := 0 - for range fastbelt.AllChildren(doc.Root) { - count++ - } - _ = count - } -} - -// BenchmarkTraverseContentSeqHalf breaks out of the loop after visiting half of the -// nodes. With short-circuiting traversal this should take roughly half the time of -// BenchmarkTraverseContentSeq. -func BenchmarkTraverseContentSeqHalf(b *testing.B) { - content, elementCount := generateStatemachineContent(0) - srv := CreateServices() - documentParser := service.MustGet[workspace.DocumentParser](srv) - doc, err := fastbelt.NewDocumentFromString("file:///workspace/statemachine_0.statemachine", "statemachine", content) - if err != nil { - b.Fatal(err) - } - documentParser.Parse(doc) - half := elementCount / 2 - - for b.Loop() { - count := 0 - for range fastbelt.AllChildren(doc.Root) { - count++ - if count == half { - break - } - } - _ = count - } -} - -func TestAllNodesEquivalence(t *testing.T) { - content, elementCount := generateStatemachineContent(0) - srv := CreateServices() - documentParser := service.MustGet[workspace.DocumentParser](srv) - doc, err := fastbelt.NewDocumentFromString("file:///workspace/statemachine_0.statemachine", "statemachine", content) - if err != nil { - t.Fatal(err) - } - documentParser.Parse(doc) - nodeCount := 0 - for range fastbelt.AllNodes(doc.Root) { - nodeCount++ - } - assert.Equal(t, elementCount, nodeCount, "AllNodes should iterate all nodes in the document") -} - -func TestAllChildrenEquivalence(t *testing.T) { - content, elementCount := generateStatemachineContent(0) - totalCount := elementCount - 1 // AllChildren does not include the root node, so we subtract 1 from the total count - srv := CreateServices() - documentParser := service.MustGet[workspace.DocumentParser](srv) - doc, err := fastbelt.NewDocumentFromString("file:///workspace/statemachine_0.statemachine", "statemachine", content) - if err != nil { - t.Fatal(err) - } - documentParser.Parse(doc) - childCount := 0 - for range fastbelt.AllChildren(doc.Root) { - childCount++ - } - assert.Equal(t, totalCount, childCount, "AllChildren should iterate all child nodes in the document") -} - // BenchmarkParser benchmarks parsing a single generated statemachine document, // reusing the pre-lexed token slice every iteration. func BenchmarkParser(b *testing.B) { @@ -153,7 +74,8 @@ func BenchmarkParser(b *testing.B) { srv := CreateServices() lexerService := service.MustGet[lexer.Lexer](srv) parserService := service.MustGet[parser.Parser](srv) - tokens := lexerService.Exec(content).Tokens + lexerResult, _ := lexerService.Exec(b.Context(), content) + tokens := lexerResult.Tokens doc, err := fastbelt.NewDocumentFromString("file:///workspace/statemachine_0.statemachine", "statemachine", content) if err != nil { b.Fatal(err) @@ -162,7 +84,7 @@ func BenchmarkParser(b *testing.B) { b.SetBytes(int64(len(content))) b.ResetTimer() for b.Loop() { - result := parserService.Parse(doc) + result, _ := parserService.Parse(b.Context(), doc) doc.Root = result.Node } } @@ -174,7 +96,7 @@ func BenchmarkLexer(b *testing.B) { b.SetBytes(int64(len(content))) b.ResetTimer() for b.Loop() { - _ = l.Exec(content) + _, _ = l.Exec(b.Context(), content) } } @@ -191,8 +113,9 @@ func BenchmarkLexerAndParser(b *testing.B) { b.SetBytes(int64(len(content))) b.ResetTimer() for b.Loop() { - doc.Tokens = lexerService.Exec(content).Tokens - result := parserService.Parse(doc) + lexerResult, _ := lexerService.Exec(b.Context(), content) + doc.Tokens = lexerResult.Tokens + result, _ := parserService.Parse(b.Context(), doc) doc.Root = result.Node } } @@ -226,9 +149,14 @@ func BenchmarkLocalLinking(b *testing.B) { // - 3 commands // - 50 states, each with transitions that cycle through events/states func generateStatemachineContent(index int) (string, int) { + return generateStatemachineContentWithStates(index, 50) +} + +// generateStatemachineContentWithStates is like [generateStatemachineContent] +// with a configurable number of states. +func generateStatemachineContentWithStates(index, numStates int) (string, int) { const numEvents = 4 const numCommands = 3 - const numStates = 50 elementCount := 1 var sb strings.Builder @@ -269,23 +197,3 @@ func generateStatemachineContent(index int) (string, int) { return sb.String(), elementCount } - -func TestAllNodesEarlyExitShortCircuits(t *testing.T) { - content, elementCount := generateStatemachineContent(0) - srv := CreateServices() - documentParser := service.MustGet[workspace.DocumentParser](srv) - doc, err := fastbelt.NewDocumentFromString("file:///workspace/statemachine_0.statemachine", "statemachine", content) - if err != nil { - t.Fatal(err) - } - documentParser.Parse(doc) - visited := 0 - for range fastbelt.AllNodes(doc.Root) { - visited++ - if visited == 3 { - break - } - } - assert.Equal(t, 3, visited) - assert.Greater(t, elementCount, 3) -} diff --git a/examples/statemachine/cancellation_test.go b/examples/statemachine/cancellation_test.go new file mode 100644 index 00000000..e7e52f66 --- /dev/null +++ b/examples/statemachine/cancellation_test.go @@ -0,0 +1,98 @@ +package statemachine + +import ( + "context" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "typefox.dev/fastbelt" + "typefox.dev/fastbelt/lexer" + "typefox.dev/fastbelt/parser" + "typefox.dev/fastbelt/util/service" +) + +func TestParseCancelledContextTerminates(t *testing.T) { + content, _ := generateStatemachineContent(0) + srv := CreateServices() + doc, err := fastbelt.NewDocumentFromString("file:///workspace/statemachine_0.statemachine", "statemachine", content) + if err != nil { + t.Fatal(err) + } + lexer := service.MustGet[lexer.Lexer](srv) + parser := service.MustGet[parser.Parser](srv) + lexerResult, _ := lexer.Exec(t.Context(), content) + doc.Tokens = lexerResult.Tokens + ctx, cancel := context.WithCancel(t.Context()) + // Cancel ahead of time + cancel() + _, err = parser.Parse(ctx, doc) + assert.Error(t, err) +} + +// hugeStateCount yields a ~20 MB document, large enough that timer and +// scheduling jitter stay small relative to a full lexer or parser run. +const hugeStateCount = 200_000 + +func TestLexerCancellationStopsEarly(t *testing.T) { + if testing.Short() { + t.Skip("skip long running lexer cancellation test") + } + content, _ := generateStatemachineContentWithStates(0, hugeStateCount) + lexer := service.MustGet[lexer.Lexer](CreateServices()) + assertCancellationStopsEarly(t, func(ctx context.Context) error { + _, err := lexer.Exec(ctx, content) + return err + }) +} + +func TestParserCancellationStopsEarly(t *testing.T) { + if testing.Short() { + t.Skip("skip long running parser cancellation test") + } + content, _ := generateStatemachineContentWithStates(0, hugeStateCount) + srv := CreateServices() + doc, err := fastbelt.NewDocumentFromString("file:///workspace/statemachine_0.statemachine", "statemachine", content) + if err != nil { + t.Fatal(err) + } + lexer := service.MustGet[lexer.Lexer](srv) + lexerResult, err := lexer.Exec(t.Context(), content) + require.NoError(t, err) + doc.Tokens = lexerResult.Tokens + parser := service.MustGet[parser.Parser](srv) + assertCancellationStopsEarly(t, func(ctx context.Context) error { + _, err := parser.Parse(ctx, doc) + return err + }) +} + +// assertCancellationStopsEarly times a full run, then reruns with +// cancellation fired after 20% of that time and asserts the cancelled run +// finishes within 30% of it. +func assertCancellationStopsEarly(t *testing.T, run func(ctx context.Context) error) { + t.Helper() + err := run(t.Context()) // warm up + require.NoError(t, err) + + runtime.GC() + start := time.Now() + err = run(t.Context()) + require.NoError(t, err) + full := time.Since(start) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + runtime.GC() + start = time.Now() + timer := time.AfterFunc(full/5, cancel) + defer timer.Stop() + err = run(ctx) + assert.Error(t, err) + cancelled := time.Since(start) + + t.Logf("full: %v, cancelled: %v", full, cancelled) + assert.Less(t, cancelled, full*3/10) +} diff --git a/examples/statemachine/completion_gen.go b/examples/statemachine/completion_gen.go index 7a110b9a..3cb1488c 100644 --- a/examples/statemachine/completion_gen.go +++ b/examples/statemachine/completion_gen.go @@ -108,8 +108,8 @@ func NewStatemachineModelCompletionAdapter(sc *service.Container) *StatemachineM return &StatemachineModelCompletionAdapter{sc: sc} } -func (a *StatemachineModelCompletionAdapter) Parse(tokens []core.Token) *parser.CompletionParseResult { - return NewCompletionParser(a.sc).Parse(tokens) +func (a *StatemachineModelCompletionAdapter) Parse(ctx context.Context, tokens []core.Token) *parser.CompletionParseResult { + return NewCompletionParser(a.sc).Parse(ctx, tokens) } func (a *StatemachineModelCompletionAdapter) ATN() *parser.RuntimeATN { diff --git a/examples/statemachine/completion_parser_gen.go b/examples/statemachine/completion_parser_gen.go index 26e32460..80ee905d 100644 --- a/examples/statemachine/completion_parser_gen.go +++ b/examples/statemachine/completion_parser_gen.go @@ -3,6 +3,7 @@ package statemachine import ( + "context" core "typefox.dev/fastbelt" "typefox.dev/fastbelt/parser" "typefox.dev/fastbelt/util/service" @@ -27,12 +28,12 @@ func NewCompletionParser(sc *service.Container) *CompletionParser { // the document's tokens up to the cursor) and returns the recorded // snapshots and rule stack. The completion provider feeds that result into // the ATN simulator. -func (p *CompletionParser) Parse(tokens []core.Token) *parser.CompletionParseResult { +func (p *CompletionParser) Parse(ctx context.Context, tokens []core.Token) *parser.CompletionParseResult { messages := service.MustGet[parser.ErrorMessageProvider](p.sc) recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) lookahead := service.MustGet[StatemachineModelParserLookahead](p.sc) cp := &CompletionParser{sc: p.sc, atn: p.atn, lookahead: lookahead} - cp.state = parser.NewParserState(tokens, p.atn(), recovery, messages) + cp.state = parser.NewParserState(ctx, tokens, p.atn(), recovery, messages) cp.cp = parser.NewCompletionParserState(cp.state) cp.ParseStatemachine() return cp.cp.Result(tokens) diff --git a/examples/statemachine/iterators_test.go b/examples/statemachine/iterators_test.go new file mode 100644 index 00000000..6562ffc7 --- /dev/null +++ b/examples/statemachine/iterators_test.go @@ -0,0 +1,108 @@ +package statemachine + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "typefox.dev/fastbelt" + "typefox.dev/fastbelt/util/service" + "typefox.dev/fastbelt/workspace" +) + +func BenchmarkTraverseContentSeq(b *testing.B) { + content, _ := generateStatemachineContent(0) + srv := CreateServices() + documentParser := service.MustGet[workspace.DocumentParser](srv) + doc, err := fastbelt.NewDocumentFromString("file:///workspace/statemachine_0.statemachine", "statemachine", content) + if err != nil { + b.Fatal(err) + } + documentParser.Parse(b.Context(), doc) + + for b.Loop() { + count := 0 + for range fastbelt.AllChildren(doc.Root) { + count++ + } + _ = count + } +} + +// BenchmarkTraverseContentSeqHalf breaks out of the loop after visiting half of the +// nodes. With short-circuiting traversal this should take roughly half the time of +// BenchmarkTraverseContentSeq. +func BenchmarkTraverseContentSeqHalf(b *testing.B) { + content, elementCount := generateStatemachineContent(0) + srv := CreateServices() + documentParser := service.MustGet[workspace.DocumentParser](srv) + doc, err := fastbelt.NewDocumentFromString("file:///workspace/statemachine_0.statemachine", "statemachine", content) + if err != nil { + b.Fatal(err) + } + documentParser.Parse(b.Context(), doc) + half := elementCount / 2 + + for b.Loop() { + count := 0 + for range fastbelt.AllChildren(doc.Root) { + count++ + if count == half { + break + } + } + _ = count + } +} + +func TestAllNodesEquivalence(t *testing.T) { + content, elementCount := generateStatemachineContent(0) + srv := CreateServices() + documentParser := service.MustGet[workspace.DocumentParser](srv) + doc, err := fastbelt.NewDocumentFromString("file:///workspace/statemachine_0.statemachine", "statemachine", content) + if err != nil { + t.Fatal(err) + } + documentParser.Parse(t.Context(), doc) + nodeCount := 0 + for range fastbelt.AllNodes(doc.Root) { + nodeCount++ + } + assert.Equal(t, elementCount, nodeCount, "AllNodes should iterate all nodes in the document") +} + +func TestAllChildrenEquivalence(t *testing.T) { + content, elementCount := generateStatemachineContent(0) + totalCount := elementCount - 1 // AllChildren does not include the root node, so we subtract 1 from the total count + srv := CreateServices() + documentParser := service.MustGet[workspace.DocumentParser](srv) + doc, err := fastbelt.NewDocumentFromString("file:///workspace/statemachine_0.statemachine", "statemachine", content) + if err != nil { + t.Fatal(err) + } + documentParser.Parse(t.Context(), doc) + childCount := 0 + for range fastbelt.AllChildren(doc.Root) { + childCount++ + } + assert.Equal(t, totalCount, childCount, "AllChildren should iterate all child nodes in the document") +} + +func TestAllNodesEarlyExitShortCircuits(t *testing.T) { + content, elementCount := generateStatemachineContent(0) + srv := CreateServices() + documentParser := service.MustGet[workspace.DocumentParser](srv) + doc, err := fastbelt.NewDocumentFromString("file:///workspace/statemachine_0.statemachine", "statemachine", content) + if err != nil { + t.Fatal(err) + } + documentParser.Parse(t.Context(), doc) + visited := 0 + for range fastbelt.AllNodes(doc.Root) { + visited++ + if visited == 3 { + break + } + } + assert.Equal(t, 3, visited) + assert.Greater(t, elementCount, 3) +} diff --git a/examples/statemachine/parser_gen.go b/examples/statemachine/parser_gen.go index fd19d21b..f24b82b3 100644 --- a/examples/statemachine/parser_gen.go +++ b/examples/statemachine/parser_gen.go @@ -3,6 +3,8 @@ package statemachine import ( + "context" + core "typefox.dev/fastbelt" "typefox.dev/fastbelt/parser" "typefox.dev/fastbelt/util/service" @@ -15,15 +17,23 @@ type Parser struct { lookahead StatemachineModelParserLookahead } -func (p *Parser) Parse(document *core.Document) *parser.ParseResult { +func (p *Parser) Parse(ctx context.Context, document *core.Document) (*parser.ParseResult, error) { recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) messages := service.MustGet[parser.ErrorMessageProvider](p.sc) referencesConstructor := service.MustGet[StatemachineModelReferencesConstructor](p.sc) lookahead := service.MustGet[StatemachineModelParserLookahead](p.sc) - cp := &Parser{sc: p.sc, referencesConstructor: referencesConstructor, lookahead: lookahead, state: parser.NewParserState(document.Tokens, ATN(), recovery, messages)} + cp := &Parser{ + sc: p.sc, + referencesConstructor: referencesConstructor, + lookahead: lookahead, + state: parser.NewParserState(ctx, document.Tokens, ATN(), recovery, messages), + } result := cp.ParseStatemachine() + if ctx.Err() != nil { + return nil, ctx.Err() + } cp.state.ExpectEndOfInput() - return &parser.ParseResult{Node: result, Errors: cp.state.Errors()} + return &parser.ParseResult{Node: result, Errors: cp.state.Errors()}, nil } func NewParser(sc *service.Container) *Parser { diff --git a/internal/generator/completion_generator.go b/internal/generator/completion_generator.go index 88edcacf..e267e2f0 100644 --- a/internal/generator/completion_generator.go +++ b/internal/generator/completion_generator.go @@ -196,8 +196,8 @@ func generateLspAdapter(ctx *LinkerGeneratorContext, actions []actionEntry) code node.AppendLine("}") node.AppendLine() - node.AppendLine("func (a *", name, "CompletionAdapter) Parse(tokens []core.Token) *parser.CompletionParseResult {") - node.AppendLine(" return NewCompletionParser(a.sc).Parse(tokens)") + node.AppendLine("func (a *", name, "CompletionAdapter) Parse(ctx context.Context, tokens []core.Token) *parser.CompletionParseResult {") + node.AppendLine(" return NewCompletionParser(a.sc).Parse(ctx, tokens)") node.AppendLine("}") node.AppendLine() diff --git a/internal/generator/parser_generator.go b/internal/generator/parser_generator.go index 1a592ef1..dbd3731f 100644 --- a/internal/generator/parser_generator.go +++ b/internal/generator/parser_generator.go @@ -527,6 +527,7 @@ func GenerateParser(grammr grammar.Grammar, entryRule grammar.ParserRule, packag node.AppendLine() node.AppendLine("import (") node.Indent(func(n codegen.Node) { + n.AppendLine("\"context\"").AppendLine() n.AppendLine("core \"typefox.dev/fastbelt\"") n.AppendLine("\"typefox.dev/fastbelt/parser\"") n.AppendLine("\"typefox.dev/fastbelt/util/service\"") @@ -544,16 +545,24 @@ func GenerateParser(grammr grammar.Grammar, entryRule grammar.ParserRule, packag node.AppendLine("}") node.AppendLine() - node.AppendLine("func (p *Parser) Parse(document *core.Document) *parser.ParseResult {") + node.AppendLine("func (p *Parser) Parse(ctx context.Context, document *core.Document) (*parser.ParseResult, error) {") node.Indent(func(n codegen.Node) { n.AppendLine("recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc)") n.AppendLine("messages := service.MustGet[parser.ErrorMessageProvider](p.sc)") n.AppendLine("referencesConstructor := service.MustGet[", grammr.Name(), "ReferencesConstructor](p.sc)") n.AppendLine("lookahead := service.MustGet[", grammr.Name(), "ParserLookahead](p.sc)") - n.AppendLine("cp := &Parser{sc: p.sc, referencesConstructor: referencesConstructor, lookahead: lookahead, state: parser.NewParserState(document.Tokens, ATN(), recovery, messages)}") + n.AppendLine("cp := &Parser{") + n.AppendLine(" sc: p.sc,") + n.AppendLine(" referencesConstructor: referencesConstructor,") + n.AppendLine(" lookahead: lookahead,") + n.AppendLine(" state: parser.NewParserState(ctx, document.Tokens, ATN(), recovery, messages),") + n.AppendLine("}") n.AppendLine("result := cp.Parse", entryRule.Name(), "()") + n.AppendLine("if ctx.Err() != nil {") + n.AppendLine(" return nil, ctx.Err()") + n.AppendLine("}") n.AppendLine("cp.state.ExpectEndOfInput()") - n.AppendLine("return &parser.ParseResult{Node: result, Errors: cp.state.Errors()}") + n.AppendLine("return &parser.ParseResult{Node: result, Errors: cp.state.Errors()}, nil") }) node.AppendLine("}") node.AppendLine() @@ -603,6 +612,7 @@ func GenerateCompletionParser(grammr grammar.Grammar, entryRule grammar.ParserRu node.AppendLine() node.AppendLine("import (") node.Indent(func(n codegen.Node) { + n.AppendLine("\"context\"") n.AppendLine("core \"typefox.dev/fastbelt\"") n.AppendLine("\"typefox.dev/fastbelt/parser\"") n.AppendLine("\"typefox.dev/fastbelt/util/service\"") @@ -635,13 +645,13 @@ func GenerateCompletionParser(grammr grammar.Grammar, entryRule grammar.ParserRu node.AppendLine("// the document's tokens up to the cursor) and returns the recorded") node.AppendLine("// snapshots and rule stack. The completion provider feeds that result into") node.AppendLine("// the ATN simulator.") - node.AppendLine("func (p *CompletionParser) Parse(tokens []core.Token) *parser.CompletionParseResult {") + node.AppendLine("func (p *CompletionParser) Parse(ctx context.Context, tokens []core.Token) *parser.CompletionParseResult {") node.Indent(func(n codegen.Node) { n.AppendLine("messages := service.MustGet[parser.ErrorMessageProvider](p.sc)") n.AppendLine("recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc)") n.AppendLine("lookahead := service.MustGet[", grammr.Name(), "ParserLookahead](p.sc)") n.AppendLine("cp := &CompletionParser{sc: p.sc, atn: p.atn, lookahead: lookahead}") - n.AppendLine("cp.state = parser.NewParserState(tokens, p.atn(), recovery, messages)") + n.AppendLine("cp.state = parser.NewParserState(ctx, tokens, p.atn(), recovery, messages)") n.AppendLine("cp.cp = parser.NewCompletionParserState(cp.state)") n.AppendLine("cp.Parse", entryRule.Name(), "()") n.AppendLine("return cp.cp.Result(tokens)") diff --git a/internal/grammar/completion_gen.go b/internal/grammar/completion_gen.go index 376a5186..19cca9df 100644 --- a/internal/grammar/completion_gen.go +++ b/internal/grammar/completion_gen.go @@ -244,8 +244,8 @@ func NewFastbeltCompletionAdapter(sc *service.Container) *FastbeltCompletionAdap return &FastbeltCompletionAdapter{sc: sc} } -func (a *FastbeltCompletionAdapter) Parse(tokens []core.Token) *parser.CompletionParseResult { - return NewCompletionParser(a.sc).Parse(tokens) +func (a *FastbeltCompletionAdapter) Parse(ctx context.Context, tokens []core.Token) *parser.CompletionParseResult { + return NewCompletionParser(a.sc).Parse(ctx, tokens) } func (a *FastbeltCompletionAdapter) ATN() *parser.RuntimeATN { diff --git a/internal/grammar/completion_parser_gen.go b/internal/grammar/completion_parser_gen.go index 86055e30..6587cbf6 100644 --- a/internal/grammar/completion_parser_gen.go +++ b/internal/grammar/completion_parser_gen.go @@ -3,6 +3,7 @@ package grammar import ( + "context" core "typefox.dev/fastbelt" "typefox.dev/fastbelt/parser" "typefox.dev/fastbelt/util/service" @@ -27,12 +28,12 @@ func NewCompletionParser(sc *service.Container) *CompletionParser { // the document's tokens up to the cursor) and returns the recorded // snapshots and rule stack. The completion provider feeds that result into // the ATN simulator. -func (p *CompletionParser) Parse(tokens []core.Token) *parser.CompletionParseResult { +func (p *CompletionParser) Parse(ctx context.Context, tokens []core.Token) *parser.CompletionParseResult { messages := service.MustGet[parser.ErrorMessageProvider](p.sc) recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) lookahead := service.MustGet[FastbeltParserLookahead](p.sc) cp := &CompletionParser{sc: p.sc, atn: p.atn, lookahead: lookahead} - cp.state = parser.NewParserState(tokens, p.atn(), recovery, messages) + cp.state = parser.NewParserState(ctx, tokens, p.atn(), recovery, messages) cp.cp = parser.NewCompletionParserState(cp.state) cp.ParseGrammar() return cp.cp.Result(tokens) diff --git a/internal/grammar/parser_gen.go b/internal/grammar/parser_gen.go index d46b1d1c..044d66a0 100644 --- a/internal/grammar/parser_gen.go +++ b/internal/grammar/parser_gen.go @@ -3,6 +3,8 @@ package grammar import ( + "context" + core "typefox.dev/fastbelt" "typefox.dev/fastbelt/parser" "typefox.dev/fastbelt/util/service" @@ -15,15 +17,23 @@ type Parser struct { lookahead FastbeltParserLookahead } -func (p *Parser) Parse(document *core.Document) *parser.ParseResult { +func (p *Parser) Parse(ctx context.Context, document *core.Document) (*parser.ParseResult, error) { recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) messages := service.MustGet[parser.ErrorMessageProvider](p.sc) referencesConstructor := service.MustGet[FastbeltReferencesConstructor](p.sc) lookahead := service.MustGet[FastbeltParserLookahead](p.sc) - cp := &Parser{sc: p.sc, referencesConstructor: referencesConstructor, lookahead: lookahead, state: parser.NewParserState(document.Tokens, ATN(), recovery, messages)} + cp := &Parser{ + sc: p.sc, + referencesConstructor: referencesConstructor, + lookahead: lookahead, + state: parser.NewParserState(ctx, document.Tokens, ATN(), recovery, messages), + } result := cp.ParseGrammar() + if ctx.Err() != nil { + return nil, ctx.Err() + } cp.state.ExpectEndOfInput() - return &parser.ParseResult{Node: result, Errors: cp.state.Errors()} + return &parser.ParseResult{Node: result, Errors: cp.state.Errors()}, nil } func NewParser(sc *service.Container) *Parser { diff --git a/internal/languages/completion/completion_gen.go b/internal/languages/completion/completion_gen.go index 593ceebb..c0e8b671 100644 --- a/internal/languages/completion/completion_gen.go +++ b/internal/languages/completion/completion_gen.go @@ -397,8 +397,8 @@ func NewCompletionCompletionAdapter(sc *service.Container) *CompletionCompletion return &CompletionCompletionAdapter{sc: sc} } -func (a *CompletionCompletionAdapter) Parse(tokens []core.Token) *parser.CompletionParseResult { - return NewCompletionParser(a.sc).Parse(tokens) +func (a *CompletionCompletionAdapter) Parse(ctx context.Context, tokens []core.Token) *parser.CompletionParseResult { + return NewCompletionParser(a.sc).Parse(ctx, tokens) } func (a *CompletionCompletionAdapter) ATN() *parser.RuntimeATN { diff --git a/internal/languages/completion/completion_parser_gen.go b/internal/languages/completion/completion_parser_gen.go index cadd7be8..e934dd65 100644 --- a/internal/languages/completion/completion_parser_gen.go +++ b/internal/languages/completion/completion_parser_gen.go @@ -3,6 +3,7 @@ package completion import ( + "context" core "typefox.dev/fastbelt" "typefox.dev/fastbelt/parser" "typefox.dev/fastbelt/util/service" @@ -27,12 +28,12 @@ func NewCompletionParser(sc *service.Container) *CompletionParser { // the document's tokens up to the cursor) and returns the recorded // snapshots and rule stack. The completion provider feeds that result into // the ATN simulator. -func (p *CompletionParser) Parse(tokens []core.Token) *parser.CompletionParseResult { +func (p *CompletionParser) Parse(ctx context.Context, tokens []core.Token) *parser.CompletionParseResult { messages := service.MustGet[parser.ErrorMessageProvider](p.sc) recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) lookahead := service.MustGet[CompletionParserLookahead](p.sc) cp := &CompletionParser{sc: p.sc, atn: p.atn, lookahead: lookahead} - cp.state = parser.NewParserState(tokens, p.atn(), recovery, messages) + cp.state = parser.NewParserState(ctx, tokens, p.atn(), recovery, messages) cp.cp = parser.NewCompletionParserState(cp.state) cp.ParseRoot() return cp.cp.Result(tokens) diff --git a/internal/languages/completion/parser_gen.go b/internal/languages/completion/parser_gen.go index 95dbd800..43a81749 100644 --- a/internal/languages/completion/parser_gen.go +++ b/internal/languages/completion/parser_gen.go @@ -3,6 +3,8 @@ package completion import ( + "context" + core "typefox.dev/fastbelt" "typefox.dev/fastbelt/parser" "typefox.dev/fastbelt/util/service" @@ -15,15 +17,23 @@ type Parser struct { lookahead CompletionParserLookahead } -func (p *Parser) Parse(document *core.Document) *parser.ParseResult { +func (p *Parser) Parse(ctx context.Context, document *core.Document) (*parser.ParseResult, error) { recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) messages := service.MustGet[parser.ErrorMessageProvider](p.sc) referencesConstructor := service.MustGet[CompletionReferencesConstructor](p.sc) lookahead := service.MustGet[CompletionParserLookahead](p.sc) - cp := &Parser{sc: p.sc, referencesConstructor: referencesConstructor, lookahead: lookahead, state: parser.NewParserState(document.Tokens, ATN(), recovery, messages)} + cp := &Parser{ + sc: p.sc, + referencesConstructor: referencesConstructor, + lookahead: lookahead, + state: parser.NewParserState(ctx, document.Tokens, ATN(), recovery, messages), + } result := cp.ParseRoot() + if ctx.Err() != nil { + return nil, ctx.Err() + } cp.state.ExpectEndOfInput() - return &parser.ParseResult{Node: result, Errors: cp.state.Errors()} + return &parser.ParseResult{Node: result, Errors: cp.state.Errors()}, nil } func NewParser(sc *service.Container) *Parser { diff --git a/internal/languages/lookahead/completion_gen.go b/internal/languages/lookahead/completion_gen.go index b482ba91..0cf69a45 100644 --- a/internal/languages/lookahead/completion_gen.go +++ b/internal/languages/lookahead/completion_gen.go @@ -35,8 +35,8 @@ func NewLookaheadCompletionAdapter(sc *service.Container) *LookaheadCompletionAd return &LookaheadCompletionAdapter{sc: sc} } -func (a *LookaheadCompletionAdapter) Parse(tokens []core.Token) *parser.CompletionParseResult { - return NewCompletionParser(a.sc).Parse(tokens) +func (a *LookaheadCompletionAdapter) Parse(ctx context.Context, tokens []core.Token) *parser.CompletionParseResult { + return NewCompletionParser(a.sc).Parse(ctx, tokens) } func (a *LookaheadCompletionAdapter) ATN() *parser.RuntimeATN { diff --git a/internal/languages/lookahead/completion_parser_gen.go b/internal/languages/lookahead/completion_parser_gen.go index 8b72dfd5..b7fd2b91 100644 --- a/internal/languages/lookahead/completion_parser_gen.go +++ b/internal/languages/lookahead/completion_parser_gen.go @@ -3,6 +3,7 @@ package lookahead import ( + "context" core "typefox.dev/fastbelt" "typefox.dev/fastbelt/parser" "typefox.dev/fastbelt/util/service" @@ -27,12 +28,12 @@ func NewCompletionParser(sc *service.Container) *CompletionParser { // the document's tokens up to the cursor) and returns the recorded // snapshots and rule stack. The completion provider feeds that result into // the ATN simulator. -func (p *CompletionParser) Parse(tokens []core.Token) *parser.CompletionParseResult { +func (p *CompletionParser) Parse(ctx context.Context, tokens []core.Token) *parser.CompletionParseResult { messages := service.MustGet[parser.ErrorMessageProvider](p.sc) recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) lookahead := service.MustGet[LookaheadParserLookahead](p.sc) cp := &CompletionParser{sc: p.sc, atn: p.atn, lookahead: lookahead} - cp.state = parser.NewParserState(tokens, p.atn(), recovery, messages) + cp.state = parser.NewParserState(ctx, tokens, p.atn(), recovery, messages) cp.cp = parser.NewCompletionParserState(cp.state) cp.ParseRoot() return cp.cp.Result(tokens) diff --git a/internal/languages/lookahead/parser_gen.go b/internal/languages/lookahead/parser_gen.go index 17a2a58b..74019442 100644 --- a/internal/languages/lookahead/parser_gen.go +++ b/internal/languages/lookahead/parser_gen.go @@ -3,6 +3,8 @@ package lookahead import ( + "context" + core "typefox.dev/fastbelt" "typefox.dev/fastbelt/parser" "typefox.dev/fastbelt/util/service" @@ -15,15 +17,23 @@ type Parser struct { lookahead LookaheadParserLookahead } -func (p *Parser) Parse(document *core.Document) *parser.ParseResult { +func (p *Parser) Parse(ctx context.Context, document *core.Document) (*parser.ParseResult, error) { recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) messages := service.MustGet[parser.ErrorMessageProvider](p.sc) referencesConstructor := service.MustGet[LookaheadReferencesConstructor](p.sc) lookahead := service.MustGet[LookaheadParserLookahead](p.sc) - cp := &Parser{sc: p.sc, referencesConstructor: referencesConstructor, lookahead: lookahead, state: parser.NewParserState(document.Tokens, ATN(), recovery, messages)} + cp := &Parser{ + sc: p.sc, + referencesConstructor: referencesConstructor, + lookahead: lookahead, + state: parser.NewParserState(ctx, document.Tokens, ATN(), recovery, messages), + } result := cp.ParseRoot() + if ctx.Err() != nil { + return nil, ctx.Err() + } cp.state.ExpectEndOfInput() - return &parser.ParseResult{Node: result, Errors: cp.state.Errors()} + return &parser.ParseResult{Node: result, Errors: cp.state.Errors()}, nil } func NewParser(sc *service.Container) *Parser { diff --git a/internal/languages/lookahead/parser_test.go b/internal/languages/lookahead/parser_test.go index ac22cd0a..71b06e19 100644 --- a/internal/languages/lookahead/parser_test.go +++ b/internal/languages/lookahead/parser_test.go @@ -22,7 +22,8 @@ func BenchmarkNestedQualifiedNameLookahead(b *testing.B) { srv := CreateServices() lexerService := service.MustGet[lexer.Lexer](srv) parserService := service.MustGet[parser.Parser](srv) - tokens := lexerService.Exec(content).Tokens + lexerResult, _ := lexerService.Exec(b.Context(), content) + tokens := lexerResult.Tokens doc, err := fastbelt.NewDocumentFromString("file:///workspace/test.la", "lookahead", content) if err != nil { b.Fatal(err) @@ -31,7 +32,8 @@ func BenchmarkNestedQualifiedNameLookahead(b *testing.B) { b.SetBytes(int64(len(content))) b.ResetTimer() for b.Loop() { - parserService.Parse(doc) + result, _ := parserService.Parse(b.Context(), doc) + doc.Root = result.Node } } diff --git a/internal/languages/token_groups/completion_gen.go b/internal/languages/token_groups/completion_gen.go index 412909a1..676c1641 100644 --- a/internal/languages/token_groups/completion_gen.go +++ b/internal/languages/token_groups/completion_gen.go @@ -35,8 +35,8 @@ func NewTokenGroupsCompletionAdapter(sc *service.Container) *TokenGroupsCompleti return &TokenGroupsCompletionAdapter{sc: sc} } -func (a *TokenGroupsCompletionAdapter) Parse(tokens []core.Token) *parser.CompletionParseResult { - return NewCompletionParser(a.sc).Parse(tokens) +func (a *TokenGroupsCompletionAdapter) Parse(ctx context.Context, tokens []core.Token) *parser.CompletionParseResult { + return NewCompletionParser(a.sc).Parse(ctx, tokens) } func (a *TokenGroupsCompletionAdapter) ATN() *parser.RuntimeATN { diff --git a/internal/languages/token_groups/completion_parser_gen.go b/internal/languages/token_groups/completion_parser_gen.go index 3e67fcf8..e6b82e3b 100644 --- a/internal/languages/token_groups/completion_parser_gen.go +++ b/internal/languages/token_groups/completion_parser_gen.go @@ -3,6 +3,7 @@ package token_groups import ( + "context" core "typefox.dev/fastbelt" "typefox.dev/fastbelt/parser" "typefox.dev/fastbelt/util/service" @@ -27,12 +28,12 @@ func NewCompletionParser(sc *service.Container) *CompletionParser { // the document's tokens up to the cursor) and returns the recorded // snapshots and rule stack. The completion provider feeds that result into // the ATN simulator. -func (p *CompletionParser) Parse(tokens []core.Token) *parser.CompletionParseResult { +func (p *CompletionParser) Parse(ctx context.Context, tokens []core.Token) *parser.CompletionParseResult { messages := service.MustGet[parser.ErrorMessageProvider](p.sc) recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) lookahead := service.MustGet[TokenGroupsParserLookahead](p.sc) cp := &CompletionParser{sc: p.sc, atn: p.atn, lookahead: lookahead} - cp.state = parser.NewParserState(tokens, p.atn(), recovery, messages) + cp.state = parser.NewParserState(ctx, tokens, p.atn(), recovery, messages) cp.cp = parser.NewCompletionParserState(cp.state) cp.ParseModel() return cp.cp.Result(tokens) diff --git a/internal/languages/token_groups/parser_gen.go b/internal/languages/token_groups/parser_gen.go index 1acb1859..1234191a 100644 --- a/internal/languages/token_groups/parser_gen.go +++ b/internal/languages/token_groups/parser_gen.go @@ -3,6 +3,8 @@ package token_groups import ( + "context" + core "typefox.dev/fastbelt" "typefox.dev/fastbelt/parser" "typefox.dev/fastbelt/util/service" @@ -15,15 +17,23 @@ type Parser struct { lookahead TokenGroupsParserLookahead } -func (p *Parser) Parse(document *core.Document) *parser.ParseResult { +func (p *Parser) Parse(ctx context.Context, document *core.Document) (*parser.ParseResult, error) { recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) messages := service.MustGet[parser.ErrorMessageProvider](p.sc) referencesConstructor := service.MustGet[TokenGroupsReferencesConstructor](p.sc) lookahead := service.MustGet[TokenGroupsParserLookahead](p.sc) - cp := &Parser{sc: p.sc, referencesConstructor: referencesConstructor, lookahead: lookahead, state: parser.NewParserState(document.Tokens, ATN(), recovery, messages)} + cp := &Parser{ + sc: p.sc, + referencesConstructor: referencesConstructor, + lookahead: lookahead, + state: parser.NewParserState(ctx, document.Tokens, ATN(), recovery, messages), + } result := cp.ParseModel() + if ctx.Err() != nil { + return nil, ctx.Err() + } cp.state.ExpectEndOfInput() - return &parser.ParseResult{Node: result, Errors: cp.state.Errors()} + return &parser.ParseResult{Node: result, Errors: cp.state.Errors()}, nil } func NewParser(sc *service.Container) *Parser { diff --git a/internal/languages/token_modes/benchmark_test.go b/internal/languages/token_modes/benchmark_test.go index b31aa4f3..9830c695 100644 --- a/internal/languages/token_modes/benchmark_test.go +++ b/internal/languages/token_modes/benchmark_test.go @@ -20,7 +20,7 @@ func BenchmarkNestedString(b *testing.B) { b.SetBytes(int64(len(content))) b.ResetTimer() for b.Loop() { - lexerResult := lexerService.Exec(content) + lexerResult, _ := lexerService.Exec(b.Context(), content) doc.Tokens = lexerResult.Tokens } } diff --git a/internal/languages/token_modes/completion_gen.go b/internal/languages/token_modes/completion_gen.go index 10238054..0dccae7a 100644 --- a/internal/languages/token_modes/completion_gen.go +++ b/internal/languages/token_modes/completion_gen.go @@ -57,8 +57,8 @@ func NewTokenModesCompletionAdapter(sc *service.Container) *TokenModesCompletion return &TokenModesCompletionAdapter{sc: sc} } -func (a *TokenModesCompletionAdapter) Parse(tokens []core.Token) *parser.CompletionParseResult { - return NewCompletionParser(a.sc).Parse(tokens) +func (a *TokenModesCompletionAdapter) Parse(ctx context.Context, tokens []core.Token) *parser.CompletionParseResult { + return NewCompletionParser(a.sc).Parse(ctx, tokens) } func (a *TokenModesCompletionAdapter) ATN() *parser.RuntimeATN { diff --git a/internal/languages/token_modes/completion_parser_gen.go b/internal/languages/token_modes/completion_parser_gen.go index 9bf7c9a9..3862e81f 100644 --- a/internal/languages/token_modes/completion_parser_gen.go +++ b/internal/languages/token_modes/completion_parser_gen.go @@ -3,6 +3,7 @@ package token_modes import ( + "context" core "typefox.dev/fastbelt" "typefox.dev/fastbelt/parser" "typefox.dev/fastbelt/util/service" @@ -27,12 +28,12 @@ func NewCompletionParser(sc *service.Container) *CompletionParser { // the document's tokens up to the cursor) and returns the recorded // snapshots and rule stack. The completion provider feeds that result into // the ATN simulator. -func (p *CompletionParser) Parse(tokens []core.Token) *parser.CompletionParseResult { +func (p *CompletionParser) Parse(ctx context.Context, tokens []core.Token) *parser.CompletionParseResult { messages := service.MustGet[parser.ErrorMessageProvider](p.sc) recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) lookahead := service.MustGet[TokenModesParserLookahead](p.sc) cp := &CompletionParser{sc: p.sc, atn: p.atn, lookahead: lookahead} - cp.state = parser.NewParserState(tokens, p.atn(), recovery, messages) + cp.state = parser.NewParserState(ctx, tokens, p.atn(), recovery, messages) cp.cp = parser.NewCompletionParserState(cp.state) cp.ParseModel() return cp.cp.Result(tokens) diff --git a/internal/languages/token_modes/parser_gen.go b/internal/languages/token_modes/parser_gen.go index 4c3713d1..272e2483 100644 --- a/internal/languages/token_modes/parser_gen.go +++ b/internal/languages/token_modes/parser_gen.go @@ -3,6 +3,8 @@ package token_modes import ( + "context" + core "typefox.dev/fastbelt" "typefox.dev/fastbelt/parser" "typefox.dev/fastbelt/util/service" @@ -15,15 +17,23 @@ type Parser struct { lookahead TokenModesParserLookahead } -func (p *Parser) Parse(document *core.Document) *parser.ParseResult { +func (p *Parser) Parse(ctx context.Context, document *core.Document) (*parser.ParseResult, error) { recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) messages := service.MustGet[parser.ErrorMessageProvider](p.sc) referencesConstructor := service.MustGet[TokenModesReferencesConstructor](p.sc) lookahead := service.MustGet[TokenModesParserLookahead](p.sc) - cp := &Parser{sc: p.sc, referencesConstructor: referencesConstructor, lookahead: lookahead, state: parser.NewParserState(document.Tokens, ATN(), recovery, messages)} + cp := &Parser{ + sc: p.sc, + referencesConstructor: referencesConstructor, + lookahead: lookahead, + state: parser.NewParserState(ctx, document.Tokens, ATN(), recovery, messages), + } result := cp.ParseModel() + if ctx.Err() != nil { + return nil, ctx.Err() + } cp.state.ExpectEndOfInput() - return &parser.ParseResult{Node: result, Errors: cp.state.Errors()} + return &parser.ParseResult{Node: result, Errors: cp.state.Errors()}, nil } func NewParser(sc *service.Container) *Parser { diff --git a/lexer/lexer.go b/lexer/lexer.go index 1ed260b8..68f07bf5 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -5,6 +5,7 @@ package lexer import ( + "context" "unicode/utf8" core "typefox.dev/fastbelt" @@ -13,7 +14,7 @@ import ( // Lexer tokenizes a complete source string in one shot. type Lexer interface { - Exec(input string) *LexerResult + Exec(ctx context.Context, input string) (*LexerResult, error) } // LexerResult holds everything produced by a single [Lexer.Exec] pass over @@ -48,7 +49,10 @@ type DefaultLexer struct { // Exec scans input from left to right using longest-match disambiguation among // token types registered at construction time. -func (l *DefaultLexer) Exec(input string) *LexerResult { +func (l *DefaultLexer) Exec(ctx context.Context, input string) (*LexerResult, error) { + if ctx == nil { + ctx = context.Background() + } length := len(input) tokens := make([]core.Token, 0, l.avgRatio.Capacity(length)) comments := make([]core.Token, 0) @@ -134,6 +138,11 @@ func (l *DefaultLexer) Exec(input string) *LexerResult { )) } offset = end + + // Don't check the context on every token + if len(tokens)&1023 == 0 && ctx.Err() != nil { + return nil, ctx.Err() + } } if length > 0 { @@ -146,7 +155,7 @@ func (l *DefaultLexer) Exec(input string) *LexerResult { Comments: comments, Errors: errors, Modifiers: modifiers, - } + }, nil } const maxChar = 256 diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 2acbda20..d1ddccf0 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -72,7 +72,8 @@ func TestExecPrefersLongestMatch(t *testing.T) { UseTokenType(long), )) - result := lexer.Exec("abc") + result, err := lexer.Exec(t.Context(), "abc") + require.NoError(t, err) require.Len(t, result.Tokens, 1) assert.Equal(t, "LONG", result.Tokens[0].Type.Name) } @@ -85,7 +86,8 @@ func TestExecFirstRegisteredWinsEqualLengthMatch(t *testing.T) { UseTokenType(second), )) - result := lexer.Exec("a") + result, err := lexer.Exec(t.Context(), "a") + require.NoError(t, err) require.Len(t, result.Tokens, 1) assert.Equal(t, "FIRST", result.Tokens[0].Type.Name, "among equal-length matches the first registered token type must win") @@ -105,7 +107,8 @@ func TestExecRoutesTokensToModifiers(t *testing.T) { UseTokenType(other).WithModifier(7), )) - result := lexer.Exec("ab #ff!") + result, err := lexer.Exec(t.Context(), "ab #ff!") + require.NoError(t, err) assert.Equal(t, []string{"ab", "ff"}, images(result.Tokens)) assert.Equal(t, []string{"#"}, images(result.Comments)) require.Contains(t, result.Modifiers, 7) @@ -117,13 +120,16 @@ func TestExecModifiersIsNilWithoutCustomModifiers(t *testing.T) { word := matchRunes(1, "WORD", lowercase) lexer := NewDefaultLexer(0, NewTokenMode("default", UseTokenType(word))) - assert.Nil(t, lexer.Exec("ab").Modifiers) + result, err := lexer.Exec(t.Context(), "ab") + require.NoError(t, err) + assert.Nil(t, result.Modifiers) } func TestExecCollectsCustomModifierFromNonDefaultMode(t *testing.T) { lexer := stringLexer(7) - result := lexer.Exec(`ab "inside" cd`) + result, err := lexer.Exec(t.Context(), `ab "inside" cd`) + require.NoError(t, err) assert.Equal(t, []string{"ab", `"`, `"`, "cd"}, images(result.Tokens)) require.Contains(t, result.Modifiers, 7) assert.Equal(t, []string{"inside"}, images(result.Modifiers[7])) @@ -132,7 +138,8 @@ func TestExecCollectsCustomModifierFromNonDefaultMode(t *testing.T) { func TestExecSkipsHiddenTokenFromNonDefaultMode(t *testing.T) { lexer := stringLexer(core.SkippedModifier) - result := lexer.Exec(`ab "inside" cd`) + result, err := lexer.Exec(t.Context(), `ab "inside" cd`) + require.NoError(t, err) assert.Equal(t, []string{"ab", `"`, `"`, "cd"}, images(result.Tokens)) assert.Nil(t, result.Modifiers) } @@ -140,7 +147,8 @@ func TestExecSkipsHiddenTokenFromNonDefaultMode(t *testing.T) { func TestExecCollectsCommentFromNonDefaultMode(t *testing.T) { lexer := stringLexer(core.CommentModifier) - result := lexer.Exec(`ab "inside" cd`) + result, err := lexer.Exec(t.Context(), `ab "inside" cd`) + require.NoError(t, err) assert.Equal(t, []string{"inside"}, images(result.Comments)) } @@ -149,7 +157,8 @@ func TestExecCollectsCommentFromNonDefaultMode(t *testing.T) { func TestExecPushAndPopMode(t *testing.T) { lexer := stringLexer(0) - result := lexer.Exec(`ab "in string" cd`) + result, err := lexer.Exec(t.Context(), `ab "in string" cd`) + require.NoError(t, err) assert.Equal(t, []string{"ID", "QUOTE", "TEXT", "QUOTE", "ID"}, names(result.Tokens)) assert.Equal(t, []string{"ab", `"`, "in string", `"`, "cd"}, images(result.Tokens)) assert.Empty(t, result.Errors) @@ -180,7 +189,8 @@ func TestExecNestedPushMode(t *testing.T) { ) lexer := NewDefaultLexer(0, modes...) - result := lexer.Exec("o(m[i))o") + result, err := lexer.Exec(t.Context(), "o(m[i))o") + require.NoError(t, err) assert.Equal(t, []string{ "OUTER", "ENTER_MIDDLE", "MIDDLE", "ENTER_INNER", "INNER", "LEAVE", "LEAVE", "OUTER", }, names(result.Tokens)) @@ -208,7 +218,8 @@ func TestExecSetModeDoesNotGrowTheModeStack(t *testing.T) { ) lexer := NewDefaultLexer(0, modes...) - result := lexer.Exec("axbxy") + result, err := lexer.Exec(t.Context(), "axbxy") + require.NoError(t, err) assert.Equal(t, []string{"A", "IN_SECOND", "B", "IN_SECOND"}, names(result.Tokens)) require.Len(t, result.Errors, 1, "y belongs to the start mode, which is no longer active") assert.Equal(t, int32(4), result.Errors[0].Range.Start) @@ -238,7 +249,8 @@ func TestExecSetModeKeepsOuterModeReachable(t *testing.T) { ) lexer := NewDefaultLexer(0, modes...) - result := lexer.Exec("d(mso)d") + result, err := lexer.Exec(t.Context(), "d(mso)d") + require.NoError(t, err) assert.Equal(t, []string{ "IN_DEFAULT", "ENTER", "IN_MIDDLE", "SET", "IN_OTHER", "LEAVE", "IN_DEFAULT", }, names(result.Tokens)) @@ -255,7 +267,8 @@ func TestExecUnbalancedPopStaysInStartMode(t *testing.T) { UseTokenType(word), )) - result := lexer.Exec(")ab)cd") + result, err := lexer.Exec(t.Context(), ")ab)cd") + require.NoError(t, err) assert.Equal(t, []string{"POP", "WORD", "POP", "WORD"}, names(result.Tokens)) assert.Empty(t, result.Errors) } @@ -278,7 +291,8 @@ func TestExecPopAndGroupApplyTogether(t *testing.T) { ) lexer := NewDefaultLexer(0, modes...) - result := lexer.Exec("(ab)!") + result, err := lexer.Exec(t.Context(), "(ab)!") + require.NoError(t, err) assert.Equal(t, []string{"ENTER", "INSIDE", "OUTSIDE"}, names(result.Tokens)) assert.Equal(t, []string{")"}, images(result.Comments)) } @@ -289,7 +303,8 @@ func TestExecReportsUnmatchedInput(t *testing.T) { word := matchRunes(1, "WORD", lowercase) lexer := NewDefaultLexer(0, NewTokenMode("default", UseTokenType(word))) - result := lexer.Exec("ab!cd") + result, err := lexer.Exec(t.Context(), "ab!cd") + require.NoError(t, err) assert.Equal(t, []string{"ab", "cd"}, images(result.Tokens)) require.Len(t, result.Errors, 1) assert.Equal(t, int32(2), result.Errors[0].Range.Start) @@ -302,7 +317,8 @@ func TestExecUnmatchedInputConsumesWholeRune(t *testing.T) { word := matchRunes(1, "WORD", lowercase) lexer := NewDefaultLexer(0, NewTokenMode("default", UseTokenType(word))) - result := lexer.Exec("aä b") + result, err := lexer.Exec(t.Context(), "aä b") + require.NoError(t, err) require.Len(t, result.Errors, 2) assert.Equal(t, int32(1), result.Errors[0].Range.Start) assert.Equal(t, int32(3), result.Errors[0].Range.End, "the two-byte rune must be consumed as a whole") @@ -314,7 +330,8 @@ func TestExecUnmatchedInputKeepsActiveMode(t *testing.T) { // the closing quote still pops. lexer := stringLexer(0) - result := lexer.Exec(`"in#side" ab`) + result, err := lexer.Exec(t.Context(), `"in#side" ab`) + require.NoError(t, err) assert.Equal(t, []string{"QUOTE", "TEXT", "TEXT", "QUOTE", "ID"}, names(result.Tokens)) require.Len(t, result.Errors, 1) assert.Equal(t, int32(3), result.Errors[0].Range.Start) @@ -323,7 +340,8 @@ func TestExecUnmatchedInputKeepsActiveMode(t *testing.T) { func TestExecUnterminatedModeAtEndOfInput(t *testing.T) { lexer := stringLexer(0) - result := lexer.Exec(`ab "unterminated`) + result, err := lexer.Exec(t.Context(), `ab "unterminated`) + require.NoError(t, err) assert.Equal(t, []string{"ID", "QUOTE", "TEXT"}, names(result.Tokens)) assert.Equal(t, []string{"ab", `"`, "unterminated"}, images(result.Tokens)) // Leaving a mode open is not a lexer error; the parser reports the @@ -334,7 +352,8 @@ func TestExecUnterminatedModeAtEndOfInput(t *testing.T) { func TestExecEmptyInput(t *testing.T) { lexer := stringLexer(0) - result := lexer.Exec("") + result, err := lexer.Exec(t.Context(), "") + require.NoError(t, err) assert.Empty(t, result.Tokens) assert.Empty(t, result.Comments) assert.Empty(t, result.Errors) @@ -348,10 +367,12 @@ func TestExecResetsModeStackBetweenRuns(t *testing.T) { // input that ends inside a pushed mode must not affect the next run. lexer := stringLexer(0) - first := lexer.Exec(`"unterminated`) + first, err := lexer.Exec(t.Context(), `"unterminated`) + require.NoError(t, err) require.Equal(t, []string{"QUOTE", "TEXT"}, names(first.Tokens)) - second := lexer.Exec(`ab "in string" cd`) + second, err := lexer.Exec(t.Context(), `ab "in string" cd`) + require.NoError(t, err) assert.Equal(t, []string{"ID", "QUOTE", "TEXT", "QUOTE", "ID"}, names(second.Tokens)) assert.Empty(t, second.Errors) } @@ -359,10 +380,12 @@ func TestExecResetsModeStackBetweenRuns(t *testing.T) { func TestExecRepeatedRunsAreIdentical(t *testing.T) { lexer := stringLexer(0) const input = `ab "in string" cd` - - first := names(lexer.Exec(input).Tokens) + firstResult, err := lexer.Exec(t.Context(), input) + require.NoError(t, err) + first := names(firstResult.Tokens) for range 5 { - assert.Equal(t, first, names(lexer.Exec(input).Tokens)) + followUp, _ := lexer.Exec(t.Context(), input) + assert.Equal(t, first, names(followUp.Tokens)) } } @@ -378,14 +401,17 @@ func TestExecIsSafeForConcurrentUse(t *testing.T) { } expected := make([][]string, len(inputs)) for i, input := range inputs { - expected[i] = names(lexer.Exec(input).Tokens) + lexerResult, err := lexer.Exec(t.Context(), input) + require.NoError(t, err) + expected[i] = names(lexerResult.Tokens) } var wg sync.WaitGroup for round := range 50 { for i, input := range inputs { wg.Go(func() { - assert.Equal(t, expected[i], names(lexer.Exec(input).Tokens), + lexerResult, _ := lexer.Exec(t.Context(), input) + assert.Equal(t, expected[i], names(lexerResult.Tokens), "round %d, input %q", round, input) }) } diff --git a/parser/completion_parser.go b/parser/completion_parser.go index 0d7d5dd0..cd1a5400 100644 --- a/parser/completion_parser.go +++ b/parser/completion_parser.go @@ -21,7 +21,7 @@ import ( type LanguageCompletionAdapter interface { // Parse runs the generated CompletionParser over the given prefix // tokens (the document's tokens up to the cursor). - Parse(tokens []core.Token) *CompletionParseResult + Parse(ctx context.Context, tokens []core.Token) *CompletionParseResult // ATN returns the RuntimeATN the simulator queries. ATN() *RuntimeATN // SyntheticOwnerFor returns a fresh, detached AST node of the given diff --git a/parser/completion_parser_test.go b/parser/completion_parser_test.go index ec9069b1..64ad4cd3 100644 --- a/parser/completion_parser_test.go +++ b/parser/completion_parser_test.go @@ -5,6 +5,7 @@ package parser import ( + "context" "testing" core "typefox.dev/fastbelt" @@ -21,7 +22,7 @@ func TestCompletionParserState_SnapshotsAndStack(t *testing.T) { atn := NewRuntimeATN([]*RuntimeATNState{s0}, nil, nil) tokens := []core.Token{tok(tA), tok(tA), tok(tA)} - state := NewParserState(tokens, atn, &BailErrorRecovery{}, &DefaultErrorMessageProvider{}) + state := NewParserState(context.Background(), tokens, atn, &BailErrorRecovery{}, &DefaultErrorMessageProvider{}) cp := NewCompletionParserState(state) // Outer rule enters at token 0, ATN start state 7. diff --git a/parser/parser.go b/parser/parser.go index 5814c1e3..1f2041e1 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -5,12 +5,14 @@ package parser import ( + "context" + core "typefox.dev/fastbelt" ) // Parser defines the interface for parsing tokens (lexer output) into AST nodes. type Parser interface { - Parse(document *core.Document) *ParseResult + Parse(ctx context.Context, document *core.Document) (*ParseResult, error) } type ParseResult struct { @@ -28,6 +30,7 @@ const ( ) type ParserState struct { + ctx context.Context Tokens []core.Token Length int Index int @@ -49,6 +52,10 @@ type ParserState struct { lastErrorIndex int // index of the last token that caused an error, used to prevent infinite loops in recovery } +func (p *ParserState) Context() context.Context { + return p.ctx +} + func (p *ParserState) ATN() *RuntimeATN { return p.atn } @@ -121,11 +128,15 @@ type LL1Lookahead struct { Lookup []int } -func NewParserState(tokens []core.Token, atn *RuntimeATN, recovery ErrorRecoveryStrategy, messages ErrorMessageProvider) *ParserState { +func NewParserState(ctx context.Context, tokens []core.Token, atn *RuntimeATN, recovery ErrorRecoveryStrategy, messages ErrorMessageProvider) *ParserState { if atn == nil { panic("atn must be provided") } + if ctx == nil { + ctx = context.Background() + } return &ParserState{ + ctx: ctx, Tokens: tokens, Length: len(tokens), Index: 0, @@ -250,6 +261,13 @@ func (p *ParserState) ExitRule() { if len(p.followStates) > 0 { p.followStates = p.followStates[:len(p.followStates)-1] } + if p.ctx.Err() != nil { + // Use ErrorMode as a proxy to abort parsing. + // Also, make sure that lookaheads terminate by jumping to the end of the token stream. + p.ErrorMode = ErrorModeFail + p.Index = p.Length + return + } if p.ErrorMode == ErrorModeFail { // Once we exit the rule where the error was detected, we can attempt recovery. p.ErrorMode = ErrorModeRecover diff --git a/server/completion_provider.go b/server/completion_provider.go index 65895d26..6813be28 100644 --- a/server/completion_provider.go +++ b/server/completion_provider.go @@ -147,7 +147,11 @@ func (s *DefaultCompletionProvider) completionsForContext( cc CompletionContext, ) []lsp.CompletionItem { prefixTokens := doc.Tokens[:cc.PrefixLen] - result := adapter.Parse(prefixTokens) + result := adapter.Parse(ctx, prefixTokens) + if result == nil || ctx.Err() != nil { + // Parsing was cancelled, the request is already aborted by the caller + return nil + } live, _, ok := result.SimulateAt(atn, cc.PrefixLen) if !ok { return nil diff --git a/workspace/builder.go b/workspace/builder.go index a7cbd387..6acc7e5e 100644 --- a/workspace/builder.go +++ b/workspace/builder.go @@ -70,7 +70,7 @@ func (s *DefaultBuilder) Build(ctx context.Context, docs []*core.Document, downg } // STEP 1.1: Parse the document and create the AST. if !doc.State.Has(core.DocStateParsed) { - parser.Parse(doc) + parser.Parse(ctx, doc) doc.State = doc.State.With(core.DocStateParsed) s.notifyListeners(ctx, core.DocStateParsed, doc) } diff --git a/workspace/document_parser.go b/workspace/document_parser.go index c935c417..281de7f5 100644 --- a/workspace/document_parser.go +++ b/workspace/document_parser.go @@ -5,6 +5,8 @@ package workspace import ( + "context" + core "typefox.dev/fastbelt" "typefox.dev/fastbelt/lexer" "typefox.dev/fastbelt/parser" @@ -15,7 +17,7 @@ import ( type DocumentParser interface { // Parse tokenizes doc.TextDoc, builds doc.Root, and stores lexer and parser // errors on doc. The caller must hold the workspace write [Lock]. - Parse(doc *core.Document) + Parse(ctx context.Context, doc *core.Document) } // DefaultDocumentParser is the default implementation of [DocumentParser]. @@ -29,17 +31,23 @@ func NewDefaultDocumentParser(sc *service.Container) DocumentParser { return &DefaultDocumentParser{sc: sc} } -func (s *DefaultDocumentParser) Parse(doc *core.Document) { +func (s *DefaultDocumentParser) Parse(ctx context.Context, doc *core.Document) { text := doc.TextDoc.Text(nil) // Run the lexer lexer := service.MustGet[lexer.Lexer](s.sc) - lexerRes := lexer.Exec(text) + lexerRes, err := lexer.Exec(ctx, text) + if err != nil { + return + } doc.LexerErrors = lexerRes.Errors doc.Tokens = lexerRes.Tokens doc.Comments = lexerRes.Comments // Run the parser parser := service.MustGet[parser.Parser](s.sc) - parserRes := parser.Parse(doc) + parserRes, err := parser.Parse(ctx, doc) + if err != nil { + return + } doc.ParserErrors = parserRes.Errors doc.Root = parserRes.Node core.AssignContainers(doc)