diff --git a/lexer.go b/lexer.go index 817900c..9f214d0 100644 --- a/lexer.go +++ b/lexer.go @@ -386,7 +386,7 @@ func (lexer *Lexer) consumeUnquotedIdentifier() token { start := lexer.currentPos - lexer.lastWidth for { r := lexer.next() - if r < 0 || r > 128 || identifierTrailingBits[uint64(r)/64]&(1<<(uint64(r)%64)) == 0 { + if r < 0 || r >= 128 || identifierTrailingBits[uint64(r)/64]&(1<<(uint64(r)%64)) == 0 { lexer.back() break } diff --git a/lexer_test.go b/lexer_test.go index 307f2af..2baa30e 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -96,6 +96,10 @@ var lexingErrorTests = []struct { }{ {"'foo", "Missing closing single quote"}, {"[?foo==bar?]", "Unknown char '?'"}, + // U+0080 is the first rune outside the ASCII bit mask used for + // unquoted identifiers. It must be rejected rather than indexed. + {"a\u0080", "Unknown char '\\u0080'"}, + {"a\u0080b", "Unknown char '\\u0080'"}, } func TestLexingErrors(t *testing.T) {