From 0b24b58fb24b462c85ebba53ee838e87887e4d25 Mon Sep 17 00:00:00 2001 From: Isaac Freund Date: Fri, 12 Jun 2026 12:10:26 +0200 Subject: [PATCH] grammar: match tokenizer string literal behavior The tokenizer does not yet perform UTF-8 validation. This violates the language spec, but fixing it is out of scope for this branch. Leave the intended language spec commented out in the grammar to be restored when the tokenizer is fixed. --- doc/langref/grammar.peg | 34 ++++--- lib/std/zig/parser_generated_oracle.zig | 121 +++++------------------- 2 files changed, 42 insertions(+), 113 deletions(-) diff --git a/doc/langref/grammar.peg b/doc/langref/grammar.peg index 45e69f2017f463952220fa1b7b2aa7c54dbc1401..59bd42eda107d6f4483dbbb8d8d32e1ae92a34c8 100644 --- a/doc/langref/grammar.peg +++ b/doc/langref/grammar.peg @@ -405,25 +405,30 @@ multibyte_utf8 <- / oxE0 oxA0_oxBF ox80_oxBF / oxC2_oxDF ox80_oxBF +# Exclude \177 which is DEL non_control_ascii <- [\040-\176] -non_control_utf8 <- [\040-\377] +non_control_utf8 <- [\040-\176\200-\377] -hex <- [0-9a-fA-F] -hex_ <- '_'? hex +# XXX: the Zig tokenizer doesn't yet perform UTF-8 validation +# When the tokenizer is fixed, switch back to these definitions +# that forbid invalid UTF-8: +# +# char_char +# <- multibyte_utf8 +# / "\\'" +# / !['] non_control_ascii +# string_char +# <- multibyte_utf8 +# / '\\"' +# / !["] non_control_ascii -char_escape - <- "\\x" hex hex - / "\\u{" hex+ "}" - / "\\" [nr\\t'"] char_char - <- multibyte_utf8 - / char_escape - / ![\\'\n] non_control_ascii + <- "\\'" + / !['] non_control_utf8 string_char - <- multibyte_utf8 - / char_escape - / ![\\"\n] non_control_ascii + <- '\\"' + / !["] non_control_utf8 container_doc_comment <- ('//!' non_control_utf8* [ \n]* skip)+ doc_comment <- ('///' non_control_utf8* [ \n]* skip)+ @@ -431,7 +436,7 @@ line_comment <- '//' ![!/] non_control_utf8* / '////' non_control_utf8* line_string <- '\\\\' non_control_utf8* [ \n]* skip <- ([ \n\t\r] / line_comment)* -CHAR_LITERAL <- ['] char_char ['] skip +CHAR_LITERAL <- ['] char_char* ['] skip digit <- [_0-9A-DF-OQ-Za-df-oq-z] digit_int <- digit / [eEpP] @@ -440,6 +445,7 @@ NUMBERLITERAL <- [0-9] digit_int* '.' digit_float+ skip / [0-9] digit_float* skip + STRINGLITERALSINGLE <- ["] string_char* ["] skip STRINGLITERAL <- STRINGLITERALSINGLE diff --git a/lib/std/zig/parser_generated_oracle.zig b/lib/std/zig/parser_generated_oracle.zig index 0a6ce6ec7a26728b0d56c224b508350688118ef1..e5764fdd1a510af2869a869789c947a48d9e4d7e 100644 --- a/lib/std/zig/parser_generated_oracle.zig +++ b/lib/std/zig/parser_generated_oracle.zig @@ -2110,92 +2110,8 @@ const Parser = struct { return blk_0: { const pos_0 = p.i; if ((p.i < p.source.len and switch (p.source[p.i]) { - ' '...'\xff', - => blk_1: { - p.i += 1; - break :blk_1 true; - }, - else => false, - })) break :blk_0 true; - p.i = pos_0; - break :blk_0 false; - }; - } - pub fn parsehex(p: *Parser) bool { - return blk_0: { - const pos_0 = p.i; - if ((p.i < p.source.len and switch (p.source[p.i]) { - '0'...'9', - 'a'...'f', - 'A'...'F', - => blk_1: { - p.i += 1; - break :blk_1 true; - }, - else => false, - })) break :blk_0 true; - p.i = pos_0; - break :blk_0 false; - }; - } - pub fn parsehex_(p: *Parser) bool { - return blk_0: { - const pos_0 = p.i; - if ((blk_2: { - if (std.mem.startsWith(u8, p.source[p.i..], "_")) { - p.i += 1; - break :blk_2 true; - } - break :blk_2 false; - } or true) and p.parsehex()) break :blk_0 true; - p.i = pos_0; - break :blk_0 false; - }; - } - pub fn parsechar_escape(p: *Parser) bool { - return blk_0: { - const pos_0 = p.i; - if (blk_1: { - if (std.mem.startsWith(u8, p.source[p.i..], "\\x")) { - p.i += 2; - break :blk_1 true; - } - break :blk_1 false; - } and p.parsehex() and p.parsehex()) break :blk_0 true; - p.i = pos_0; - if (blk_1: { - if (std.mem.startsWith(u8, p.source[p.i..], "\\u{")) { - p.i += 3; - break :blk_1 true; - } - break :blk_1 false; - } and blk_1: { - var match_1 = false; - while (p.parsehex()) { - match_1 = true; - } - break :blk_1 match_1; - } and blk_1: { - if (std.mem.startsWith(u8, p.source[p.i..], "}")) { - p.i += 1; - break :blk_1 true; - } - break :blk_1 false; - }) break :blk_0 true; - p.i = pos_0; - if (blk_1: { - if (std.mem.startsWith(u8, p.source[p.i..], "\\")) { - p.i += 1; - break :blk_1 true; - } - break :blk_1 false; - } and (p.i < p.source.len and switch (p.source[p.i]) { - 'n'...'n', - 'r'...'r', - '\\'...'\\', - 't'...'t', - '\''...'\'', - '"'...'"', + ' '...'~', + '\x80'...'\xff', => blk_1: { p.i += 1; break :blk_1 true; @@ -2209,16 +2125,18 @@ const Parser = struct { pub fn parsechar_char(p: *Parser) bool { return blk_0: { const pos_0 = p.i; - if (p.parsemultibyte_utf8()) break :blk_0 true; - p.i = pos_0; - if (p.parsechar_escape()) break :blk_0 true; + if (blk_1: { + if (std.mem.startsWith(u8, p.source[p.i..], "\\'")) { + p.i += 2; + break :blk_1 true; + } + break :blk_1 false; + }) break :blk_0 true; p.i = pos_0; if (blk_1: { const pos_1 = p.i; const match_1 = (p.i < p.source.len and switch (p.source[p.i]) { - '\\'...'\\', '\''...'\'', - '\n'...'\n', => blk_2: { p.i += 1; break :blk_2 true; @@ -2227,7 +2145,7 @@ const Parser = struct { }); p.i = pos_1; break :blk_1 !match_1; - } and p.parsenon_control_ascii()) break :blk_0 true; + } and p.parsenon_control_utf8()) break :blk_0 true; p.i = pos_0; break :blk_0 false; }; @@ -2235,16 +2153,18 @@ const Parser = struct { pub fn parsestring_char(p: *Parser) bool { return blk_0: { const pos_0 = p.i; - if (p.parsemultibyte_utf8()) break :blk_0 true; - p.i = pos_0; - if (p.parsechar_escape()) break :blk_0 true; + if (blk_1: { + if (std.mem.startsWith(u8, p.source[p.i..], "\\\"")) { + p.i += 2; + break :blk_1 true; + } + break :blk_1 false; + }) break :blk_0 true; p.i = pos_0; if (blk_1: { const pos_1 = p.i; const match_1 = (p.i < p.source.len and switch (p.source[p.i]) { - '\\'...'\\', '"'...'"', - '\n'...'\n', => blk_2: { p.i += 1; break :blk_2 true; @@ -2253,7 +2173,7 @@ const Parser = struct { }); p.i = pos_1; break :blk_1 !match_1; - } and p.parsenon_control_ascii()) break :blk_0 true; + } and p.parsenon_control_utf8()) break :blk_0 true; p.i = pos_0; break :blk_0 false; }; @@ -2443,7 +2363,10 @@ const Parser = struct { break :blk_1 true; }, else => false, - }) and p.parsechar_char() and (p.i < p.source.len and switch (p.source[p.i]) { + }) and blk_1: { + while (p.parsechar_char()) {} + break :blk_1 true; + } and (p.i < p.source.len and switch (p.source[p.i]) { '\''...'\'', => blk_1: { p.i += 1; -- 2.54.0