diff --git a/doc/langref/grammar.peg b/doc/langref/grammar.peg index 45e69f2017f463952220fa1b7b2aa7c54dbc1401..59bd42eda107d6f4483dbbb8d8d32e1ae92a34c8 100644 --- a/doc/langref/grammar.peg +++ b/doc/langref/grammar.peg @@ -405,25 +405,30 @@ multibyte_utf8 <- / oxE0 oxA0_oxBF ox80_oxBF / oxC2_oxDF ox80_oxBF +# Exclude \177 which is DEL non_control_ascii <- [\040-\176] -non_control_utf8 <- [\040-\377] +non_control_utf8 <- [\040-\176\200-\377] -hex <- [0-9a-fA-F] -hex_ <- '_'? hex +# XXX: the Zig tokenizer doesn't yet perform UTF-8 validation +# When the tokenizer is fixed, switch back to these definitions +# that forbid invalid UTF-8: +# +# char_char +# <- multibyte_utf8 +# / "\\'" +# / !['] non_control_ascii +# string_char +# <- multibyte_utf8 +# / '\\"' +# / !["] non_control_ascii -char_escape - <- "\\x" hex hex - / "\\u{" hex+ "}" - / "\\" [nr\\t'"] char_char - <- multibyte_utf8 - / char_escape - / ![\\'\n] non_control_ascii + <- "\\'" + / !['] non_control_utf8 string_char - <- multibyte_utf8 - / char_escape - / ![\\"\n] non_control_ascii + <- '\\"' + / !["] non_control_utf8 container_doc_comment <- ('//!' non_control_utf8* [ \n]* skip)+ doc_comment <- ('///' non_control_utf8* [ \n]* skip)+ @@ -431,7 +436,7 @@ line_comment <- '//' ![!/] non_control_utf8* / '////' non_control_utf8* line_string <- '\\\\' non_control_utf8* [ \n]* skip <- ([ \n\t\r] / line_comment)* -CHAR_LITERAL <- ['] char_char ['] skip +CHAR_LITERAL <- ['] char_char* ['] skip digit <- [_0-9A-DF-OQ-Za-df-oq-z] digit_int <- digit / [eEpP] @@ -440,6 +445,7 @@ NUMBERLITERAL <- [0-9] digit_int* '.' digit_float+ skip / [0-9] digit_float* skip + STRINGLITERALSINGLE <- ["] string_char* ["] skip STRINGLITERAL <- STRINGLITERALSINGLE diff --git a/lib/std/zig/parser_generated_oracle.zig b/lib/std/zig/parser_generated_oracle.zig index 0a6ce6ec7a26728b0d56c224b508350688118ef1..e5764fdd1a510af2869a869789c947a48d9e4d7e 100644 --- a/lib/std/zig/parser_generated_oracle.zig +++ b/lib/std/zig/parser_generated_oracle.zig @@ -2110,92 +2110,8 @@ const Parser = struct { return blk_0: { const pos_0 = p.i; if ((p.i < p.source.len and switch (p.source[p.i]) { - ' '...'\xff', - => blk_1: { - p.i += 1; - break :blk_1 true; - }, - else => false, - })) break :blk_0 true; - p.i = pos_0; - break :blk_0 false; - }; - } - pub fn parsehex(p: *Parser) bool { - return blk_0: { - const pos_0 = p.i; - if ((p.i < p.source.len and switch (p.source[p.i]) { - '0'...'9', - 'a'...'f', - 'A'...'F', - => blk_1: { - p.i += 1; - break :blk_1 true; - }, - else => false, - })) break :blk_0 true; - p.i = pos_0; - break :blk_0 false; - }; - } - pub fn parsehex_(p: *Parser) bool { - return blk_0: { - const pos_0 = p.i; - if ((blk_2: { - if (std.mem.startsWith(u8, p.source[p.i..], "_")) { - p.i += 1; - break :blk_2 true; - } - break :blk_2 false; - } or true) and p.parsehex()) break :blk_0 true; - p.i = pos_0; - break :blk_0 false; - }; - } - pub fn parsechar_escape(p: *Parser) bool { - return blk_0: { - const pos_0 = p.i; - if (blk_1: { - if (std.mem.startsWith(u8, p.source[p.i..], "\\x")) { - p.i += 2; - break :blk_1 true; - } - break :blk_1 false; - } and p.parsehex() and p.parsehex()) break :blk_0 true; - p.i = pos_0; - if (blk_1: { - if (std.mem.startsWith(u8, p.source[p.i..], "\\u{")) { - p.i += 3; - break :blk_1 true; - } - break :blk_1 false; - } and blk_1: { - var match_1 = false; - while (p.parsehex()) { - match_1 = true; - } - break :blk_1 match_1; - } and blk_1: { - if (std.mem.startsWith(u8, p.source[p.i..], "}")) { - p.i += 1; - break :blk_1 true; - } - break :blk_1 false; - }) break :blk_0 true; - p.i = pos_0; - if (blk_1: { - if (std.mem.startsWith(u8, p.source[p.i..], "\\")) { - p.i += 1; - break :blk_1 true; - } - break :blk_1 false; - } and (p.i < p.source.len and switch (p.source[p.i]) { - 'n'...'n', - 'r'...'r', - '\\'...'\\', - 't'...'t', - '\''...'\'', - '"'...'"', + ' '...'~', + '\x80'...'\xff', => blk_1: { p.i += 1; break :blk_1 true; @@ -2209,16 +2125,18 @@ const Parser = struct { pub fn parsechar_char(p: *Parser) bool { return blk_0: { const pos_0 = p.i; - if (p.parsemultibyte_utf8()) break :blk_0 true; - p.i = pos_0; - if (p.parsechar_escape()) break :blk_0 true; + if (blk_1: { + if (std.mem.startsWith(u8, p.source[p.i..], "\\'")) { + p.i += 2; + break :blk_1 true; + } + break :blk_1 false; + }) break :blk_0 true; p.i = pos_0; if (blk_1: { const pos_1 = p.i; const match_1 = (p.i < p.source.len and switch (p.source[p.i]) { - '\\'...'\\', '\''...'\'', - '\n'...'\n', => blk_2: { p.i += 1; break :blk_2 true; @@ -2227,7 +2145,7 @@ const Parser = struct { }); p.i = pos_1; break :blk_1 !match_1; - } and p.parsenon_control_ascii()) break :blk_0 true; + } and p.parsenon_control_utf8()) break :blk_0 true; p.i = pos_0; break :blk_0 false; }; @@ -2235,16 +2153,18 @@ const Parser = struct { pub fn parsestring_char(p: *Parser) bool { return blk_0: { const pos_0 = p.i; - if (p.parsemultibyte_utf8()) break :blk_0 true; - p.i = pos_0; - if (p.parsechar_escape()) break :blk_0 true; + if (blk_1: { + if (std.mem.startsWith(u8, p.source[p.i..], "\\\"")) { + p.i += 2; + break :blk_1 true; + } + break :blk_1 false; + }) break :blk_0 true; p.i = pos_0; if (blk_1: { const pos_1 = p.i; const match_1 = (p.i < p.source.len and switch (p.source[p.i]) { - '\\'...'\\', '"'...'"', - '\n'...'\n', => blk_2: { p.i += 1; break :blk_2 true; @@ -2253,7 +2173,7 @@ const Parser = struct { }); p.i = pos_1; break :blk_1 !match_1; - } and p.parsenon_control_ascii()) break :blk_0 true; + } and p.parsenon_control_utf8()) break :blk_0 true; p.i = pos_0; break :blk_0 false; }; @@ -2443,7 +2363,10 @@ const Parser = struct { break :blk_1 true; }, else => false, - }) and p.parsechar_char() and (p.i < p.source.len and switch (p.source[p.i]) { + }) and blk_1: { + while (p.parsechar_char()) {} + break :blk_1 true; + } and (p.i < p.source.len and switch (p.source[p.i]) { '\''...'\'', => blk_1: { p.i += 1;