authorgravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-06-12 12:10:26+02:00
committergravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-07-08 11:48:57+02:00
log0b24b58fb24b462c85ebba53ee838e87887e4d25
tree20e73d475892f8ca9f0c298d2076e6f7c28c2824
parentd11dfc9b9c9b6bb5933f10df8b618184f921c033
signaturelock-open Commit is signed but in an unrecognized format.

grammar: match tokenizer string literal behavior

The tokenizer does not yet perform UTF-8 validation. This violates the language spec, but fixing it is out of scope for this branch. Leave the intended language spec commented out in the grammar to be restored when the tokenizer is fixed.

2 files changed, 38 insertions(+), 109 deletions(-)

doc/langref/grammar.peg+21-15
......@@ -405,25 +405,30 @@ multibyte_utf8 <-
405405 / oxE0 oxA0_oxBF ox80_oxBF
406406 / oxC2_oxDF ox80_oxBF
407407
408# Exclude \177 which is DEL
408409non_control_ascii <- [\040-\176]
409non_control_utf8 <- [\040-\377]
410non_control_utf8 <- [\040-\176\200-\377]
411
412# XXX: the Zig tokenizer doesn't yet perform UTF-8 validation
413# When the tokenizer is fixed, switch back to these definitions
414# that forbid invalid UTF-8:
415#
416# char_char
417# <- multibyte_utf8
418# / "\\'"
419# / !['] non_control_ascii
420# string_char
421# <- multibyte_utf8
422# / '\\"'
423# / !["] non_control_ascii
410424
411hex <- [0-9a-fA-F]
412hex_ <- '_'? hex
413
414char_escape
415 <- "\\x" hex hex
416 / "\\u{" hex+ "}"
417 / "\\" [nr\\t'"]
418425char_char
419 <- multibyte_utf8
420 / char_escape
421 / ![\\'\n] non_control_ascii
426 <- "\\'"
427 / !['] non_control_utf8
422428
423429string_char
424 <- multibyte_utf8
425 / char_escape
426 / ![\\"\n] non_control_ascii
430 <- '\\"'
431 / !["] non_control_utf8
427432
428433container_doc_comment <- ('//!' non_control_utf8* [ \n]* skip)+
429434doc_comment <- ('///' non_control_utf8* [ \n]* skip)+
......@@ -431,7 +436,7 @@ line_comment <- '//' ![!/] non_control_utf8* / '////' non_control_utf8*
431436line_string <- '\\\\' non_control_utf8* [ \n]*
432437skip <- ([ \n\t\r] / line_comment)*
433438
434CHAR_LITERAL <- ['] char_char ['] skip
439CHAR_LITERAL <- ['] char_char* ['] skip
435440
436441digit <- [_0-9A-DF-OQ-Za-df-oq-z]
437442digit_int <- digit / [eEpP]
......@@ -440,6 +445,7 @@ NUMBERLITERAL
440445 <- [0-9] digit_int* '.' digit_float+ skip
441446 / [0-9] digit_float* skip
442447
448
443449STRINGLITERALSINGLE <- ["] string_char* ["] skip
444450STRINGLITERAL
445451 <- STRINGLITERALSINGLE
lib/std/zig/parser_generated_oracle.zig+17-94
......@@ -2110,24 +2110,8 @@ const Parser = struct {
21102110 return blk_0: {
21112111 const pos_0 = p.i;
21122112 if ((p.i < p.source.len and switch (p.source[p.i]) {
2113 ' '...'\xff',
2114 => blk_1: {
2115 p.i += 1;
2116 break :blk_1 true;
2117 },
2118 else => false,
2119 })) break :blk_0 true;
2120 p.i = pos_0;
2121 break :blk_0 false;
2122 };
2123 }
2124 pub fn parsehex(p: *Parser) bool {
2125 return blk_0: {
2126 const pos_0 = p.i;
2127 if ((p.i < p.source.len and switch (p.source[p.i]) {
2128 '0'...'9',
2129 'a'...'f',
2130 'A'...'F',
2113 ' '...'~',
2114 '\x80'...'\xff',
21312115 => blk_1: {
21322116 p.i += 1;
21332117 break :blk_1 true;
......@@ -2138,87 +2122,21 @@ const Parser = struct {
21382122 break :blk_0 false;
21392123 };
21402124 }
2141 pub fn parsehex_(p: *Parser) bool {
2142 return blk_0: {
2143 const pos_0 = p.i;
2144 if ((blk_2: {
2145 if (std.mem.startsWith(u8, p.source[p.i..], "_")) {
2146 p.i += 1;
2147 break :blk_2 true;
2148 }
2149 break :blk_2 false;
2150 } or true) and p.parsehex()) break :blk_0 true;
2151 p.i = pos_0;
2152 break :blk_0 false;
2153 };
2154 }
2155 pub fn parsechar_escape(p: *Parser) bool {
2125 pub fn parsechar_char(p: *Parser) bool {
21562126 return blk_0: {
21572127 const pos_0 = p.i;
21582128 if (blk_1: {
2159 if (std.mem.startsWith(u8, p.source[p.i..], "\\x")) {
2129 if (std.mem.startsWith(u8, p.source[p.i..], "\\'")) {
21602130 p.i += 2;
21612131 break :blk_1 true;
21622132 }
21632133 break :blk_1 false;
2164 } and p.parsehex() and p.parsehex()) break :blk_0 true;
2165 p.i = pos_0;
2166 if (blk_1: {
2167 if (std.mem.startsWith(u8, p.source[p.i..], "\\u{")) {
2168 p.i += 3;
2169 break :blk_1 true;
2170 }
2171 break :blk_1 false;
2172 } and blk_1: {
2173 var match_1 = false;
2174 while (p.parsehex()) {
2175 match_1 = true;
2176 }
2177 break :blk_1 match_1;
2178 } and blk_1: {
2179 if (std.mem.startsWith(u8, p.source[p.i..], "}")) {
2180 p.i += 1;
2181 break :blk_1 true;
2182 }
2183 break :blk_1 false;
21842134 }) break :blk_0 true;
21852135 p.i = pos_0;
2186 if (blk_1: {
2187 if (std.mem.startsWith(u8, p.source[p.i..], "\\")) {
2188 p.i += 1;
2189 break :blk_1 true;
2190 }
2191 break :blk_1 false;
2192 } and (p.i < p.source.len and switch (p.source[p.i]) {
2193 'n'...'n',
2194 'r'...'r',
2195 '\\'...'\\',
2196 't'...'t',
2197 '\''...'\'',
2198 '"'...'"',
2199 => blk_1: {
2200 p.i += 1;
2201 break :blk_1 true;
2202 },
2203 else => false,
2204 })) break :blk_0 true;
2205 p.i = pos_0;
2206 break :blk_0 false;
2207 };
2208 }
2209 pub fn parsechar_char(p: *Parser) bool {
2210 return blk_0: {
2211 const pos_0 = p.i;
2212 if (p.parsemultibyte_utf8()) break :blk_0 true;
2213 p.i = pos_0;
2214 if (p.parsechar_escape()) break :blk_0 true;
2215 p.i = pos_0;
22162136 if (blk_1: {
22172137 const pos_1 = p.i;
22182138 const match_1 = (p.i < p.source.len and switch (p.source[p.i]) {
2219 '\\'...'\\',
22202139 '\''...'\'',
2221 '\n'...'\n',
22222140 => blk_2: {
22232141 p.i += 1;
22242142 break :blk_2 true;
......@@ -2227,7 +2145,7 @@ const Parser = struct {
22272145 });
22282146 p.i = pos_1;
22292147 break :blk_1 !match_1;
2230 } and p.parsenon_control_ascii()) break :blk_0 true;
2148 } and p.parsenon_control_utf8()) break :blk_0 true;
22312149 p.i = pos_0;
22322150 break :blk_0 false;
22332151 };
......@@ -2235,16 +2153,18 @@ const Parser = struct {
22352153 pub fn parsestring_char(p: *Parser) bool {
22362154 return blk_0: {
22372155 const pos_0 = p.i;
2238 if (p.parsemultibyte_utf8()) break :blk_0 true;
2239 p.i = pos_0;
2240 if (p.parsechar_escape()) break :blk_0 true;
2156 if (blk_1: {
2157 if (std.mem.startsWith(u8, p.source[p.i..], "\\\"")) {
2158 p.i += 2;
2159 break :blk_1 true;
2160 }
2161 break :blk_1 false;
2162 }) break :blk_0 true;
22412163 p.i = pos_0;
22422164 if (blk_1: {
22432165 const pos_1 = p.i;
22442166 const match_1 = (p.i < p.source.len and switch (p.source[p.i]) {
2245 '\\'...'\\',
22462167 '"'...'"',
2247 '\n'...'\n',
22482168 => blk_2: {
22492169 p.i += 1;
22502170 break :blk_2 true;
......@@ -2253,7 +2173,7 @@ const Parser = struct {
22532173 });
22542174 p.i = pos_1;
22552175 break :blk_1 !match_1;
2256 } and p.parsenon_control_ascii()) break :blk_0 true;
2176 } and p.parsenon_control_utf8()) break :blk_0 true;
22572177 p.i = pos_0;
22582178 break :blk_0 false;
22592179 };
......@@ -2443,7 +2363,10 @@ const Parser = struct {
24432363 break :blk_1 true;
24442364 },
24452365 else => false,
2446 }) and p.parsechar_char() and (p.i < p.source.len and switch (p.source[p.i]) {
2366 }) and blk_1: {
2367 while (p.parsechar_char()) {}
2368 break :blk_1 true;
2369 } and (p.i < p.source.len and switch (p.source[p.i]) {
24472370 '\''...'\'',
24482371 => blk_1: {
24492372 p.i += 1;