Commit 5c2510ba58d2c5b0b3082244ab9b9320662d1d65 | zig

author	Isaac Freund <mail@isaacfreund.com> 2026-06-12 13:17:01+02:00
committer	Isaac Freund <mail@isaacfreund.com> 2026-07-08 11:48:57+02:00
log	5c2510ba58d2c5b0b3082244ab9b9320662d1d65
tree	b42d85d96a54523eb78cc8299331612b93ec2411
parent	0b24b58fb24b462c85ebba53ee838e87887e4d25
signature	Commit is signed but in an unrecognized format.

grammar: match tokenizer newline handling

The tokenizer currently handles termination of comments/multiline string
literal lines differently from the grammar. The tokenizer forbids
e.g. tab characters at the end of comments before a newline and requires
\r to be directly followed by \n.

The tokenizer behavior here is intended, see c2b8afcac9e427102370dc.

2 files changed, 35 insertions(+), 41 deletions(-)

doc
- langref
  - grammar.peg
lib
- std
  - zig
    parser_generated_oracle.zig

doc/langref/grammar.peg+7-4

 @@ -430,10 +430,13 @@ string_char
     <- '\\"'
      / !["] non_control_utf8
-container_doc_comment <- ('//!' non_control_utf8* [ \n]* skip)+
+container_doc_comment <- ('//!' non_control_utf8* newline skip)+
-doc_comment <- ('///' non_control_utf8* [ \n]* skip)+
+doc_comment <- ('///' non_control_utf8* newline skip)+
-line_comment <- '//' ![!/] non_control_utf8* / '////' non_control_utf8*
+line_comment
-line_string <- '\\\\' non_control_utf8* [ \n]*
+    <- '//' ![!/] non_control_utf8* newline
+     / '////' non_control_utf8* newline
+line_string <- '\\\\' non_control_utf8* newline
+newline <- "\n" / "\r\n" / eof
 skip <- ([ \n\t\r] / line_comment)*
 CHAR_LITERAL <- ['] char_char* ['] skip

lib/std/zig/parser_generated_oracle.zig+28-37

 @@ -2194,18 +2194,7 @@ const Parser = struct {
                     } and blk_4: {
                         while (p.parsenon_control_utf8()) {}
                         break :blk_4 true;
-                    } and blk_4: {
+                    } and p.parsenewline() and p.parseskip()) break :blk_3 true;
-                        while ((p.i < p.source.len and switch (p.source[p.i]) {
-                            ' '...' ',
-                            '\n'...'\n',
-                            => blk_5: {
-                                p.i += 1;
-                                break :blk_5 true;
-                            },
-                            else => false,
-                        })) {}
-                        break :blk_4 true;
-                    } and p.parseskip()) break :blk_3 true;
                     p.i = pos_3;
                     break :blk_3 false;
                 }) {
 @@ -2233,18 +2222,7 @@ const Parser = struct {
                     } and blk_4: {
                         while (p.parsenon_control_utf8()) {}
                         break :blk_4 true;
-                    } and blk_4: {
+                    } and p.parsenewline() and p.parseskip()) break :blk_3 true;
-                        while ((p.i < p.source.len and switch (p.source[p.i]) {
-                            ' '...' ',
-                            '\n'...'\n',
-                            => blk_5: {
-                                p.i += 1;
-                                break :blk_5 true;
-                            },
-                            else => false,
-                        })) {}
-                        break :blk_4 true;
-                    } and p.parseskip()) break :blk_3 true;
                     p.i = pos_3;
                     break :blk_3 false;
                 }) {
 @@ -2281,7 +2259,7 @@ const Parser = struct {
             } and blk_1: {
                 while (p.parsenon_control_utf8()) {}
                 break :blk_1 true;
-            }) break :blk_0 true;
+            } and p.parsenewline()) break :blk_0 true;
             p.i = pos_0;
             if (blk_1: {
                 if (std.mem.startsWith(u8, p.source[p.i..], "////")) {
 @@ -2292,7 +2270,7 @@ const Parser = struct {
             } and blk_1: {
                 while (p.parsenon_control_utf8()) {}
                 break :blk_1 true;
-            }) break :blk_0 true;
+            } and p.parsenewline()) break :blk_0 true;
             p.i = pos_0;
             break :blk_0 false;
         };
 @@ -2309,19 +2287,32 @@ const Parser = struct {
             } and blk_1: {
                 while (p.parsenon_control_utf8()) {}
                 break :blk_1 true;
-            } and blk_1: {
+            } and p.parsenewline()) break :blk_0 true;
-                while ((p.i < p.source.len and switch (p.source[p.i]) {
+            p.i = pos_0;
-                    ' '...' ',
+            break :blk_0 false;
-                    '\n'...'\n',
+        };
-                    => blk_2: {
+    }
-                        p.i += 1;
+    pub fn parsenewline(p: *Parser) bool {
-                        break :blk_2 true;
+        return blk_0: {
-                    },
+            const pos_0 = p.i;
-                    else => false,
+            if (blk_1: {
-                })) {}
+                if (std.mem.startsWith(u8, p.source[p.i..], "\n")) {
-                break :blk_1 true;
+                    p.i += 1;
+                    break :blk_1 true;
+                }
+                break :blk_1 false;
             }) break :blk_0 true;
             p.i = pos_0;
+            if (blk_1: {
+                if (std.mem.startsWith(u8, p.source[p.i..], "\r\n")) {
+                    p.i += 2;
+                    break :blk_1 true;
+                }
+                break :blk_1 false;
+            }) break :blk_0 true;
+            p.i = pos_0;
+            if (p.parseeof()) break :blk_0 true;
+            p.i = pos_0;
             break :blk_0 false;
         };
     }