authorgravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-06-16 10:14:37+02:00
committergravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-07-08 11:48:59+02:00
log068d4837807570550e44db99d92b25544a50cad8
tree70394fae461ac1eedbf16221a2d59c2770616b1b
parent99babb4ae44688c85b786f742d8cf54379819e5e
signaturelock-open Commit is signed but in an unrecognized format.

grammar: fix doc comment at start of file

My attempt to handle this case in 6f151109b5fdf88eb8b has proven insufficient, the test case added by this commit was failing. Solve this with a "start of file" non-terminal in the PEG grammar, which unfortunately isn't portable but much better reflects the actual implementation and is much simpler.

4 files changed, 41 insertions(+), 63 deletions(-)

doc/langref/grammar.peg+10-6
...@@ -1,4 +1,4 @@...@@ -1,4 +1,4 @@
1Root <- initial_doc_comment? ContainerMembers skip eof1Root <- ContainerMembers skip eof
22
3# *** Top level ***3# *** Top level ***
4ContainerMembers <- container_doc_comment? ContainerDeclaration* (ContainerField COMMA)* (ContainerField / ContainerDeclaration*)4ContainerMembers <- container_doc_comment? ContainerDeclaration* (ContainerField COMMA)* (ContainerField / ContainerDeclaration*)
...@@ -366,6 +366,12 @@ ParamDeclList <- (ParamDecl COMMA)* (ParamDecl / DOT3 COMMA?)?...@@ -366,6 +366,12 @@ ParamDeclList <- (ParamDecl COMMA)* (ParamDecl / DOT3 COMMA?)?
366ExprList <- (Expr COMMA)* Expr?366ExprList <- (Expr COMMA)* Expr?
367367
368# *** Tokens ***368# *** Tokens ***
369
370# Unfortunately, there is not a standard way to match the start of the file in PEG.
371# This rule definition is compatible with the original peg(1) tool but is not
372# portable. It is however trivial to implement an equivalent rule in hand written
373# parsers and other PEG tooling should have similar mechanisms.
374sof <- &{ (yy->__pos == 0) }
369eof <- !.375eof <- !.
370376
371ox80_oxBF <- [\200-\277]377ox80_oxBF <- [\200-\277]
...@@ -430,12 +436,10 @@ string_char...@@ -430,12 +436,10 @@ string_char
430 / !["] non_control_utf8436 / !["] non_control_utf8
431437
432438
433# We forbid same-line doc comments to disambiguate the mapping to e.g. struct
434# fields for documentation generation tooling. However, this requires a special
435# case in the grammar to handle a doc comment right at the beginning of a file.
436initial_doc_comment <- (skip '///' non_control_utf8* newline)+ skip !'//!' !KEYWORD_test !KEYWORD_comptime !eof
437container_doc_comment <- (skip '//!' non_control_utf8* newline)+439container_doc_comment <- (skip '//!' non_control_utf8* newline)+
438doc_comment <- skip_require_newline (skip '///' non_control_utf8* newline)+440# We forbid same-line doc comments to disambiguate the mapping to e.g. struct
441# fields for documentation generation tooling.
442doc_comment <- (sof / skip_require_newline) (skip '///' non_control_utf8* newline)+
439line_comment443line_comment
440 <- '//' ![!/] non_control_utf8* newline444 <- '//' ![!/] non_control_utf8* newline
441 / '////' non_control_utf8* newline445 / '////' non_control_utf8* newline
lib/std/zig/parser_fuzz.zig+1
...@@ -50,6 +50,7 @@ test "newline required before doc comment not at start of file" {...@@ -50,6 +50,7 @@ test "newline required before doc comment not at start of file" {
50 try checkAgainstOracle("///"); // error50 try checkAgainstOracle("///"); // error
51 try checkAgainstOracle("///\n//!");51 try checkAgainstOracle("///\n//!");
52 try checkAgainstOracle("///\ntest {}");52 try checkAgainstOracle("///\ntest {}");
53 try checkAgainstOracle("///\ncomptime 0");
53}54}
5455
55// Found using AFL++56// Found using AFL++
lib/std/zig/parser_generated_oracle.zig+17-56
...@@ -16,7 +16,7 @@ const Parser = struct {...@@ -16,7 +16,7 @@ const Parser = struct {
16 pub fn parseRoot(p: *Parser) bool {16 pub fn parseRoot(p: *Parser) bool {
17 return blk_0: {17 return blk_0: {
18 const pos_0 = p.i;18 const pos_0 = p.i;
19 if ((p.parseinitial_doc_comment() or true) and p.parseContainerMembers() and p.parseskip() and p.parseeof()) break :blk_0 true;19 if (p.parseContainerMembers() and p.parseskip() and p.parseeof()) break :blk_0 true;
20 p.i = pos_0;20 p.i = pos_0;
21 break :blk_0 false;21 break :blk_0 false;
22 };22 };
...@@ -1764,6 +1764,14 @@ const Parser = struct {...@@ -1764,6 +1764,14 @@ const Parser = struct {
1764 break :blk_0 false;1764 break :blk_0 false;
1765 };1765 };
1766 }1766 }
1767 pub fn parsesof(p: *Parser) bool {
1768 return blk_0: {
1769 const pos_0 = p.i;
1770 if ((p.i == 0)) break :blk_0 true;
1771 p.i = pos_0;
1772 break :blk_0 false;
1773 };
1774 }
1767 pub fn parseeof(p: *Parser) bool {1775 pub fn parseeof(p: *Parser) bool {
1768 return blk_0: {1776 return blk_0: {
1769 const pos_0 = p.i;1777 const pos_0 = p.i;
...@@ -2083,60 +2091,6 @@ const Parser = struct {...@@ -2083,60 +2091,6 @@ const Parser = struct {
2083 break :blk_0 false;2091 break :blk_0 false;
2084 };2092 };
2085 }2093 }
2086 pub fn parseinitial_doc_comment(p: *Parser) bool {
2087 return blk_0: {
2088 const pos_0 = p.i;
2089 if (blk_1: {
2090 var match_1 = false;
2091 while (blk_3: {
2092 const pos_3 = p.i;
2093 if (p.parseskip() and blk_4: {
2094 if (std.mem.startsWith(u8, p.source[p.i..], "///")) {
2095 p.i += 3;
2096 break :blk_4 true;
2097 }
2098 break :blk_4 false;
2099 } and blk_4: {
2100 while (p.parsenon_control_utf8()) {}
2101 break :blk_4 true;
2102 } and p.parsenewline()) break :blk_3 true;
2103 p.i = pos_3;
2104 break :blk_3 false;
2105 }) {
2106 match_1 = true;
2107 }
2108 break :blk_1 match_1;
2109 } and p.parseskip() and blk_1: {
2110 const pos_1 = p.i;
2111 const match_1 = blk_2: {
2112 if (std.mem.startsWith(u8, p.source[p.i..], "//!")) {
2113 p.i += 3;
2114 break :blk_2 true;
2115 }
2116 break :blk_2 false;
2117 };
2118 p.i = pos_1;
2119 break :blk_1 !match_1;
2120 } and blk_1: {
2121 const pos_1 = p.i;
2122 const match_1 = p.parseKEYWORD_test();
2123 p.i = pos_1;
2124 break :blk_1 !match_1;
2125 } and blk_1: {
2126 const pos_1 = p.i;
2127 const match_1 = p.parseKEYWORD_comptime();
2128 p.i = pos_1;
2129 break :blk_1 !match_1;
2130 } and blk_1: {
2131 const pos_1 = p.i;
2132 const match_1 = p.parseeof();
2133 p.i = pos_1;
2134 break :blk_1 !match_1;
2135 }) break :blk_0 true;
2136 p.i = pos_0;
2137 break :blk_0 false;
2138 };
2139 }
2140 pub fn parsecontainer_doc_comment(p: *Parser) bool {2094 pub fn parsecontainer_doc_comment(p: *Parser) bool {
2141 return blk_0: {2095 return blk_0: {
2142 const pos_0 = p.i;2096 const pos_0 = p.i;
...@@ -2168,7 +2122,14 @@ const Parser = struct {...@@ -2168,7 +2122,14 @@ const Parser = struct {
2168 pub fn parsedoc_comment(p: *Parser) bool {2122 pub fn parsedoc_comment(p: *Parser) bool {
2169 return blk_0: {2123 return blk_0: {
2170 const pos_0 = p.i;2124 const pos_0 = p.i;
2171 if (p.parseskip_require_newline() and blk_1: {2125 if (blk_2: {
2126 const pos_2 = p.i;
2127 if (p.parsesof()) break :blk_2 true;
2128 p.i = pos_2;
2129 if (p.parseskip_require_newline()) break :blk_2 true;
2130 p.i = pos_2;
2131 break :blk_2 false;
2132 } and blk_1: {
2172 var match_1 = false;2133 var match_1 = false;
2173 while (blk_3: {2134 while (blk_3: {
2174 const pos_3 = p.i;2135 const pos_3 = p.i;
tools/gen_parser_oracle.zig+13-1
...@@ -242,6 +242,7 @@ const Generator = struct {...@@ -242,6 +242,7 @@ const Generator = struct {
242 \\}})242 \\}})
243 , .{ suffix, suffix });243 , .{ suffix, suffix });
244 },244 },
245 .sof => try g.w.writeAll("(p.i == 0)"),
245 else => unreachable,246 else => unreachable,
246 }247 }
247 }248 }
...@@ -337,6 +338,8 @@ const Parser = struct {...@@ -337,6 +338,8 @@ const Parser = struct {
337 start: u8,338 start: u8,
338 end: u8,339 end: u8,
339 },340 },
341 /// Start of file
342 sof,
340343
341 const Index = enum(u32) {344 const Index = enum(u32) {
342 _,345 _,
...@@ -422,8 +425,17 @@ const Parser = struct {...@@ -422,8 +425,17 @@ const Parser = struct {
422 // Prefix <- AND Action425 // Prefix <- AND Action
423 // / ( AND / NOT )? Suffix426 // / ( AND / NOT )? Suffix
424 fn parsePrefix(p: *Parser) !?Node.Index {427 fn parsePrefix(p: *Parser) !?Node.Index {
425 // We don't implement Action
426 if (p.eatAnd()) {428 if (p.eatAnd()) {
429 // We only support a single hardcoded "start of file" Action
430 if (p.eat('{')) {
431 // Action <- '{' < [^}]* > '}' Spacing
432 if (std.mem.startsWith(u8, p.source[p.i..], " (yy->__pos == 0) }")) {
433 while (!p.eat('}')) p.i += 1;
434 _ = p.eatSpacing();
435 return try p.addNode(.sof);
436 }
437 return null;
438 }
427 const suffix = try p.parseSuffix() orelse return null;439 const suffix = try p.parseSuffix() orelse return null;
428 return try p.addNode(.{ .@"&" = suffix });440 return try p.addNode(.{ .@"&" = suffix });
429 }441 }