authorgravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-06-16 10:14:37+02:00
committergravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-07-08 11:48:59+02:00
log068d4837807570550e44db99d92b25544a50cad8
tree70394fae461ac1eedbf16221a2d59c2770616b1b
parent99babb4ae44688c85b786f742d8cf54379819e5e
signaturelock-open Commit is signed but in an unrecognized format.

grammar: fix doc comment at start of file

My attempt to handle this case in 6f151109b5fdf88eb8b has proven insufficient, the test case added by this commit was failing. Solve this with a "start of file" non-terminal in the PEG grammar, which unfortunately isn't portable but much better reflects the actual implementation and is much simpler.

4 files changed, 41 insertions(+), 63 deletions(-)

doc/langref/grammar.peg+10-6
......@@ -1,4 +1,4 @@
1Root <- initial_doc_comment? ContainerMembers skip eof
1Root <- ContainerMembers skip eof
22
33# *** Top level ***
44ContainerMembers <- container_doc_comment? ContainerDeclaration* (ContainerField COMMA)* (ContainerField / ContainerDeclaration*)
......@@ -366,6 +366,12 @@ ParamDeclList <- (ParamDecl COMMA)* (ParamDecl / DOT3 COMMA?)?
366366ExprList <- (Expr COMMA)* Expr?
367367
368368# *** Tokens ***
369
370# Unfortunately, there is not a standard way to match the start of the file in PEG.
371# This rule definition is compatible with the original peg(1) tool but is not
372# portable. It is however trivial to implement an equivalent rule in hand written
373# parsers and other PEG tooling should have similar mechanisms.
374sof <- &{ (yy->__pos == 0) }
369375eof <- !.
370376
371377ox80_oxBF <- [\200-\277]
......@@ -430,12 +436,10 @@ string_char
430436 / !["] non_control_utf8
431437
432438
433# We forbid same-line doc comments to disambiguate the mapping to e.g. struct
434# fields for documentation generation tooling. However, this requires a special
435# case in the grammar to handle a doc comment right at the beginning of a file.
436initial_doc_comment <- (skip '///' non_control_utf8* newline)+ skip !'//!' !KEYWORD_test !KEYWORD_comptime !eof
437439container_doc_comment <- (skip '//!' non_control_utf8* newline)+
438doc_comment <- skip_require_newline (skip '///' non_control_utf8* newline)+
440# We forbid same-line doc comments to disambiguate the mapping to e.g. struct
441# fields for documentation generation tooling.
442doc_comment <- (sof / skip_require_newline) (skip '///' non_control_utf8* newline)+
439443line_comment
440444 <- '//' ![!/] non_control_utf8* newline
441445 / '////' non_control_utf8* newline
lib/std/zig/parser_fuzz.zig+1
......@@ -50,6 +50,7 @@ test "newline required before doc comment not at start of file" {
5050 try checkAgainstOracle("///"); // error
5151 try checkAgainstOracle("///\n//!");
5252 try checkAgainstOracle("///\ntest {}");
53 try checkAgainstOracle("///\ncomptime 0");
5354}
5455
5556// Found using AFL++
lib/std/zig/parser_generated_oracle.zig+17-56
......@@ -16,7 +16,7 @@ const Parser = struct {
1616 pub fn parseRoot(p: *Parser) bool {
1717 return blk_0: {
1818 const pos_0 = p.i;
19 if ((p.parseinitial_doc_comment() or true) and p.parseContainerMembers() and p.parseskip() and p.parseeof()) break :blk_0 true;
19 if (p.parseContainerMembers() and p.parseskip() and p.parseeof()) break :blk_0 true;
2020 p.i = pos_0;
2121 break :blk_0 false;
2222 };
......@@ -1764,6 +1764,14 @@ const Parser = struct {
17641764 break :blk_0 false;
17651765 };
17661766 }
1767 pub fn parsesof(p: *Parser) bool {
1768 return blk_0: {
1769 const pos_0 = p.i;
1770 if ((p.i == 0)) break :blk_0 true;
1771 p.i = pos_0;
1772 break :blk_0 false;
1773 };
1774 }
17671775 pub fn parseeof(p: *Parser) bool {
17681776 return blk_0: {
17691777 const pos_0 = p.i;
......@@ -2083,60 +2091,6 @@ const Parser = struct {
20832091 break :blk_0 false;
20842092 };
20852093 }
2086 pub fn parseinitial_doc_comment(p: *Parser) bool {
2087 return blk_0: {
2088 const pos_0 = p.i;
2089 if (blk_1: {
2090 var match_1 = false;
2091 while (blk_3: {
2092 const pos_3 = p.i;
2093 if (p.parseskip() and blk_4: {
2094 if (std.mem.startsWith(u8, p.source[p.i..], "///")) {
2095 p.i += 3;
2096 break :blk_4 true;
2097 }
2098 break :blk_4 false;
2099 } and blk_4: {
2100 while (p.parsenon_control_utf8()) {}
2101 break :blk_4 true;
2102 } and p.parsenewline()) break :blk_3 true;
2103 p.i = pos_3;
2104 break :blk_3 false;
2105 }) {
2106 match_1 = true;
2107 }
2108 break :blk_1 match_1;
2109 } and p.parseskip() and blk_1: {
2110 const pos_1 = p.i;
2111 const match_1 = blk_2: {
2112 if (std.mem.startsWith(u8, p.source[p.i..], "//!")) {
2113 p.i += 3;
2114 break :blk_2 true;
2115 }
2116 break :blk_2 false;
2117 };
2118 p.i = pos_1;
2119 break :blk_1 !match_1;
2120 } and blk_1: {
2121 const pos_1 = p.i;
2122 const match_1 = p.parseKEYWORD_test();
2123 p.i = pos_1;
2124 break :blk_1 !match_1;
2125 } and blk_1: {
2126 const pos_1 = p.i;
2127 const match_1 = p.parseKEYWORD_comptime();
2128 p.i = pos_1;
2129 break :blk_1 !match_1;
2130 } and blk_1: {
2131 const pos_1 = p.i;
2132 const match_1 = p.parseeof();
2133 p.i = pos_1;
2134 break :blk_1 !match_1;
2135 }) break :blk_0 true;
2136 p.i = pos_0;
2137 break :blk_0 false;
2138 };
2139 }
21402094 pub fn parsecontainer_doc_comment(p: *Parser) bool {
21412095 return blk_0: {
21422096 const pos_0 = p.i;
......@@ -2168,7 +2122,14 @@ const Parser = struct {
21682122 pub fn parsedoc_comment(p: *Parser) bool {
21692123 return blk_0: {
21702124 const pos_0 = p.i;
2171 if (p.parseskip_require_newline() and blk_1: {
2125 if (blk_2: {
2126 const pos_2 = p.i;
2127 if (p.parsesof()) break :blk_2 true;
2128 p.i = pos_2;
2129 if (p.parseskip_require_newline()) break :blk_2 true;
2130 p.i = pos_2;
2131 break :blk_2 false;
2132 } and blk_1: {
21722133 var match_1 = false;
21732134 while (blk_3: {
21742135 const pos_3 = p.i;
tools/gen_parser_oracle.zig+13-1
......@@ -242,6 +242,7 @@ const Generator = struct {
242242 \\}})
243243 , .{ suffix, suffix });
244244 },
245 .sof => try g.w.writeAll("(p.i == 0)"),
245246 else => unreachable,
246247 }
247248 }
......@@ -337,6 +338,8 @@ const Parser = struct {
337338 start: u8,
338339 end: u8,
339340 },
341 /// Start of file
342 sof,
340343
341344 const Index = enum(u32) {
342345 _,
......@@ -422,8 +425,17 @@ const Parser = struct {
422425 // Prefix <- AND Action
423426 // / ( AND / NOT )? Suffix
424427 fn parsePrefix(p: *Parser) !?Node.Index {
425 // We don't implement Action
426428 if (p.eatAnd()) {
429 // We only support a single hardcoded "start of file" Action
430 if (p.eat('{')) {
431 // Action <- '{' < [^}]* > '}' Spacing
432 if (std.mem.startsWith(u8, p.source[p.i..], " (yy->__pos == 0) }")) {
433 while (!p.eat('}')) p.i += 1;
434 _ = p.eatSpacing();
435 return try p.addNode(.sof);
436 }
437 return null;
438 }
427439 const suffix = try p.parseSuffix() orelse return null;
428440 return try p.addNode(.{ .@"&" = suffix });
429441 }