authorgravatar for manlio.perillo@gmail.comManlio Perillo <manlio.perillo@gmail.com> 2022-12-29 07:22:35+01:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2022-12-30 17:11:59+02:00
logf8f1c6ac06636d816e5dff205d6713a337df387d
tree567433e6e3321862260db5b610c33f87788b1f80
parent4b5fc5239c37fa3f49965188256a781c7b1277a3

parser: ensure the documented grammar matches grammar.y

Move the parsing of Root from the parse function to the Parser.parseRoot method. Add an empty line between each rule, for consistency with grammar.y. Ensure that normal doc-comments are always on the top, for consistency. Add extract-grammar.zig to the tools directory; it is used to extract the PEG grammar from parse.zig, so that it can be compared with grammar.y.

2 files changed, 174 insertions(+), 26 deletions(-)

lib/std/zig/parse.zig+74-26
......@@ -50,22 +50,7 @@ pub fn parse(gpa: Allocator, source: [:0]const u8) Allocator.Error!Ast {
5050 const estimated_node_count = (tokens.len + 2) / 2;
5151 try parser.nodes.ensureTotalCapacity(gpa, estimated_node_count);
5252
53 // Root node must be index 0.
54 // Root <- skip ContainerMembers eof
55 parser.nodes.appendAssumeCapacity(.{
56 .tag = .root,
57 .main_token = 0,
58 .data = undefined,
59 });
60 const root_members = try parser.parseContainerMembers();
61 const root_decls = try root_members.toSpan(&parser);
62 if (parser.token_tags[parser.tok_i] != .eof) {
63 try parser.warnExpected(.eof);
64 }
65 parser.nodes.items(.data)[0] = .{
66 .lhs = root_decls.start,
67 .rhs = root_decls.end,
68 };
53 try parser.parseRoot();
6954
7055 // TODO experiment with compacting the MultiArrayList slices here
7156 return Ast{
......@@ -237,12 +222,33 @@ const Parser = struct {
237222 return error.ParseError;
238223 }
239224
225 /// Root <- skip container_doc_comment? ContainerMembers eof
226 fn parseRoot(p: *Parser) !void {
227 // Root node must be index 0.
228 p.nodes.appendAssumeCapacity(.{
229 .tag = .root,
230 .main_token = 0,
231 .data = undefined,
232 });
233 const root_members = try p.parseContainerMembers();
234 const root_decls = try root_members.toSpan(p);
235 if (p.token_tags[p.tok_i] != .eof) {
236 try p.warnExpected(.eof);
237 }
238 p.nodes.items(.data)[0] = .{
239 .lhs = root_decls.start,
240 .rhs = root_decls.end,
241 };
242 }
243
240244 /// ContainerMembers <- ContainerDeclarations (ContainerField COMMA)* (ContainerField / ContainerDeclarations)
245 ///
241246 /// ContainerDeclarations
242247 /// <- TestDecl ContainerDeclarations
243248 /// / ComptimeDecl ContainerDeclarations
244249 /// / doc_comment? KEYWORD_pub? Decl ContainerDeclarations
245250 /// /
251 ///
246252 /// ComptimeDecl <- KEYWORD_comptime Block
247253 fn parseContainerMembers(p: *Parser) !Members {
248254 const scratch_top = p.scratch.items.len;
......@@ -887,7 +893,9 @@ const Parser = struct {
887893 }
888894 }
889895
890 /// ContainerField <- KEYWORD_comptime? IDENTIFIER (COLON TypeExpr ByteAlign?)? (EQUAL Expr)?
896 /// ContainerField
897 /// <- doc_comment? KEYWORD_comptime? IDENTIFIER (COLON TypeExpr)? ByteAlign? (EQUAL Expr)?
898 /// / doc_comment? KEYWORD_comptime? (IDENTIFIER COLON)? !KEYWORD_fn TypeExpr ByteAlign? (EQUAL Expr)?
891899 fn expectContainerField(p: *Parser) !Node.Index {
892900 var main_token = p.tok_i;
893901 _ = p.eatToken(.keyword_comptime);
......@@ -1173,6 +1181,7 @@ const Parser = struct {
11731181 }
11741182
11751183 /// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload
1184 ///
11761185 /// ForStatement
11771186 /// <- ForPrefix BlockExpr ( KEYWORD_else Statement )?
11781187 /// / ForPrefix AssignExpr ( SEMICOLON / KEYWORD_else Statement )
......@@ -1234,6 +1243,7 @@ const Parser = struct {
12341243 }
12351244
12361245 /// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr?
1246 ///
12371247 /// WhileStatement
12381248 /// <- WhilePrefix BlockExpr ( KEYWORD_else Payload? Statement )?
12391249 /// / WhilePrefix AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement )
......@@ -1368,13 +1378,18 @@ const Parser = struct {
13681378 }
13691379
13701380 /// AssignExpr <- Expr (AssignOp Expr)?
1381 ///
13711382 /// AssignOp
13721383 /// <- ASTERISKEQUAL
1384 /// / ASTERISKPIPEEQUAL
13731385 /// / SLASHEQUAL
13741386 /// / PERCENTEQUAL
13751387 /// / PLUSEQUAL
1388 /// / PLUSPIPEEQUAL
13761389 /// / MINUSEQUAL
1390 /// / MINUSPIPEEQUAL
13771391 /// / LARROW2EQUAL
1392 /// / LARROW2PIPEEQUAL
13781393 /// / RARROW2EQUAL
13791394 /// / AMPERSANDEQUAL
13801395 /// / CARETEQUAL
......@@ -1553,6 +1568,7 @@ const Parser = struct {
15531568 }
15541569
15551570 /// PrefixExpr <- PrefixOp* PrimaryExpr
1571 ///
15561572 /// PrefixOp
15571573 /// <- EXCLAMATIONMARK
15581574 /// / MINUS
......@@ -1591,17 +1607,21 @@ const Parser = struct {
15911607 }
15921608
15931609 /// TypeExpr <- PrefixTypeOp* ErrorUnionExpr
1610 ///
15941611 /// PrefixTypeOp
15951612 /// <- QUESTIONMARK
15961613 /// / KEYWORD_anyframe MINUSRARROW
15971614 /// / SliceTypeStart (ByteAlign / AddrSpace / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)*
15981615 /// / PtrTypeStart (AddrSpace / KEYWORD_align LPAREN Expr (COLON Expr COLON Expr)? RPAREN / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)*
15991616 /// / ArrayTypeStart
1617 ///
16001618 /// SliceTypeStart <- LBRACKET (COLON Expr)? RBRACKET
1619 ///
16011620 /// PtrTypeStart
16021621 /// <- ASTERISK
16031622 /// / ASTERISK2
16041623 /// / LBRACKET ASTERISK (LETTERC / COLON Expr)? RBRACKET
1624 ///
16051625 /// ArrayTypeStart <- LBRACKET Expr (COLON Expr)? RBRACKET
16061626 fn parseTypeExpr(p: *Parser) Error!Node.Index {
16071627 switch (p.token_tags[p.tok_i]) {
......@@ -2068,6 +2088,7 @@ const Parser = struct {
20682088 }
20692089
20702090 /// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload
2091 ///
20712092 /// ForExpr <- ForPrefix Expr (KEYWORD_else Expr)?
20722093 fn parseForExpr(p: *Parser) !Node.Index {
20732094 const for_token = p.eatToken(.keyword_for) orelse return null_node;
......@@ -2103,6 +2124,7 @@ const Parser = struct {
21032124 }
21042125
21052126 /// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr?
2127 ///
21062128 /// WhileExpr <- WhilePrefix Expr (KEYWORD_else Payload? Expr)?
21072129 fn parseWhileExpr(p: *Parser) !Node.Index {
21082130 const while_token = p.eatToken(.keyword_while) orelse return null_node;
......@@ -2154,6 +2176,7 @@ const Parser = struct {
21542176 }
21552177
21562178 /// CurlySuffixExpr <- TypeExpr InitList?
2179 ///
21572180 /// InitList
21582181 /// <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE
21592182 /// / LBRACE Expr (COMMA Expr)* COMMA? RBRACE
......@@ -2272,7 +2295,9 @@ const Parser = struct {
22722295 /// SuffixExpr
22732296 /// <- KEYWORD_async PrimaryTypeExpr SuffixOp* FnCallArguments
22742297 /// / PrimaryTypeExpr (SuffixOp / FnCallArguments)*
2298 ///
22752299 /// FnCallArguments <- LPAREN ExprList RPAREN
2300 ///
22762301 /// ExprList <- (Expr COMMA)* Expr?
22772302 fn parseSuffixExpr(p: *Parser) !Node.Index {
22782303 if (p.eatToken(.keyword_async)) |_| {
......@@ -2410,18 +2435,26 @@ const Parser = struct {
24102435 /// / KEYWORD_unreachable
24112436 /// / STRINGLITERAL
24122437 /// / SwitchExpr
2438 ///
24132439 /// ContainerDecl <- (KEYWORD_extern / KEYWORD_packed)? ContainerDeclAuto
2414 /// ContainerDeclAuto <- ContainerDeclType LBRACE ContainerMembers RBRACE
2440 ///
2441 /// ContainerDeclAuto <- ContainerDeclType LBRACE container_doc_comment? ContainerMembers RBRACE
2442 ///
24152443 /// InitList
24162444 /// <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE
24172445 /// / LBRACE Expr (COMMA Expr)* COMMA? RBRACE
24182446 /// / LBRACE RBRACE
2447 ///
24192448 /// ErrorSetDecl <- KEYWORD_error LBRACE IdentifierList RBRACE
2449 ///
24202450 /// GroupedExpr <- LPAREN Expr RPAREN
2451 ///
24212452 /// IfTypeExpr <- IfPrefix TypeExpr (KEYWORD_else Payload? TypeExpr)?
2453 ///
24222454 /// LabeledTypeExpr
24232455 /// <- BlockLabel Block
24242456 /// / BlockLabel? LoopTypeExpr
2457 ///
24252458 /// LoopTypeExpr <- KEYWORD_inline? (ForTypeExpr / WhileTypeExpr)
24262459 fn parsePrimaryTypeExpr(p: *Parser) !Node.Index {
24272460 switch (p.token_tags[p.tok_i]) {
......@@ -2751,6 +2784,7 @@ const Parser = struct {
27512784 }
27522785
27532786 /// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload
2787 ///
27542788 /// ForTypeExpr <- ForPrefix TypeExpr (KEYWORD_else TypeExpr)?
27552789 fn parseForTypeExpr(p: *Parser) !Node.Index {
27562790 const for_token = p.eatToken(.keyword_for) orelse return null_node;
......@@ -2786,6 +2820,7 @@ const Parser = struct {
27862820 }
27872821
27882822 /// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr?
2823 ///
27892824 /// WhileTypeExpr <- WhilePrefix TypeExpr (KEYWORD_else Payload? TypeExpr)?
27902825 fn parseWhileTypeExpr(p: *Parser) !Node.Index {
27912826 const while_token = p.eatToken(.keyword_while) orelse return null_node;
......@@ -2861,11 +2896,17 @@ const Parser = struct {
28612896 }
28622897
28632898 /// AsmExpr <- KEYWORD_asm KEYWORD_volatile? LPAREN Expr AsmOutput? RPAREN
2899 ///
28642900 /// AsmOutput <- COLON AsmOutputList AsmInput?
2901 ///
28652902 /// AsmInput <- COLON AsmInputList AsmClobbers?
2903 ///
28662904 /// AsmClobbers <- COLON StringList
2905 ///
28672906 /// StringList <- (STRINGLITERAL COMMA)* STRINGLITERAL?
2907 ///
28682908 /// AsmOutputList <- (AsmOutputItem COMMA)* AsmOutputItem?
2909 ///
28692910 /// AsmInputList <- (AsmInputItem COMMA)* AsmInputItem?
28702911 fn expectAsmExpr(p: *Parser) !Node.Index {
28712912 const asm_token = p.assertToken(.keyword_asm);
......@@ -3069,15 +3110,17 @@ const Parser = struct {
30693110 return expr_node;
30703111 }
30713112
3072 /// ParamDecl
3073 /// <- (KEYWORD_noalias / KEYWORD_comptime)? (IDENTIFIER COLON)? ParamType
3074 /// / DOT3
3075 /// ParamType
3076 /// <- Keyword_anytype
3077 /// / TypeExpr
30783113 /// This function can return null nodes and then still return nodes afterwards,
30793114 /// such as in the case of anytype and `...`. Caller must look for rparen to find
30803115 /// out when there are no more param decls left.
3116 ///
3117 /// ParamDecl
3118 /// <- doc_comment? (KEYWORD_noalias / KEYWORD_comptime)? (IDENTIFIER COLON)? ParamType
3119 /// / DOT3
3120 ///
3121 /// ParamType
3122 /// <- KEYWORD_anytype
3123 /// / TypeExpr
30813124 fn expectParamDecl(p: *Parser) !Node.Index {
30823125 _ = try p.eatDocComments();
30833126 switch (p.token_tags[p.tok_i]) {
......@@ -3119,8 +3162,9 @@ const Parser = struct {
31193162 return identifier;
31203163 }
31213164
3122 /// PtrIndexPayload <- PIPE ASTERISK? IDENTIFIER (COMMA IDENTIFIER)? PIPE
31233165 /// Returns the first identifier token, if any.
3166 ///
3167 /// PtrIndexPayload <- PIPE ASTERISK? IDENTIFIER (COMMA IDENTIFIER)? PIPE
31243168 fn parsePtrIndexPayload(p: *Parser) !TokenIndex {
31253169 _ = p.eatToken(.pipe) orelse return @as(TokenIndex, 0);
31263170 _ = p.eatToken(.asterisk);
......@@ -3133,6 +3177,7 @@ const Parser = struct {
31333177 }
31343178
31353179 /// SwitchProng <- KEYWORD_inline? SwitchCase EQUALRARROW PtrIndexPayload? AssignExpr
3180 ///
31363181 /// SwitchCase
31373182 /// <- SwitchItem (COMMA SwitchItem)* COMMA?
31383183 /// / KEYWORD_else
......@@ -3385,6 +3430,7 @@ const Parser = struct {
33853430 }
33863431
33873432 /// Caller must have already verified the first token.
3433 ///
33883434 /// ContainerDeclAuto <- ContainerDeclType LBRACE container_doc_comment? ContainerMembers RBRACE
33893435 ///
33903436 /// ContainerDeclType
......@@ -3556,6 +3602,7 @@ const Parser = struct {
35563602 }
35573603
35583604 /// Holds temporary data until we are ready to construct the full ContainerDecl AST node.
3605 ///
35593606 /// ByteAlign <- KEYWORD_align LPAREN Expr RPAREN
35603607 fn parseByteAlign(p: *Parser) !Node.Index {
35613608 _ = p.eatToken(.keyword_align) orelse return null_node;
......@@ -3625,6 +3672,7 @@ const Parser = struct {
36253672 }
36263673
36273674 /// FnCallArguments <- LPAREN ExprList RPAREN
3675 ///
36283676 /// ExprList <- (Expr COMMA)* Expr?
36293677 fn parseBuiltinCall(p: *Parser) !Node.Index {
36303678 const builtin_token = p.assertToken(.builtin);
......@@ -3698,7 +3746,7 @@ const Parser = struct {
36983746 }
36993747 }
37003748
3701 /// KEYWORD_if LPAREN Expr RPAREN PtrPayload? Body (KEYWORD_else Payload? Body)?
3749 /// IfPrefix <- KEYWORD_if LPAREN Expr RPAREN PtrPayload?
37023750 fn parseIf(p: *Parser, comptime bodyParseFn: fn (p: *Parser) Error!Node.Index) !Node.Index {
37033751 const if_token = p.eatToken(.keyword_if) orelse return null_node;
37043752 _ = try p.expectToken(.l_paren);
tools/extract-grammar.zig created+100
......@@ -0,0 +1,100 @@
1//! Extract the "de facto" Zig Grammar from the parser in lib/std/zig/parse.zig.
2//!
3//! The generated file must be edited by hand, in order to remove normal doc-comments.
4
5const std = @import("std");
6const fs = std.fs;
7const heap = std.heap;
8const io = std.io;
9const mem = std.mem;
10const process = std.process;
11const zig = std.zig;
12
13const Buffer = struct {
14 const buf_size = 4096;
15
16 buf: [buf_size]u8 = undefined,
17 pos: usize = 0,
18
19 pub fn append(self: *Buffer, src: []const u8) !void {
20 if (self.pos + src.len > buf_size) {
21 return error.BufferOverflow;
22 }
23
24 mem.copy(u8, self.buf[self.pos..buf_size], src);
25 self.pos += src.len;
26 }
27
28 pub fn reset(self: *Buffer) void {
29 self.pos = 0;
30 }
31
32 pub fn slice(self: *Buffer) []const u8 {
33 return self.buf[0..self.pos];
34 }
35};
36
37/// There are many assumptions in the entire codebase that Zig source files can
38/// be byte-indexed with a u32 integer.
39const max_src_size = std.math.maxInt(u32);
40
41var stdout = io.getStdOut().writer();
42
43pub fn main() !void {
44 var arena = heap.ArenaAllocator.init(heap.page_allocator);
45 defer arena.deinit(); // NOTE(mperillo): Can be removed.
46
47 const allocator = arena.allocator();
48
49 var args_it = try process.argsWithAllocator(allocator);
50 _ = args_it.skip(); // it is safe to ignore
51
52 const path = args_it.next() orelse return error.SourceFileRequired;
53 const src = try read(path, allocator);
54
55 var tokenizer = zig.Tokenizer.init(src);
56 var buf: Buffer = Buffer{};
57 while (true) {
58 const token = tokenizer.next();
59 switch (token.tag) {
60 .eof => break,
61 .doc_comment => {
62 const line = blk: {
63 // Strip leading whitespace.
64 const len = token.loc.end - token.loc.start;
65 break :blk if (len == 3) src[token.loc.start + 3 .. token.loc.end] else src[token.loc.start + 4 .. token.loc.end];
66 };
67
68 try buf.append(line);
69 try buf.append("\n");
70 },
71 .keyword_fn => {
72 const doc = buf.slice();
73 buf.reset();
74
75 // Check if doc contains a PEG grammar block, so that normal
76 // doc-comments are ignored.
77 if (mem.indexOf(u8, doc, "<-") != null) {
78 // Separate each doc with an empty line. This in turn will
79 // ensure that rules are separate by an empty line.
80 try stdout.print("{s}\n", .{doc});
81 }
82 },
83 else => {},
84 }
85 }
86}
87
88fn read(path: []const u8, allocator: mem.Allocator) ![:0]const u8 {
89 var f = try fs.cwd().openFile(path, .{ .mode = .read_only });
90 defer f.close();
91
92 const st = try f.stat();
93 if (st.size > max_src_size) return error.FileTooBig;
94
95 const src = try allocator.allocSentinel(u8, @intCast(usize, st.size), 0);
96 const n = try f.readAll(src);
97 if (n != st.size) return error.UnexpectedEndOfFile;
98
99 return src;
100}