Commit f8f1c6ac06636d816e5dff205d6713a337df387d | zig

author	Manlio Perillo <manlio.perillo@gmail.com> 2022-12-29 07:22:35+01:00
committer	Veikka Tuominen <git@vexu.eu> 2022-12-30 17:11:59+02:00
log	f8f1c6ac06636d816e5dff205d6713a337df387d
tree	567433e6e3321862260db5b610c33f87788b1f80
parent	4b5fc5239c37fa3f49965188256a781c7b1277a3

parser: ensure the documented grammar matches grammar.y

Move the parsing of Root from the parse function to the Parser.parseRoot
method.

Add an empty line between each rule, for consistency with grammar.y.

Ensure that normal doc-comments are always on the top, for consistency.

Add extract-grammar.zig to the tools directory; it is used to extract
the PEG grammar from parse.zig, so that it can be compared with
grammar.y.

2 files changed, 174 insertions(+), 26 deletions(-)

lib
- std
  - zig
    parse.zig
tools
- extract-grammar.zig

lib/std/zig/parse.zig+74-26

 @@ -50,22 +50,7 @@ pub fn parse(gpa: Allocator, source: [:0]const u8) Allocator.Error!Ast {
     const estimated_node_count = (tokens.len + 2) / 2;
     try parser.nodes.ensureTotalCapacity(gpa, estimated_node_count);
-    // Root node must be index 0.
+    try parser.parseRoot();
-    // Root <- skip ContainerMembers eof
-    parser.nodes.appendAssumeCapacity(.{
-        .tag = .root,
-        .main_token = 0,
-        .data = undefined,
-    });
-    const root_members = try parser.parseContainerMembers();
-    const root_decls = try root_members.toSpan(&parser);
-    if (parser.token_tags[parser.tok_i] != .eof) {
-        try parser.warnExpected(.eof);
-    }
-    parser.nodes.items(.data)[0] = .{
-        .lhs = root_decls.start,
-        .rhs = root_decls.end,
-    };
     // TODO experiment with compacting the MultiArrayList slices here
     return Ast{
 @@ -237,12 +222,33 @@ const Parser = struct {
         return error.ParseError;
     }
+    /// Root <- skip container_doc_comment? ContainerMembers eof
+    fn parseRoot(p: *Parser) !void {
+        // Root node must be index 0.
+        p.nodes.appendAssumeCapacity(.{
+            .tag = .root,
+            .main_token = 0,
+            .data = undefined,
+        });
+        const root_members = try p.parseContainerMembers();
+        const root_decls = try root_members.toSpan(p);
+        if (p.token_tags[p.tok_i] != .eof) {
+            try p.warnExpected(.eof);
+        }
+        p.nodes.items(.data)[0] = .{
+            .lhs = root_decls.start,
+            .rhs = root_decls.end,
+        };
+    }
     /// ContainerMembers <- ContainerDeclarations (ContainerField COMMA)* (ContainerField / ContainerDeclarations)
+    ///
     /// ContainerDeclarations
     ///     <- TestDecl ContainerDeclarations
     ///      / ComptimeDecl ContainerDeclarations
     ///      / doc_comment? KEYWORD_pub? Decl ContainerDeclarations
     ///      /
+    ///
     /// ComptimeDecl <- KEYWORD_comptime Block
     fn parseContainerMembers(p: *Parser) !Members {
         const scratch_top = p.scratch.items.len;
 @@ -887,7 +893,9 @@ const Parser = struct {
         }
     }
-    /// ContainerField <- KEYWORD_comptime? IDENTIFIER (COLON TypeExpr ByteAlign?)? (EQUAL Expr)?
+    /// ContainerField
+    ///     <- doc_comment? KEYWORD_comptime? IDENTIFIER (COLON TypeExpr)? ByteAlign? (EQUAL Expr)?
+    ///      / doc_comment? KEYWORD_comptime? (IDENTIFIER COLON)? !KEYWORD_fn TypeExpr ByteAlign? (EQUAL Expr)?
     fn expectContainerField(p: *Parser) !Node.Index {
         var main_token = p.tok_i;
         _ = p.eatToken(.keyword_comptime);
 @@ -1173,6 +1181,7 @@ const Parser = struct {
     }
     /// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload
+    ///
     /// ForStatement
     ///     <- ForPrefix BlockExpr ( KEYWORD_else Statement )?
     ///      / ForPrefix AssignExpr ( SEMICOLON / KEYWORD_else Statement )
 @@ -1234,6 +1243,7 @@ const Parser = struct {
     }
     /// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr?
+    ///
     /// WhileStatement
     ///     <- WhilePrefix BlockExpr ( KEYWORD_else Payload? Statement )?
     ///      / WhilePrefix AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement )
 @@ -1368,13 +1378,18 @@ const Parser = struct {
     }
     /// AssignExpr <- Expr (AssignOp Expr)?
+    ///
     /// AssignOp
     ///     <- ASTERISKEQUAL
+    ///      / ASTERISKPIPEEQUAL
     ///      / SLASHEQUAL
     ///      / PERCENTEQUAL
     ///      / PLUSEQUAL
+    ///      / PLUSPIPEEQUAL
     ///      / MINUSEQUAL
+    ///      / MINUSPIPEEQUAL
     ///      / LARROW2EQUAL
+    ///      / LARROW2PIPEEQUAL
     ///      / RARROW2EQUAL
     ///      / AMPERSANDEQUAL
     ///      / CARETEQUAL
 @@ -1553,6 +1568,7 @@ const Parser = struct {
     }
     /// PrefixExpr <- PrefixOp* PrimaryExpr
+    ///
     /// PrefixOp
     ///     <- EXCLAMATIONMARK
     ///      / MINUS
 @@ -1591,17 +1607,21 @@ const Parser = struct {
     }
     /// TypeExpr <- PrefixTypeOp* ErrorUnionExpr
+    ///
     /// PrefixTypeOp
     ///     <- QUESTIONMARK
     ///      / KEYWORD_anyframe MINUSRARROW
     ///      / SliceTypeStart (ByteAlign / AddrSpace / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)*
     ///      / PtrTypeStart (AddrSpace / KEYWORD_align LPAREN Expr (COLON Expr COLON Expr)? RPAREN / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)*
     ///      / ArrayTypeStart
+    ///
     /// SliceTypeStart <- LBRACKET (COLON Expr)? RBRACKET
+    ///
     /// PtrTypeStart
     ///     <- ASTERISK
     ///      / ASTERISK2
     ///      / LBRACKET ASTERISK (LETTERC / COLON Expr)? RBRACKET
+    ///
     /// ArrayTypeStart <- LBRACKET Expr (COLON Expr)? RBRACKET
     fn parseTypeExpr(p: *Parser) Error!Node.Index {
         switch (p.token_tags[p.tok_i]) {
 @@ -2068,6 +2088,7 @@ const Parser = struct {
     }
     /// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload
+    ///
     /// ForExpr <- ForPrefix Expr (KEYWORD_else Expr)?
     fn parseForExpr(p: *Parser) !Node.Index {
         const for_token = p.eatToken(.keyword_for) orelse return null_node;
 @@ -2103,6 +2124,7 @@ const Parser = struct {
     }
     /// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr?
+    ///
     /// WhileExpr <- WhilePrefix Expr (KEYWORD_else Payload? Expr)?
     fn parseWhileExpr(p: *Parser) !Node.Index {
         const while_token = p.eatToken(.keyword_while) orelse return null_node;
 @@ -2154,6 +2176,7 @@ const Parser = struct {
     }
     /// CurlySuffixExpr <- TypeExpr InitList?
+    ///
     /// InitList
     ///     <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE
     ///      / LBRACE Expr (COMMA Expr)* COMMA? RBRACE
 @@ -2272,7 +2295,9 @@ const Parser = struct {
     /// SuffixExpr
     ///     <- KEYWORD_async PrimaryTypeExpr SuffixOp* FnCallArguments
     ///      / PrimaryTypeExpr (SuffixOp / FnCallArguments)*
+    ///
     /// FnCallArguments <- LPAREN ExprList RPAREN
+    ///
     /// ExprList <- (Expr COMMA)* Expr?
     fn parseSuffixExpr(p: *Parser) !Node.Index {
         if (p.eatToken(.keyword_async)) |_| {
 @@ -2410,18 +2435,26 @@ const Parser = struct {
     ///      / KEYWORD_unreachable
     ///      / STRINGLITERAL
     ///      / SwitchExpr
+    ///
     /// ContainerDecl <- (KEYWORD_extern / KEYWORD_packed)? ContainerDeclAuto
-    /// ContainerDeclAuto <- ContainerDeclType LBRACE ContainerMembers RBRACE
+    ///
+    /// ContainerDeclAuto <- ContainerDeclType LBRACE container_doc_comment? ContainerMembers RBRACE
+    ///
     /// InitList
     ///     <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE
     ///      / LBRACE Expr (COMMA Expr)* COMMA? RBRACE
     ///      / LBRACE RBRACE
+    ///
     /// ErrorSetDecl <- KEYWORD_error LBRACE IdentifierList RBRACE
+    ///
     /// GroupedExpr <- LPAREN Expr RPAREN
+    ///
     /// IfTypeExpr <- IfPrefix TypeExpr (KEYWORD_else Payload? TypeExpr)?
+    ///
     /// LabeledTypeExpr
     ///     <- BlockLabel Block
     ///      / BlockLabel? LoopTypeExpr
+    ///
     /// LoopTypeExpr <- KEYWORD_inline? (ForTypeExpr / WhileTypeExpr)
     fn parsePrimaryTypeExpr(p: *Parser) !Node.Index {
         switch (p.token_tags[p.tok_i]) {
 @@ -2751,6 +2784,7 @@ const Parser = struct {
     }
     /// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload
+    ///
     /// ForTypeExpr <- ForPrefix TypeExpr (KEYWORD_else TypeExpr)?
     fn parseForTypeExpr(p: *Parser) !Node.Index {
         const for_token = p.eatToken(.keyword_for) orelse return null_node;
 @@ -2786,6 +2820,7 @@ const Parser = struct {
     }
     /// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr?
+    ///
     /// WhileTypeExpr <- WhilePrefix TypeExpr (KEYWORD_else Payload? TypeExpr)?
     fn parseWhileTypeExpr(p: *Parser) !Node.Index {
         const while_token = p.eatToken(.keyword_while) orelse return null_node;
 @@ -2861,11 +2896,17 @@ const Parser = struct {
     }
     /// AsmExpr <- KEYWORD_asm KEYWORD_volatile? LPAREN Expr AsmOutput? RPAREN
+    ///
     /// AsmOutput <- COLON AsmOutputList AsmInput?
+    ///
     /// AsmInput <- COLON AsmInputList AsmClobbers?
+    ///
     /// AsmClobbers <- COLON StringList
+    ///
     /// StringList <- (STRINGLITERAL COMMA)* STRINGLITERAL?
+    ///
     /// AsmOutputList <- (AsmOutputItem COMMA)* AsmOutputItem?
+    ///
     /// AsmInputList <- (AsmInputItem COMMA)* AsmInputItem?
     fn expectAsmExpr(p: *Parser) !Node.Index {
         const asm_token = p.assertToken(.keyword_asm);
 @@ -3069,15 +3110,17 @@ const Parser = struct {
         return expr_node;
     }
-    /// ParamDecl
-    ///     <- (KEYWORD_noalias / KEYWORD_comptime)? (IDENTIFIER COLON)? ParamType
-    ///     / DOT3
-    /// ParamType
-    ///     <- Keyword_anytype
-    ///      / TypeExpr
     /// This function can return null nodes and then still return nodes afterwards,
     /// such as in the case of anytype and `...`. Caller must look for rparen to find
     /// out when there are no more param decls left.
+    ///
+    /// ParamDecl
+    ///     <- doc_comment? (KEYWORD_noalias / KEYWORD_comptime)? (IDENTIFIER COLON)? ParamType
+    ///      / DOT3
+    ///
+    /// ParamType
+    ///     <- KEYWORD_anytype
+    ///      / TypeExpr
     fn expectParamDecl(p: *Parser) !Node.Index {
         _ = try p.eatDocComments();
         switch (p.token_tags[p.tok_i]) {
 @@ -3119,8 +3162,9 @@ const Parser = struct {
         return identifier;
     }
-    /// PtrIndexPayload <- PIPE ASTERISK? IDENTIFIER (COMMA IDENTIFIER)? PIPE
     /// Returns the first identifier token, if any.
+    ///
+    /// PtrIndexPayload <- PIPE ASTERISK? IDENTIFIER (COMMA IDENTIFIER)? PIPE
     fn parsePtrIndexPayload(p: *Parser) !TokenIndex {
         _ = p.eatToken(.pipe) orelse return @as(TokenIndex, 0);
         _ = p.eatToken(.asterisk);
 @@ -3133,6 +3177,7 @@ const Parser = struct {
     }
     /// SwitchProng <- KEYWORD_inline? SwitchCase EQUALRARROW PtrIndexPayload? AssignExpr
+    ///
     /// SwitchCase
     ///     <- SwitchItem (COMMA SwitchItem)* COMMA?
     ///      / KEYWORD_else
 @@ -3385,6 +3430,7 @@ const Parser = struct {
     }
     /// Caller must have already verified the first token.
+    ///
     /// ContainerDeclAuto <- ContainerDeclType LBRACE container_doc_comment? ContainerMembers RBRACE
     ///
     /// ContainerDeclType
 @@ -3556,6 +3602,7 @@ const Parser = struct {
     }
     /// Holds temporary data until we are ready to construct the full ContainerDecl AST node.
+    ///
     /// ByteAlign <- KEYWORD_align LPAREN Expr RPAREN
     fn parseByteAlign(p: *Parser) !Node.Index {
         _ = p.eatToken(.keyword_align) orelse return null_node;
 @@ -3625,6 +3672,7 @@ const Parser = struct {
     }
     /// FnCallArguments <- LPAREN ExprList RPAREN
+    ///
     /// ExprList <- (Expr COMMA)* Expr?
     fn parseBuiltinCall(p: *Parser) !Node.Index {
         const builtin_token = p.assertToken(.builtin);
 @@ -3698,7 +3746,7 @@ const Parser = struct {
         }
     }
-    /// KEYWORD_if LPAREN Expr RPAREN PtrPayload? Body (KEYWORD_else Payload? Body)?
+    /// IfPrefix <- KEYWORD_if LPAREN Expr RPAREN PtrPayload?
     fn parseIf(p: *Parser, comptime bodyParseFn: fn (p: *Parser) Error!Node.Index) !Node.Index {
         const if_token = p.eatToken(.keyword_if) orelse return null_node;
         _ = try p.expectToken(.l_paren);

tools/extract-grammar.zig created+100

...	@@ -0,0 +1,100 @@
1	//! Extract the "de facto" Zig Grammar from the parser in lib/std/zig/parse.zig.
2	//!
3	//! The generated file must be edited by hand, in order to remove normal doc-comments.
4
5	const std = @import("std");
6	const fs = std.fs;
7	const heap = std.heap;
8	const io = std.io;
9	const mem = std.mem;
10	const process = std.process;
11	const zig = std.zig;
12
13	const Buffer = struct {
14	const buf_size = 4096;
15
16	buf: [buf_size]u8 = undefined,
17	pos: usize = 0,
18
19	pub fn append(self: *Buffer, src: []const u8) !void {
20	if (self.pos + src.len > buf_size) {
21	return error.BufferOverflow;
22	}
23
24	mem.copy(u8, self.buf[self.pos..buf_size], src);
25	self.pos += src.len;
26	}
27
28	pub fn reset(self: *Buffer) void {
29	self.pos = 0;
30	}
31
32	pub fn slice(self: *Buffer) []const u8 {
33	return self.buf[0..self.pos];
34	}
35	};
36
37	/// There are many assumptions in the entire codebase that Zig source files can
38	/// be byte-indexed with a u32 integer.
39	const max_src_size = std.math.maxInt(u32);
40
41	var stdout = io.getStdOut().writer();
42
43	pub fn main() !void {
44	var arena = heap.ArenaAllocator.init(heap.page_allocator);
45	defer arena.deinit(); // NOTE(mperillo): Can be removed.
46
47	const allocator = arena.allocator();
48
49	var args_it = try process.argsWithAllocator(allocator);
50	_ = args_it.skip(); // it is safe to ignore
51
52	const path = args_it.next() orelse return error.SourceFileRequired;
53	const src = try read(path, allocator);
54
55	var tokenizer = zig.Tokenizer.init(src);
56	var buf: Buffer = Buffer{};
57	while (true) {
58	const token = tokenizer.next();
59	switch (token.tag) {
60	.eof => break,
61	.doc_comment => {
62	const line = blk: {
63	// Strip leading whitespace.
64	const len = token.loc.end - token.loc.start;
65	break :blk if (len == 3) src[token.loc.start + 3 .. token.loc.end] else src[token.loc.start + 4 .. token.loc.end];
66	};
67
68	try buf.append(line);
69	try buf.append("\n");
70	},
71	.keyword_fn => {
72	const doc = buf.slice();
73	buf.reset();
74
75	// Check if doc contains a PEG grammar block, so that normal
76	// doc-comments are ignored.
77	if (mem.indexOf(u8, doc, "<-") != null) {
78	// Separate each doc with an empty line. This in turn will
79	// ensure that rules are separate by an empty line.
80	try stdout.print("{s}\n", .{doc});
81	}
82	},
83	else => {},
84	}
85	}
86	}
87
88	fn read(path: []const u8, allocator: mem.Allocator) ![:0]const u8 {
89	var f = try fs.cwd().openFile(path, .{ .mode = .read_only });
90	defer f.close();
91
92	const st = try f.stat();
93	if (st.size > max_src_size) return error.FileTooBig;
94
95	const src = try allocator.allocSentinel(u8, @intCast(usize, st.size), 0);
96	const n = try f.readAll(src);
97	if (n != st.size) return error.UnexpectedEndOfFile;
98
99	return src;
100	}