diff --git a/build.zig b/build.zig index 03774ca9ac707c2c01d8c8d7815aa0aa1cb4bbfd..385d4d5b7e0288d79a07c4c2c314cb7e6bc30c5d 100644 --- a/build.zig +++ b/build.zig @@ -719,6 +719,30 @@ pub fn build(b: *std.Build) !void { check_mingw_run.addDirectoryArg(b.path("lib/libc/mingw")); check_mingw_step.dependOn(&check_mingw_run.step); + { + const gen_oracle_exe = b.addExecutable(.{ + .name = "gen_parser_oracle", + .root_module = b.createModule(.{ + .root_source_file = b.path("tools/gen_parser_oracle.zig"), + .target = b.graph.host, + }), + }); + + const gen_oracle_step = b.step("gen-parser-oracle", "Regenerate lib/std/zig/parser_generated_oracle.zig from doc/langref/grammar.peg"); + const gen_oracle_run = b.addRunArtifact(gen_oracle_exe); + gen_oracle_run.addFileArg(b.path("doc/langref/grammar.peg")); + gen_oracle_run.addFileArg(b.path("lib/std/zig/parser_generated_oracle.zig")); + gen_oracle_step.dependOn(&gen_oracle_run.step); + + const check_oracle_step = b.step("check-parser-oracle", "Check if doc/langref/grammar.peg was modified without regenerating the oracle"); + const check_oracle_run = b.addRunArtifact(gen_oracle_exe); + check_oracle_run.addFileArg(b.path("doc/langref/grammar.peg")); + check_oracle_run.addFileArg(b.path("lib/std/zig/parser_generated_oracle.zig")); + check_oracle_run.addArg("--check"); + check_oracle_step.dependOn(&check_oracle_run.step); + test_step.dependOn(check_oracle_step); + } + const test_incremental_step = b.step("test-incremental", "Run the incremental compilation test cases"); try tests.addIncrementalTests(b, test_incremental_step, test_filters); if (!skip_test_incremental) test_step.dependOn(test_incremental_step); @@ -1569,8 +1593,9 @@ fn generateLangRef(b: *std.Build) !std.Build.LazyPath { var it = dir.iterateAssumeFirstIteration(); while (it.next(io) catch @panic("failed to read dir")) |entry| { - if (std.mem.startsWith(u8, entry.name, ".") or entry.kind != .file) - continue; + if (entry.kind != .file) continue; + if (std.mem.startsWith(u8, entry.name, ".")) continue; + if (!std.mem.endsWith(u8, entry.name, ".zig")) continue; const out_basename = b.fmt("{s}.out", .{std.fs.path.stem(entry.name)}); const cmd = b.addRunArtifact(doctest_exe); @@ -1603,6 +1628,8 @@ fn generateLangRef(b: *std.Build) !std.Build.LazyPath { const docgen_cmd = b.addRunArtifact(docgen_exe); docgen_cmd.addArgs(&.{"--code-dir"}); docgen_cmd.addDirectoryArg(wf.getDirectory()); + docgen_cmd.addArgs(&.{"--grammar"}); + docgen_cmd.addFileArg(b.path("doc/langref/grammar.peg")); docgen_cmd.addFileArg(b.path("doc/langref.html.in")); return docgen_cmd.addOutputFileArg("langref.html"); diff --git a/doc/langref.html.in b/doc/langref.html.in index f24f03c981f5d78513468aa31c3983b06ae2670d..4766e8d508d31b76b6258501ef1226f7d809c1e1 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -7798,597 +7798,7 @@ fn readU32Be() u32 {} {#header_open|Appendix#} {#header_open|Grammar#} - {#syntax_block|peg|grammar.peg#} -Root <- skip ContainerMembers eof - -# *** Top level *** -ContainerMembers <- container_doc_comment? ContainerDeclaration* (ContainerField COMMA)* (ContainerField / ContainerDeclaration*) - -ContainerDeclaration <- TestDecl / ComptimeDecl / doc_comment? KEYWORD_pub? Decl - -TestDecl <- KEYWORD_test (STRINGLITERALSINGLE / IDENTIFIER)? Block - -ComptimeDecl <- KEYWORD_comptime Block - -Decl - <- (KEYWORD_export / KEYWORD_inline / KEYWORD_noinline)? FnProto (SEMICOLON / Block) - / KEYWORD_extern STRINGLITERALSINGLE? FnProto SEMICOLON - / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? GlobalVarDecl - -FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? AddrSpace? LinkSection? CallConv? EXCLAMATIONMARK? TypeExpr !ExprSuffix - -VarDeclProto <- (KEYWORD_const / KEYWORD_var) IDENTIFIER (COLON TypeExpr)? ByteAlign? AddrSpace? LinkSection? - -GlobalVarDecl <- VarDeclProto (EQUAL Expr)? SEMICOLON - -ContainerField <- doc_comment? (KEYWORD_comptime / !KEYWORD_comptime) !KEYWORD_fn (IDENTIFIER COLON / !(IDENTIFIER COLON))? TypeExpr ByteAlign? (EQUAL Expr)? - -# *** Block Level *** -BlockStatement - <- Statement - / KEYWORD_defer BlockExprStatement - / KEYWORD_errdefer BlockExprStatement - / !ExprStatement (KEYWORD_comptime !BlockExpr)? VarAssignStatement - -Statement - <- ExprStatement - / KEYWORD_suspend BlockExprStatement - / !ExprStatement (KEYWORD_comptime !BlockExpr)? AssignExpr SEMICOLON - -ExprStatement - <- IfStatement - / LabeledStatement - / KEYWORD_nosuspend BlockExprStatement - / KEYWORD_comptime BlockExpr - -IfStatement - <- IfPrefix BlockExpr ( KEYWORD_else Payload? Statement )? - / IfPrefix !BlockExpr AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement ) - -LabeledStatement <- BlockLabel? (Block / LoopStatement / SwitchExpr) - -LoopStatement <- KEYWORD_inline? (ForStatement / WhileStatement) - -ForStatement - <- ForPrefix BlockExpr ( KEYWORD_else Statement / !KEYWORD_else ) - / ForPrefix !BlockExpr AssignExpr ( SEMICOLON / KEYWORD_else Statement ) - -WhileStatement - <- WhilePrefix BlockExpr ( KEYWORD_else Payload? Statement )? - / WhilePrefix !BlockExpr AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement ) - -BlockExprStatement - <- BlockExpr - / !BlockExpr AssignExpr SEMICOLON - -BlockExpr <- BlockLabel? Block - -# An assignment or a destructure whose LHS are all lvalue expressions or variable declarations. -VarAssignStatement <- (VarDeclProto / Expr) (COMMA (VarDeclProto / Expr))* EQUAL Expr SEMICOLON - -# *** Expression Level *** - -# An assignment or a destructure whose LHS are all lvalue expressions. -AssignExpr <- Expr (AssignOp Expr / (COMMA Expr)+ EQUAL Expr)? - -SingleAssignExpr <- Expr (AssignOp Expr)? - -Expr <- BoolOrExpr - -BoolOrExpr <- BoolAndExpr (KEYWORD_or BoolAndExpr)* - -BoolAndExpr <- CompareExpr (KEYWORD_and CompareExpr)* - -CompareExpr <- BitwiseExpr (CompareOp BitwiseExpr)? - -BitwiseExpr <- BitShiftExpr (BitwiseOp BitShiftExpr)* - -BitShiftExpr <- AdditionExpr (BitShiftOp AdditionExpr)* - -AdditionExpr <- MultiplyExpr (AdditionOp MultiplyExpr)* - -MultiplyExpr <- PrefixExpr (MultiplyOp PrefixExpr)* - -PrefixExpr <- PrefixOp* PrimaryExpr - -PrimaryExpr - <- AsmExpr - / IfExpr - / KEYWORD_break (BreakLabel / !BreakLabel) (Expr !ExprSuffix / !SinglePtrTypeStart) - / KEYWORD_comptime Expr !ExprSuffix - / KEYWORD_nosuspend Expr !ExprSuffix - / KEYWORD_continue (BreakLabel / !BreakLabel) (Expr !ExprSuffix / !SinglePtrTypeStart) - / KEYWORD_resume Expr !ExprSuffix - / KEYWORD_return (Expr !ExprSuffix / !SinglePtrTypeStart) - / BlockLabel? LoopExpr - / Block - / CurlySuffixExpr - -IfExpr <- IfPrefix Expr (KEYWORD_else Payload? Expr)? !ExprSuffix - -Block <- LBRACE BlockStatement* RBRACE - -LoopExpr <- KEYWORD_inline? (ForExpr / WhileExpr) - -ForExpr <- ForPrefix Expr (KEYWORD_else Expr / !KEYWORD_else) !ExprSuffix - -WhileExpr <- WhilePrefix Expr (KEYWORD_else Payload? Expr)? !ExprSuffix - -CurlySuffixExpr <- TypeExpr InitList? - -InitList - <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE - / LBRACE Expr (COMMA Expr)* COMMA? RBRACE - / LBRACE RBRACE - -TypeExpr <- PrefixTypeOp* ErrorUnionExpr - -ErrorUnionExpr <- SuffixExpr (EXCLAMATIONMARK TypeExpr)? - -SuffixExpr - <- PrimaryTypeExpr (SuffixOp / FnCallArguments)* - -PrimaryTypeExpr - <- BUILTINIDENTIFIER FnCallArguments - / CHAR_LITERAL - / ContainerDecl - / DOT IDENTIFIER - / DOT InitList - / ErrorSetDecl - / FLOAT - / FnProto - / GroupedExpr - / LabeledTypeExpr - / IDENTIFIER !(COLON LabelableExpr) - / IfTypeExpr - / INTEGER - / KEYWORD_comptime TypeExpr !ExprSuffix - / KEYWORD_error DOT IDENTIFIER - / KEYWORD_anyframe - / KEYWORD_unreachable - / STRINGLITERAL - -ContainerDecl <- (KEYWORD_extern / KEYWORD_packed)? ContainerDeclAuto - -ErrorSetDecl <- KEYWORD_error LBRACE IdentifierList RBRACE - -GroupedExpr <- LPAREN Expr RPAREN - -IfTypeExpr <- IfPrefix TypeExpr (KEYWORD_else Payload? TypeExpr)? !ExprSuffix - -LabeledTypeExpr - <- BlockLabel Block - / BlockLabel? LoopTypeExpr - / BlockLabel? SwitchExpr - -LoopTypeExpr <- KEYWORD_inline? (ForTypeExpr / WhileTypeExpr) - -ForTypeExpr <- ForPrefix TypeExpr (KEYWORD_else TypeExpr / !KEYWORD_else) !ExprSuffix - -WhileTypeExpr <- WhilePrefix TypeExpr (KEYWORD_else Payload? TypeExpr)? !ExprSuffix - -SwitchExpr <- KEYWORD_switch LPAREN Expr RPAREN LBRACE SwitchProngList RBRACE - -# *** Assembly *** -AsmExpr <- KEYWORD_asm KEYWORD_volatile? LPAREN Expr AsmOutput? RPAREN - -AsmOutput <- COLON AsmOutputList AsmInput? - -AsmOutputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERALSINGLE LPAREN (MINUSRARROW TypeExpr / IDENTIFIER) RPAREN - -AsmInput <- COLON AsmInputList AsmClobbers? - -AsmInputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERALSINGLE LPAREN Expr RPAREN - -AsmClobbers <- COLON Expr - -# *** Helper grammar *** -BreakLabel <- COLON IDENTIFIER - -BlockLabel <- IDENTIFIER COLON - -FieldInit <- DOT IDENTIFIER EQUAL Expr - -WhileContinueExpr <- COLON LPAREN AssignExpr RPAREN - -LinkSection <- KEYWORD_linksection LPAREN Expr RPAREN - -AddrSpace <- KEYWORD_addrspace LPAREN Expr RPAREN - -# Fn specific -CallConv <- KEYWORD_callconv LPAREN Expr RPAREN - -ParamDecl <- doc_comment? (KEYWORD_noalias / KEYWORD_comptime / !KEYWORD_comptime) (IDENTIFIER COLON / !(IDENTIFIER COLON)) ParamType - -ParamType - <- KEYWORD_anytype - / TypeExpr - -# Control flow prefixes -IfPrefix <- KEYWORD_if LPAREN Expr RPAREN PtrPayload? - -WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr? - -ForPrefix <- KEYWORD_for LPAREN ForArgumentsList RPAREN PtrListPayload - -# Payloads -Payload <- PIPE IDENTIFIER PIPE - -PtrPayload <- PIPE ASTERISK? IDENTIFIER PIPE - -PtrIndexPayload <- PIPE ASTERISK? IDENTIFIER (COMMA IDENTIFIER)? PIPE - -PtrListPayload <- PIPE ASTERISK? IDENTIFIER (COMMA ASTERISK? IDENTIFIER)* COMMA? PIPE - -# Switch specific -SwitchProng <- KEYWORD_inline? SwitchCase EQUALRARROW PtrIndexPayload? SingleAssignExpr - -SwitchCase - <- SwitchItem (COMMA SwitchItem)* COMMA? - / KEYWORD_else - -SwitchItem <- Expr (DOT3 Expr)? - -# For specific -ForArgumentsList <- ForItem (COMMA ForItem)* COMMA? - -ForItem <- Expr (DOT2 Expr?)? - -# Operators -AssignOp - <- ASTERISKEQUAL - / ASTERISKPIPEEQUAL - / SLASHEQUAL - / PERCENTEQUAL - / PLUSEQUAL - / PLUSPIPEEQUAL - / MINUSEQUAL - / MINUSPIPEEQUAL - / LARROW2EQUAL - / LARROW2PIPEEQUAL - / RARROW2EQUAL - / AMPERSANDEQUAL - / CARETEQUAL - / PIPEEQUAL - / ASTERISKPERCENTEQUAL - / PLUSPERCENTEQUAL - / MINUSPERCENTEQUAL - / EQUAL - -CompareOp - <- EQUALEQUAL - / EXCLAMATIONMARKEQUAL - / LARROW - / RARROW - / LARROWEQUAL - / RARROWEQUAL - -BitwiseOp - <- AMPERSAND - / CARET - / PIPE - / KEYWORD_orelse - / KEYWORD_catch Payload? - -BitShiftOp - <- LARROW2 - / RARROW2 - / LARROW2PIPE - -AdditionOp - <- PLUS - / MINUS - / PLUS2 - / PLUSPERCENT - / MINUSPERCENT - / PLUSPIPE - / MINUSPIPE - -MultiplyOp - <- PIPE2 - / ASTERISK - / SLASH - / PERCENT - / ASTERISKPERCENT - / ASTERISKPIPE - -PrefixOp - <- EXCLAMATIONMARK - / MINUS - / TILDE - / MINUSPERCENT - / AMPERSAND - / KEYWORD_try - -PrefixTypeOp - <- QUESTIONMARK - / KEYWORD_anyframe MINUSRARROW - / (ManyPtrTypeStart / SliceTypeStart) KEYWORD_allowzero? ByteAlign? AddrSpace? KEYWORD_const? KEYWORD_volatile? - / SinglePtrTypeStart KEYWORD_allowzero? BitAlign? AddrSpace? KEYWORD_const? KEYWORD_volatile? - / ArrayTypeStart - -SuffixOp - <- LBRACKET Expr (DOT2 Expr? (COLON Expr)?)? RBRACKET - / DOT IDENTIFIER - / DOTASTERISK - / DOTQUESTIONMARK - -FnCallArguments <- LPAREN ExprList RPAREN - -ExprSuffix - <- KEYWORD_or - / KEYWORD_and - / CompareOp - / BitwiseOp - / BitShiftOp - / AdditionOp - / MultiplyOp - / EXCLAMATIONMARK - / SuffixOp - / FnCallArguments - -LabelableExpr - <- Block - / SwitchExpr - / LoopExpr - -# Ptr specific -SliceTypeStart <- LBRACKET (COLON Expr)? RBRACKET - -SinglePtrTypeStart <- ASTERISK - -ManyPtrTypeStart <- LBRACKET ASTERISK (LETTERC / COLON Expr)? RBRACKET - -ArrayTypeStart <- LBRACKET Expr !ASTERISK (COLON Expr)? RBRACKET - -# ContainerDecl specific -ContainerDeclAuto <- ContainerDeclType LBRACE ContainerMembers RBRACE - -ContainerDeclType - <- KEYWORD_struct (LPAREN Expr RPAREN)? - / KEYWORD_opaque - / KEYWORD_enum (LPAREN Expr RPAREN)? - / KEYWORD_union (LPAREN (KEYWORD_enum (LPAREN Expr RPAREN)? / !KEYWORD_enum Expr) RPAREN)? - -# Alignment -ByteAlign <- KEYWORD_align LPAREN Expr RPAREN - -BitAlign <- KEYWORD_align LPAREN Expr (COLON Expr COLON Expr)? RPAREN - -# Lists -IdentifierList <- (doc_comment? IDENTIFIER COMMA)* (doc_comment? IDENTIFIER)? - -SwitchProngList <- (SwitchProng COMMA)* SwitchProng? - -AsmOutputList <- (AsmOutputItem COMMA)* AsmOutputItem? - -AsmInputList <- (AsmInputItem COMMA)* AsmInputItem? - -ParamDeclList <- (ParamDecl COMMA)* (ParamDecl / DOT3 COMMA?)? - -ExprList <- (Expr COMMA)* Expr? - -# *** Tokens *** -eof <- !. -bin <- [01] -bin_ <- '_'? bin -oct <- [0-7] -oct_ <- '_'? oct -hex <- [0-9a-fA-F] -hex_ <- '_'? hex -dec <- [0-9] -dec_ <- '_'? dec - -bin_int <- bin bin_* -oct_int <- oct oct_* -dec_int <- dec dec_* -hex_int <- hex hex_* - -ox80_oxBF <- [\200-\277] -oxF4 <- '\364' -ox80_ox8F <- [\200-\217] -oxF1_oxF3 <- [\361-\363] -oxF0 <- '\360' -ox90_0xBF <- [\220-\277] -oxEE_oxEF <- [\356-\357] -oxED <- '\355' -ox80_ox9F <- [\200-\237] -oxE1_oxEC <- [\341-\354] -oxE0 <- '\340' -oxA0_oxBF <- [\240-\277] -oxC2_oxDF <- [\302-\337] - -# From https://lemire.me/blog/2018/05/09/how-quickly-can-you-check-that-a-string-is-valid-unicode-utf-8/ -# First Byte Second Byte Third Byte Fourth Byte -# [0x00,0x7F] -# [0xC2,0xDF] [0x80,0xBF] -# 0xE0 [0xA0,0xBF] [0x80,0xBF] -# [0xE1,0xEC] [0x80,0xBF] [0x80,0xBF] -# 0xED [0x80,0x9F] [0x80,0xBF] -# [0xEE,0xEF] [0x80,0xBF] [0x80,0xBF] -# 0xF0 [0x90,0xBF] [0x80,0xBF] [0x80,0xBF] -# [0xF1,0xF3] [0x80,0xBF] [0x80,0xBF] [0x80,0xBF] -# 0xF4 [0x80,0x8F] [0x80,0xBF] [0x80,0xBF] - -multibyte_utf8 <- - oxF4 ox80_ox8F ox80_oxBF ox80_oxBF - / oxF1_oxF3 ox80_oxBF ox80_oxBF ox80_oxBF - / oxF0 ox90_0xBF ox80_oxBF ox80_oxBF - / oxEE_oxEF ox80_oxBF ox80_oxBF - / oxED ox80_ox9F ox80_oxBF - / oxE1_oxEC ox80_oxBF ox80_oxBF - / oxE0 oxA0_oxBF ox80_oxBF - / oxC2_oxDF ox80_oxBF - -non_control_ascii <- [\040-\176] -non_control_utf8 <- [\040-\377] - -char_escape - <- "\\x" hex hex - / "\\u{" hex+ "}" - / "\\" [nr\\t'"] -char_char - <- multibyte_utf8 - / char_escape - / ![\\'\n] non_control_ascii - -string_char - <- multibyte_utf8 - / char_escape - / ![\\"\n] non_control_ascii - -container_doc_comment <- ('//!' non_control_utf8* [ \n]* skip)+ -doc_comment <- ('///' non_control_utf8* [ \n]* skip)+ -line_comment <- '//' ![!/] non_control_utf8* / '////' non_control_utf8* -line_string <- '\\\\' non_control_utf8* [ \n]* -skip <- ([ \n] / line_comment)* - -CHAR_LITERAL <- ['] char_char ['] skip -FLOAT - <- '0x' hex_int '.' hex_int ([pP] [-+]? dec_int)? skip - / dec_int '.' dec_int ([eE] [-+]? dec_int)? skip - / '0x' hex_int [pP] [-+]? dec_int skip - / dec_int [eE] [-+]? dec_int skip -INTEGER - <- '0b' bin_int skip - / '0o' oct_int skip - / '0x' hex_int skip - / dec_int skip -STRINGLITERALSINGLE <- ["] string_char* ["] skip -STRINGLITERAL - <- STRINGLITERALSINGLE - / (line_string skip)+ -IDENTIFIER - <- !keyword [A-Za-z_] [A-Za-z0-9_]* skip - / '@' STRINGLITERALSINGLE -BUILTINIDENTIFIER <- '@'[A-Za-z_][A-Za-z0-9_]* skip - - -AMPERSAND <- '&' ![=] skip -AMPERSANDEQUAL <- '&=' skip -ASTERISK <- '*' ![%=|] skip -ASTERISKEQUAL <- '*=' skip -ASTERISKPERCENT <- '*%' ![=] skip -ASTERISKPERCENTEQUAL <- '*%=' skip -ASTERISKPIPE <- '*|' ![=] skip -ASTERISKPIPEEQUAL <- '*|=' skip -CARET <- '^' ![=] skip -CARETEQUAL <- '^=' skip -COLON <- ':' skip -COMMA <- ',' skip -DOT <- '.' ![*.?] skip -DOT2 <- '..' ![.] skip -DOT3 <- '...' skip -DOTASTERISK <- '.*' skip -DOTQUESTIONMARK <- '.?' skip -EQUAL <- '=' ![>=] skip -EQUALEQUAL <- '==' skip -EQUALRARROW <- '=>' skip -EXCLAMATIONMARK <- '!' ![=] skip -EXCLAMATIONMARKEQUAL <- '!=' skip -LARROW <- '<' ![<=] skip -LARROW2 <- '<<' ![=|] skip -LARROW2EQUAL <- '<<=' skip -LARROW2PIPE <- '<<|' ![=] skip -LARROW2PIPEEQUAL <- '<<|=' skip -LARROWEQUAL <- '<=' skip -LBRACE <- '{' skip -LBRACKET <- '[' skip -LPAREN <- '(' skip -MINUS <- '-' ![%=>|] skip -MINUSEQUAL <- '-=' skip -MINUSPERCENT <- '-%' ![=] skip -MINUSPERCENTEQUAL <- '-%=' skip -MINUSPIPE <- '-|' ![=] skip -MINUSPIPEEQUAL <- '-|=' skip -MINUSRARROW <- '->' skip -PERCENT <- '%' ![=] skip -PERCENTEQUAL <- '%=' skip -PIPE <- '|' ![|=] skip -PIPE2 <- '||' skip -PIPEEQUAL <- '|=' skip -PLUS <- '+' ![%+=|] skip -PLUS2 <- '++' skip -PLUSEQUAL <- '+=' skip -PLUSPERCENT <- '+%' ![=] skip -PLUSPERCENTEQUAL <- '+%=' skip -PLUSPIPE <- '+|' ![=] skip -PLUSPIPEEQUAL <- '+|=' skip -LETTERC <- 'c' skip -QUESTIONMARK <- '?' skip -RARROW <- '>' ![>=] skip -RARROW2 <- '>>' ![=] skip -RARROW2EQUAL <- '>>=' skip -RARROWEQUAL <- '>=' skip -RBRACE <- '}' skip -RBRACKET <- ']' skip -RPAREN <- ')' skip -SEMICOLON <- ';' skip -SLASH <- '/' ![=] skip -SLASHEQUAL <- '/=' skip -TILDE <- '~' skip - -end_of_word <- ![a-zA-Z0-9_] skip -KEYWORD_addrspace <- 'addrspace' end_of_word -KEYWORD_align <- 'align' end_of_word -KEYWORD_allowzero <- 'allowzero' end_of_word -KEYWORD_and <- 'and' end_of_word -KEYWORD_anyframe <- 'anyframe' end_of_word -KEYWORD_anytype <- 'anytype' end_of_word -KEYWORD_asm <- 'asm' end_of_word -KEYWORD_break <- 'break' end_of_word -KEYWORD_callconv <- 'callconv' end_of_word -KEYWORD_catch <- 'catch' end_of_word -KEYWORD_comptime <- 'comptime' end_of_word -KEYWORD_const <- 'const' end_of_word -KEYWORD_continue <- 'continue' end_of_word -KEYWORD_defer <- 'defer' end_of_word -KEYWORD_else <- 'else' end_of_word -KEYWORD_enum <- 'enum' end_of_word -KEYWORD_errdefer <- 'errdefer' end_of_word -KEYWORD_error <- 'error' end_of_word -KEYWORD_export <- 'export' end_of_word -KEYWORD_extern <- 'extern' end_of_word -KEYWORD_fn <- 'fn' end_of_word -KEYWORD_for <- 'for' end_of_word -KEYWORD_if <- 'if' end_of_word -KEYWORD_inline <- 'inline' end_of_word -KEYWORD_noalias <- 'noalias' end_of_word -KEYWORD_nosuspend <- 'nosuspend' end_of_word -KEYWORD_noinline <- 'noinline' end_of_word -KEYWORD_opaque <- 'opaque' end_of_word -KEYWORD_or <- 'or' end_of_word -KEYWORD_orelse <- 'orelse' end_of_word -KEYWORD_packed <- 'packed' end_of_word -KEYWORD_pub <- 'pub' end_of_word -KEYWORD_resume <- 'resume' end_of_word -KEYWORD_return <- 'return' end_of_word -KEYWORD_linksection <- 'linksection' end_of_word -KEYWORD_struct <- 'struct' end_of_word -KEYWORD_suspend <- 'suspend' end_of_word -KEYWORD_switch <- 'switch' end_of_word -KEYWORD_test <- 'test' end_of_word -KEYWORD_threadlocal <- 'threadlocal' end_of_word -KEYWORD_try <- 'try' end_of_word -KEYWORD_union <- 'union' end_of_word -KEYWORD_unreachable <- 'unreachable' end_of_word -KEYWORD_var <- 'var' end_of_word -KEYWORD_volatile <- 'volatile' end_of_word -KEYWORD_while <- 'while' end_of_word - -keyword <- KEYWORD_addrspace / KEYWORD_align / KEYWORD_allowzero / KEYWORD_and - / KEYWORD_anyframe / KEYWORD_anytype / KEYWORD_asm - / KEYWORD_break / KEYWORD_callconv / KEYWORD_catch - / KEYWORD_comptime / KEYWORD_const / KEYWORD_continue / KEYWORD_defer - / KEYWORD_else / KEYWORD_enum / KEYWORD_errdefer / KEYWORD_error / KEYWORD_export - / KEYWORD_extern / KEYWORD_fn / KEYWORD_for / KEYWORD_if - / KEYWORD_inline / KEYWORD_noalias / KEYWORD_nosuspend / KEYWORD_noinline - / KEYWORD_opaque / KEYWORD_or / KEYWORD_orelse / KEYWORD_packed - / KEYWORD_pub / KEYWORD_resume / KEYWORD_return / KEYWORD_linksection - / KEYWORD_struct / KEYWORD_suspend / KEYWORD_switch / KEYWORD_test - / KEYWORD_threadlocal / KEYWORD_try / KEYWORD_union / KEYWORD_unreachable - / KEYWORD_var / KEYWORD_volatile / KEYWORD_while - {#end_syntax_block#} + {#grammar#} {#header_close#} {#header_open|Zen#}
", .{ source_type, syntax_block.name });
switch (syntax_block.source_type) {
.zig => try tokenizeAndPrint(allocator, docgen_tokenizer, out, syntax_block.source_token),
else => {
- const raw_source = docgen_tokenizer.buffer[syntax_block.source_token.start..syntax_block.source_token.end];
+ const raw_source = content orelse docgen_tokenizer.buffer[syntax_block.source_token.start..syntax_block.source_token.end];
const trimmed_raw_source = mem.trim(u8, raw_source, " \r\n");
try out.writeAll("");
@@ -995,6 +1011,7 @@ fn genHtml(
tokenizer: *Tokenizer,
toc: *Toc,
code_dir: Path,
+ grammar: []const u8,
out: *Writer,
) !void {
for (toc.nodes) |node| {
@@ -1042,7 +1059,7 @@ fn genHtml(
try printShell(out, raw_shell_content, true);
},
.SyntaxBlock => |syntax_block| {
- try printSourceBlock(allocator, tokenizer, out, syntax_block);
+ try printSourceBlock(allocator, tokenizer, out, syntax_block, null);
},
.Code => |code| {
const out_basename = try std.fmt.allocPrint(allocator, "{s}.out", .{
@@ -1062,6 +1079,13 @@ fn genHtml(
try out.writeAll(contents);
},
+ .Grammar => {
+ try printSourceBlock(allocator, tokenizer, out, .{
+ .source_type = .peg,
+ .name = "grammar.peg",
+ .source_token = undefined,
+ }, grammar);
+ },
}
}
}
diff --git a/tools/gen_parser_oracle.zig b/tools/gen_parser_oracle.zig
new file mode 100644
index 0000000000000000000000000000000000000000..fafed31eb9eeea1fa0cf69fe9a75374a7bd78b6d
--- /dev/null
+++ b/tools/gen_parser_oracle.zig
@@ -0,0 +1,789 @@
+//! Example usage:
+//! zig build gen-parser-oracle
+//! zig build check-parser-oracle
+
+// This program implements a subset of the PEG grammar definition
+// in the peg(1) man page.
+//
+// It generates a recursive descent parser that returns true if a given input is
+// matched by the grammar. This generated parser is used as an oracle for fuzz testing.
+
+const std = @import("std");
+const assert = std.debug.assert;
+const Io = std.Io;
+const mem = std.mem;
+const Allocator = mem.Allocator;
+const log = std.log;
+
+pub fn main(init: std.process.Init) !void {
+ const gpa = init.gpa;
+ const arena = init.arena.allocator();
+ const io = init.io;
+ const args = try init.minimal.args.toSlice(arena);
+
+ const grammar_path = args[1];
+ const out_path = args[2];
+ const check = args.len > 3 and mem.eql(u8, args[3], "--check");
+
+ const grammar = try Io.Dir.cwd().readFileAlloc(io, grammar_path, gpa, .unlimited);
+ defer gpa.free(grammar);
+
+ var parser: Parser = .init(gpa, grammar);
+ defer parser.deinit();
+
+ const root = try parser.parseGrammar() orelse {
+ log.err("Invalid grammar", .{});
+ return;
+ };
+
+ var buffer: Io.Writer.Allocating = .init(gpa);
+ defer buffer.deinit();
+
+ var g: Generator = .init(&buffer.writer, &parser);
+ try g.genRoot(root);
+
+ const generated = try buffer.toOwnedSliceSentinel(0);
+ defer gpa.free(generated);
+
+ // Parse the generated Zig code and render it in the canonical format
+ var tree = try std.zig.Ast.parse(gpa, generated, .zig, .{});
+ defer tree.deinit(gpa);
+
+ if (tree.errors.len != 0) {
+ // This should never be reached, but helps a lot when debugging this script.
+ try std.zig.printAstErrorsToStderr(gpa, io, tree, "generated", .auto);
+ return error.ParseError;
+ }
+
+ if (check) {
+ const current = try Io.Dir.cwd().readFileAlloc(io, out_path, gpa, .unlimited);
+ defer gpa.free(current);
+ var aw: Io.Writer.Allocating = .init(gpa);
+ defer aw.deinit();
+ try tree.render(gpa, &aw.writer, .{});
+ if (!mem.eql(u8, current, aw.written())) {
+ std.log.err("grammar.peg modified without regenerating oracle", .{});
+ std.log.info("Run zig build gen-parser-oracle to regenerate", .{});
+ std.process.exit(1);
+ }
+ } else {
+ var out_file = try Io.Dir.cwd().createFile(io, out_path, .{});
+ defer out_file.close(io);
+ var out_buffer: [4096]u8 = undefined;
+ var out_writer = out_file.writer(io, &out_buffer);
+ const out = &out_writer.interface;
+
+ try tree.render(gpa, out, .{});
+ try out.flush();
+ }
+}
+
+const Generator = struct {
+ w: *Io.Writer,
+ p: *const Parser,
+ /// Suffix for generated identifiers, incremented for each nested scope to avoid shadowing,
+ /// Decremented at end of each generated scope to give smaller git diffs when regenerating
+ /// lib/std/zig/parser_generated_oracle.zig.
+ suffix: usize,
+
+ fn init(w: *Io.Writer, p: *const Parser) Generator {
+ return .{ .w = w, .p = p, .suffix = 0 };
+ }
+
+ const Error = Io.Writer.Error;
+ const Node = Parser.Node;
+
+ fn genRoot(g: *Generator, node: Node.Index) Error!void {
+ try g.w.writeAll(
+ \\//! This file is generated, do not edit manually! To generate, run:
+ \\//! zig build gen-parser-oracle
+ \\
+ \\const std = @import("std");
+ \\
+ \\const Error = error{MaxDepth};
+ \\const max_depth = 5;
+ \\
+ \\/// Returns true if the input source is in the language defined by
+ \\/// the grammar.
+ \\/// Returns error.MaxDepth if more than `max_depth` levels of recursion/iteration are reached.
+ \\pub fn parse(source: []const u8) Error!bool {
+ \\ var p: Parser = .{ .source = source, .i = 0, .depths = @splat(1) };
+ \\ return p.parseRoot();
+ \\}
+ );
+
+ const defs = g.p.getExtra(node.get(g.p).root);
+
+ // This enum exists to minimize git diffs in the generated oracle when
+ // the grammar is modified. It allows mapping from def name (stable) to
+ // def index (unstable).
+ try g.w.writeAll("const Def = enum {");
+ for (defs) |n| {
+ const def = n.get(g.p).def;
+ const id = def.id.get(g.p).id;
+ try g.w.print("def{s},", .{id});
+ }
+ try g.w.writeAll("};");
+
+ try g.w.print(
+ \\const Parser = struct {{
+ \\ source: []const u8,
+ \\ i: usize,
+ \\ depths: [{d}]u8,
+ , .{defs.len});
+ for (defs) |def| {
+ try g.genDef(def);
+ }
+ try g.w.writeAll("};");
+ }
+
+ fn genDef(g: *Generator, node: Node.Index) Error!void {
+ const def = node.get(g.p).def;
+ const id = def.id.get(g.p).id;
+ assert(g.suffix == 0);
+ try g.w.print("pub fn parse{s}(p: *Parser) Error!bool {{", .{id});
+ try g.w.print(
+ \\const def_index = @intFromEnum(Def.def{s});
+ \\if (p.depths[def_index] > max_depth) return error.MaxDepth;
+ \\p.depths[def_index] += 1;
+ \\defer p.depths[def_index] -= 1;
+ , .{id});
+ try g.w.writeAll("return ");
+ try g.genExpr(def.expr);
+ try g.w.writeAll(";}");
+ }
+
+ fn genExpr(g: *Generator, node: Node.Index) Error!void {
+ const suffix = g.suffix;
+ g.suffix += 1;
+ defer g.suffix -= 1;
+ try g.w.print(
+ \\blk_{d}: {{
+ \\const pos_{d} = p.i;
+ , .{ suffix, suffix });
+ for (g.p.getExtra(node.get(g.p).expr)) |seq| {
+ try g.w.writeAll("if (");
+ try g.genSeq(seq);
+ try g.w.print(") break :blk_{d} true;", .{suffix});
+ try g.w.print("p.i = pos_{d};", .{suffix});
+ }
+ try g.w.print("break :blk_{d} false; }}", .{suffix});
+ }
+
+ fn genSeq(g: *Generator, node: Node.Index) Error!void {
+ const items = g.p.getExtra(node.get(g.p).seq);
+ for (items, 0..) |item, i| {
+ if (i > 0) try g.w.writeAll(" and ");
+ try g.genNode(item);
+ }
+ }
+
+ fn genNode(g: *Generator, node: Node.Index) Error!void {
+ const suffix = g.suffix;
+ g.suffix += 1;
+ defer g.suffix -= 1;
+ switch (node.get(g.p)) {
+ .id => |id| try g.w.print("try p.parse{s}()", .{id}),
+ .expr => try g.genExpr(node),
+ .@"&" => |child| {
+ // XXX forbid unbounded lookahead
+ try g.w.print(
+ \\blk_{d}: {{
+ \\const pos_{d} = p.i;
+ \\const match_{d} =
+ , .{ suffix, suffix, suffix });
+ try g.genNode(child);
+ try g.w.print(
+ \\;
+ \\p.i = pos_{d};
+ \\ break :blk_{d} match_{d};
+ \\}}
+ , .{ suffix, suffix, suffix });
+ },
+ .@"!" => |child| {
+ // XXX forbid unbounded lookahead
+ try g.w.print(
+ \\blk_{d}: {{
+ \\const pos_{d} = p.i;
+ \\const match_{d} =
+ , .{ suffix, suffix, suffix });
+ try g.genNode(child);
+ try g.w.print(
+ \\;
+ \\p.i = pos_{d};
+ \\ break :blk_{d} !match_{d};
+ \\}}
+ , .{ suffix, suffix, suffix });
+ },
+ .@"?" => |child| {
+ try g.w.writeAll("(");
+ try g.genNode(child);
+ try g.w.writeAll(" or true )");
+ },
+ .@"*" => |child| {
+ try g.w.print(
+ \\blk_{d}: {{
+ \\var i_{d}: usize = 0;
+ \\while (
+ , .{ suffix, suffix });
+ try g.genNode(child);
+ try g.w.print(
+ \\) {{
+ \\ if (i_{d} > max_depth) return error.MaxDepth;
+ \\ i_{d} += 1;
+ \\}}
+ \\break :blk_{d} true; }}
+ , .{ suffix, suffix, suffix });
+ },
+ .@"+" => |child| {
+ try g.w.print(
+ \\blk_{d}: {{
+ \\var match_{d} = false;
+ \\var i_{d}: usize = 0;
+ \\while (
+ , .{ suffix, suffix, suffix });
+ try g.genNode(child);
+ try g.w.print(
+ \\) {{
+ \\ match_{d} = true;
+ \\ if (i_{d} > max_depth) return error.MaxDepth;
+ \\ i_{d} += 1;
+ \\}}
+ \\break :blk_{d} match_{d}; }}
+ , .{ suffix, suffix, suffix, suffix, suffix });
+ },
+ .@"." => {
+ try g.w.print(
+ \\blk_{d}: {{
+ \\ if (p.i < p.source.len) {{
+ \\ p.i += 1;
+ \\ break :blk_{d} true;
+ \\ }}
+ \\ break :blk_{d} false;
+ \\}}
+ , .{ suffix, suffix, suffix });
+ },
+ .literal => |literal| {
+ const bytes = g.p.strings.items[literal.off..][0..literal.len];
+ try g.w.print(
+ \\blk_{d}: {{
+ \\if (std.mem.startsWith(u8, p.source[p.i..], "
+ , .{suffix});
+ try std.zig.stringEscape(bytes, g.w);
+ try g.w.print(
+ \\")) {{
+ \\p.i += {d};
+ \\ break :blk_{d} true;
+ \\}}
+ \\break :blk_{d} false;
+ \\}}
+ , .{ bytes.len, suffix, suffix });
+ },
+ .class => |ranges| {
+ try g.w.writeAll("(p.i < p.source.len and switch (p.source[p.i]) {");
+ for (g.p.getExtra(ranges)) |n| {
+ const range = n.get(g.p).range;
+ try g.w.writeAll("'");
+ try std.zig.charEscape(range.start, g.w);
+ try g.w.writeAll("'...'");
+ try std.zig.charEscape(range.end, g.w);
+ try g.w.writeAll("',");
+ }
+ try g.w.print(
+ \\=> blk_{d}: {{ p.i += 1; break :blk_{d} true; }},
+ \\else => false,
+ \\}})
+ , .{ suffix, suffix });
+ },
+ .sof => try g.w.writeAll("(p.i == 0)"),
+ else => unreachable,
+ }
+ }
+};
+
+/// Parser implements a subset of the PEG grammar definition.
+/// We don't bother implementing the Action, BEGIN, and END rules
+/// and also omit unneeded character escape sequences.
+///
+/// The full PEG grammar found in the peg(1) man page:
+///
+/// Grammar <- Spacing Definition+ EndOfFile
+///
+/// Definition <- Identifier LEFTARROW Expression
+/// Expression <- Sequence ( SLASH Sequence )*
+/// Sequence <- Prefix*
+/// Prefix <- AND Action
+/// / ( AND / NOT )? Suffix
+/// Suffix <- Primary ( QUERY / STAR / PLUS )?
+/// Primary <- Identifier !LEFTARROW
+/// / OPEN Expression CLOSE
+/// / Literal
+/// / Class
+/// / DOT
+/// / Action
+/// / BEGIN
+/// / END
+///
+/// Identifier <- < IdentStart IdentCont* > Spacing
+/// IdentStart <- [a-zA-Z_]
+/// IdentCont <- IdentStart / [0-9]
+/// Literal <- ['] < ( !['] Char )* > ['] Spacing
+/// / ["] < ( !["] Char )* > ["] Spacing
+/// Class <- '[' < ( !']' Range )* > ']' Spacing
+/// Range <- Char '-' Char / Char
+/// Char <- '\\' [abefnrtv'"\[\]\\]
+/// / '\\' [0-3][0-7][0-7]
+/// / '\\' [0-7][0-7]?
+/// / '\\' '-'
+/// / !'\\' .
+/// LEFTARROW <- '<-' Spacing
+/// SLASH <- '/' Spacing
+/// AND <- '&' Spacing
+/// NOT <- '!' Spacing
+/// QUERY <- '?' Spacing
+/// STAR <- '*' Spacing
+/// PLUS <- '+' Spacing
+/// OPEN <- '(' Spacing
+/// CLOSE <- ')' Spacing
+/// DOT <- '.' Spacing
+/// Spacing <- ( Space / Comment )*
+/// Comment <- '#' ( !EndOfLine . )* EndOfLine
+/// Space <- ' ' / '\t' / EndOfLine
+/// EndOfLine <- '\r\n' / '\n' / '\r'
+/// EndOfFile <- !.
+/// Action <- '{' < [^}]* > '}' Spacing
+/// BEGIN <- '<' Spacing
+/// END <- '>' Spacing
+const Parser = struct {
+ gpa: Allocator,
+ /// PEG grammar source
+ source: []const u8,
+ /// Current index into source
+ i: u32,
+ nodes: std.ArrayList(Node),
+ extra: std.ArrayList(Node.Index),
+ strings: std.ArrayList(u8),
+
+ const Node = union(enum) {
+ /// Slice into extra
+ root: Slice,
+ def: struct {
+ id: Index,
+ expr: Index,
+ },
+ /// Slice into Parser.source
+ id: []const u8,
+ /// Slice into extra
+ expr: Slice,
+ /// Slice into extra
+ seq: Slice,
+ @"&": Index,
+ @"!": Index,
+ @"?": Index,
+ @"*": Index,
+ @"+": Index,
+ @".",
+ /// Slice into strings
+ literal: Slice,
+ /// Slice into extra
+ class: Slice,
+ range: struct {
+ start: u8,
+ end: u8,
+ },
+ /// Start of file
+ sof,
+
+ const Index = enum(u32) {
+ _,
+
+ fn get(index: Index, p: *const Parser) Node {
+ return p.nodes.items[@intFromEnum(index)];
+ }
+ };
+
+ const Slice = struct {
+ off: u32,
+ len: u32,
+ };
+ };
+
+ fn init(gpa: Allocator, source: []const u8) Parser {
+ return .{
+ .gpa = gpa,
+ .source = source,
+ .i = 0,
+ .nodes = .empty,
+ .extra = .empty,
+ .strings = .empty,
+ };
+ }
+
+ fn deinit(p: *Parser) void {
+ p.nodes.deinit(p.gpa);
+ p.extra.deinit(p.gpa);
+ p.strings.deinit(p.gpa);
+ }
+
+ // Grammar <- Spacing Definition+ EndOfFile
+ // EndOfFile <- !.
+ fn parseGrammar(p: *Parser) !?Node.Index {
+ var scratch: std.ArrayList(Node.Index) = .empty;
+ defer scratch.deinit(p.gpa);
+ _ = p.eatSpacing();
+ while (try p.parseDefinition()) |def| {
+ try scratch.append(p.gpa, def);
+ }
+ if (scratch.items.len == 0) return null;
+ if (p.peek() != null) return null;
+ const defs = try p.addExtra(scratch.items);
+ return try p.addNode(.{ .root = defs });
+ }
+
+ // Definition <- Identifier LEFTARROW Expression
+ fn parseDefinition(p: *Parser) !?Node.Index {
+ const id = try p.parseIdentifier() orelse return null;
+ if (!p.eatLeftArrow()) return null;
+ const expr = try p.parseExpression() orelse return null;
+ return try p.addNode(.{ .def = .{
+ .id = id,
+ .expr = expr,
+ } });
+ }
+
+ // Expression <- Sequence ( SLASH Sequence )*
+ fn parseExpression(p: *Parser) error{OutOfMemory}!?Node.Index {
+ var scratch: std.ArrayList(Node.Index) = .empty;
+ defer scratch.deinit(p.gpa);
+ while (try p.parseSequence()) |seq| {
+ try scratch.append(p.gpa, seq);
+ if (!p.eatSlash()) break;
+ }
+ if (scratch.items.len == 0) return null;
+ const seqs = try p.addExtra(scratch.items);
+ return try p.addNode(.{ .expr = seqs });
+ }
+
+ // Sequence <- Prefix*
+ fn parseSequence(p: *Parser) !?Node.Index {
+ var scratch: std.ArrayList(Node.Index) = .empty;
+ defer scratch.deinit(p.gpa);
+ while (try p.parsePrefix()) |primary| {
+ try scratch.append(p.gpa, primary);
+ }
+ const primaries = try p.addExtra(scratch.items);
+ return try p.addNode(.{ .seq = primaries });
+ }
+
+ // Prefix <- AND Action
+ // / ( AND / NOT )? Suffix
+ fn parsePrefix(p: *Parser) !?Node.Index {
+ if (p.eatAnd()) {
+ // We only support a single hardcoded "start of file" Action
+ if (p.eat('{')) {
+ // Action <- '{' < [^}]* > '}' Spacing
+ if (std.mem.startsWith(u8, p.source[p.i..], " (yy->__pos == 0) }")) {
+ while (!p.eat('}')) p.i += 1;
+ _ = p.eatSpacing();
+ return try p.addNode(.sof);
+ }
+ return null;
+ }
+ const suffix = try p.parseSuffix() orelse return null;
+ return try p.addNode(.{ .@"&" = suffix });
+ }
+ if (p.eatNot()) {
+ const suffix = try p.parseSuffix() orelse return null;
+ return try p.addNode(.{ .@"!" = suffix });
+ }
+ return try p.parseSuffix();
+ }
+
+ // Suffix <- Primary ( QUERY / STAR / PLUS )?
+ fn parseSuffix(p: *Parser) !?Node.Index {
+ const primary = try p.parsePrimary() orelse return null;
+ if (p.eatQuery()) {
+ return try p.addNode(.{ .@"?" = primary });
+ }
+ if (p.eatStar()) {
+ return try p.addNode(.{ .@"*" = primary });
+ }
+ if (p.eatPlus()) {
+ return try p.addNode(.{ .@"+" = primary });
+ }
+ return primary;
+ }
+
+ // Primary <- Identifier !LEFTARROW
+ // / OPEN Expression CLOSE
+ // / Literal
+ // / Class
+ // / DOT
+ // / Action
+ // / BEGIN
+ // / END
+ fn parsePrimary(p: *Parser) !?Node.Index {
+ const init_pos = p.savePos();
+ if (try p.parseIdentifier()) |id| {
+ const pos = p.savePos();
+ if (!p.eatLeftArrow()) {
+ p.restorePos(pos);
+ return id;
+ }
+ }
+ p.restorePos(init_pos);
+ if (p.eatOpen()) if (try p.parseExpression()) |expr| if (p.eatClose()) return expr;
+ p.restorePos(init_pos);
+ if (try p.parseLiteral()) |literal| return literal;
+ p.restorePos(init_pos);
+ if (try p.parseClass()) |class| return class;
+ p.restorePos(init_pos);
+ if (p.eatDot()) return try p.addNode(.@".");
+ // We don't implement Action, BEGIN, and END.
+ return null;
+ }
+
+ // Identifier <- < IdentStart IdentCont* > Spacing
+ // IdentStart <- [a-zA-Z_]
+ // IdentCont <- IdentStart / [0-9]
+ fn parseIdentifier(p: *Parser) !?Node.Index {
+ const start = p.i;
+ switch (p.next() orelse return null) {
+ 'a'...'z', 'A'...'Z', '_' => {},
+ else => return null,
+ }
+ while (p.peek()) |cont| {
+ switch (cont) {
+ 'a'...'z', 'A'...'Z', '_', '0'...'9' => p.i += 1,
+ else => break,
+ }
+ }
+ const id = p.source[start..p.i];
+ _ = p.eatSpacing();
+ return try p.addNode(.{ .id = id });
+ }
+
+ // Literal <- ['] < ( !['] Char )* > ['] Spacing
+ // / ["] < ( !["] Char )* > ["] Spacing
+ fn parseLiteral(p: *Parser) !?Node.Index {
+ const quote: u8 = if (p.eat('\'')) '\'' else if (p.eat('"')) '"' else return null;
+ const off = p.strings.items.len;
+ while (!p.eat(quote)) {
+ const byte = p.parseChar() orelse return null;
+ try p.strings.append(p.gpa, byte);
+ }
+ _ = p.eatSpacing();
+ return try p.addNode(.{ .literal = .{
+ .off = @intCast(off),
+ .len = @intCast(p.strings.items.len - off),
+ } });
+ }
+
+ // Class <- '[' < ( !']' Range )* > ']' Spacing
+ fn parseClass(p: *Parser) !?Node.Index {
+ var scratch: std.ArrayList(Node.Index) = .empty;
+ defer scratch.deinit(p.gpa);
+ if (!p.eat('[')) return null;
+ while (!p.eat(']')) {
+ const range = try p.parseRange() orelse return null;
+ try scratch.append(p.gpa, range);
+ }
+ _ = p.eatSpacing();
+ const ranges = try p.addExtra(scratch.items);
+ return try p.addNode(.{ .class = ranges });
+ }
+
+ // Range <- Char '-' Char / Char
+ fn parseRange(p: *Parser) !?Node.Index {
+ const start = p.parseChar() orelse return null;
+ const end = blk: {
+ if (p.eat('-')) {
+ break :blk p.parseChar() orelse return null;
+ }
+ break :blk start;
+ };
+ return try p.addNode(.{ .range = .{
+ .start = start,
+ .end = end,
+ } });
+ }
+
+ // Char <- '\\' [abefnrtv'"\[\]\\]
+ // / '\\' [0-3][0-7][0-7]
+ // / '\\' [0-7][0-7]?
+ // / '\\' '-'
+ // / !'\\' .
+ fn parseChar(p: *Parser) ?u8 {
+ if (p.eat('\\')) {
+ const c = p.next() orelse return null;
+ return switch (c) {
+ // Only the escape sequences actually used in the Zig grammar are implemented
+ 'n' => '\n',
+ 'r' => '\r',
+ 't' => '\t',
+ '\'' => '\'',
+ '"' => '"',
+ '[' => '[',
+ ']' => ']',
+ '\\' => '\\',
+ '-' => '-',
+ '0'...'7' => {
+ // octal
+ if (c <= '3') {
+ const c2 = p.next() orelse return null;
+ if (c2 < '0' or c2 > '7') return null;
+ const c3 = p.next() orelse return null;
+ if (c3 < '0' or c3 > '7') return null;
+ return (c - '0') * 8 * 8 + (c2 - '0') * 8 + (c3 - '0');
+ } else {
+ if (p.peek()) |c2| {
+ if (c2 >= '0' and c2 <= '7') {
+ p.i += 1;
+ return (c - '0') * 8 + (c2 - '0');
+ }
+ }
+ return (c - '0');
+ }
+ },
+ else => null,
+ };
+ } else {
+ return p.next();
+ }
+ }
+
+ // LEFTARROW <- '<-' Spacing
+ fn eatLeftArrow(p: *Parser) bool {
+ return p.eat('<') and p.eat('-') and p.eatSpacing();
+ }
+
+ // SLASH <- '/' Spacing
+ fn eatSlash(p: *Parser) bool {
+ return p.eat('/') and p.eatSpacing();
+ }
+
+ // AND <- '&' Spacing
+ fn eatAnd(p: *Parser) bool {
+ return p.eat('&') and p.eatSpacing();
+ }
+
+ // NOT <- '!' Spacing
+ fn eatNot(p: *Parser) bool {
+ return p.eat('!') and p.eatSpacing();
+ }
+
+ // QUERY <- '?' Spacing
+ fn eatQuery(p: *Parser) bool {
+ return p.eat('?') and p.eatSpacing();
+ }
+
+ // STAR <- '*' Spacing
+ fn eatStar(p: *Parser) bool {
+ return p.eat('*') and p.eatSpacing();
+ }
+
+ // PLUS <- '+' Spacing
+ fn eatPlus(p: *Parser) bool {
+ return p.eat('+') and p.eatSpacing();
+ }
+
+ // OPEN <- '(' Spacing
+ fn eatOpen(p: *Parser) bool {
+ return p.eat('(') and p.eatSpacing();
+ }
+
+ // CLOSE <- ')' Spacing
+ fn eatClose(p: *Parser) bool {
+ return p.eat(')') and p.eatSpacing();
+ }
+
+ // DOT <- '.' Spacing
+ fn eatDot(p: *Parser) bool {
+ return p.eat('.') and p.eatSpacing();
+ }
+
+ // Spacing <- ( Space / Comment )*
+ fn eatSpacing(p: *Parser) bool {
+ while (p.eatSpace() or p.eatComment()) {}
+ return true;
+ }
+
+ // Comment <- '#' ( !EndOfLine . )* EndOfLine
+ fn eatComment(p: *Parser) bool {
+ if (!p.eat('#')) return false;
+ while (!p.eatEndOfLine()) p.i += 1;
+ return true;
+ }
+
+ // Space <- ' ' / '\t' / EndOfLine
+ fn eatSpace(p: *Parser) bool {
+ return p.eat(' ') or p.eat('\t') or p.eatEndOfLine();
+ }
+
+ // EndOfLine <- '\r\n' / '\n' / '\r'
+ fn eatEndOfLine(p: *Parser) bool {
+ return p.eat('\n');
+ }
+
+ fn peek(p: *Parser) ?u8 {
+ if (p.i < p.source.len) {
+ return p.source[p.i];
+ }
+ return null;
+ }
+
+ fn next(p: *Parser) ?u8 {
+ if (p.i < p.source.len) {
+ defer p.i += 1;
+ return p.source[p.i];
+ }
+ return null;
+ }
+
+ fn eat(p: *Parser, byte: u8) bool {
+ if (p.i < p.source.len and p.source[p.i] == byte) {
+ p.i += 1;
+ return true;
+ }
+ return false;
+ }
+
+ fn addNode(p: *Parser, node: Node) !Node.Index {
+ try p.nodes.append(p.gpa, node);
+ return @enumFromInt(p.nodes.items.len - 1);
+ }
+
+ fn addExtra(p: *Parser, nodes: []const Node.Index) !Node.Slice {
+ const off = p.extra.items.len;
+ try p.extra.appendSlice(p.gpa, nodes);
+ return .{ .off = @intCast(off), .len = @intCast(p.extra.items.len - off) };
+ }
+
+ const Pos = struct {
+ i: u32,
+ nodes_len: u32,
+ extra_len: u32,
+ strings_len: u32,
+ };
+
+ fn savePos(p: *const Parser) Pos {
+ return .{
+ .i = p.i,
+ .nodes_len = @intCast(p.nodes.items.len),
+ .extra_len = @intCast(p.extra.items.len),
+ .strings_len = @intCast(p.strings.items.len),
+ };
+ }
+
+ fn restorePos(p: *Parser, pos: Pos) void {
+ assert(p.i >= pos.i);
+ p.i = pos.i;
+ p.nodes.shrinkRetainingCapacity(pos.nodes_len);
+ p.extra.shrinkRetainingCapacity(pos.extra_len);
+ p.strings.shrinkRetainingCapacity(pos.strings_len);
+ }
+
+ fn getExtra(p: *const Parser, s: Node.Slice) []const Node.Index {
+ return p.extra.items[s.off..][0..s.len];
+ }
+};
diff --git a/tools/gen_spirv_spec.zig b/tools/gen_spirv_spec.zig
index fbf8fa167ae6fc0ae5dcba040a87046b66d34c5c..8258c0c464af12be489b099d7ad0105ab2c0d1c2 100644
--- a/tools/gen_spirv_spec.zig
+++ b/tools/gen_spirv_spec.zig
@@ -116,7 +116,7 @@ pub fn main(init: std.process.Init) !void {
try allocating.writer.writeByte(0);
const output = allocating.written()[0 .. allocating.written().len - 1 :0];
- var tree = try std.zig.Ast.parse(arena, output, .zig);
+ var tree = try std.zig.Ast.parse(arena, output, .zig, .{});
if (tree.errors.len != 0) {
try std.zig.printAstErrorsToStderr(arena, io, tree, "", .auto);