From 9303d4be1029dc42845aafeff38c811a08ac29b0 Mon Sep 17 00:00:00 2001 From: Isaac Freund Date: Mon, 15 Jun 2026 11:50:05 +0200 Subject: [PATCH] grammar: match Parse.zig operator whitespace rules Parse.zig requires whitespace to be either immediately before and after a binary operator or neither before nor after the operator. --- doc/langref/grammar.peg | 32 +++- lib/std/zig/parser_fuzz.zig | 19 +- lib/std/zig/parser_generated_oracle.zig | 227 ++++++++++++++++++++---- 3 files changed, 233 insertions(+), 45 deletions(-) diff --git a/doc/langref/grammar.peg b/doc/langref/grammar.peg index b356bee35cf543efe127e80fc6fb79da53e08d84..5d34b8a4868b52ee0c10bd8862da6a3b83de1d50 100644 --- a/doc/langref/grammar.peg +++ b/doc/langref/grammar.peg @@ -74,9 +74,9 @@ SingleAssignExpr <- Expr (AssignOp Expr)? Expr <- BoolOrExpr -BoolOrExpr <- BoolAndExpr (KEYWORD_or BoolAndExpr)* +BoolOrExpr <- BoolAndExpr (OrOp BoolAndExpr)* -BoolAndExpr <- CompareExpr (KEYWORD_and CompareExpr)* +BoolAndExpr <- CompareExpr (AndOp CompareExpr)* CompareExpr <- BitwiseExpr (CompareOp BitwiseExpr)? @@ -253,7 +253,15 @@ AssignOp / MINUSPERCENTEQUAL / EQUAL -CompareOp +OrOp <- pre_op_white KEYWORD_or post_op_white + / !pre_op_white KEYWORD_or !post_op_white + +AndOp <- pre_op_white KEYWORD_and post_op_white + / !pre_op_white KEYWORD_and !post_op_white + +CompareOp <- pre_op_white CompareOpTok post_op_white + / !pre_op_white CompareOpTok !post_op_white +CompareOpTok <- EQUALEQUAL / EXCLAMATIONMARKEQUAL / LARROW @@ -261,19 +269,25 @@ CompareOp / LARROWEQUAL / RARROWEQUAL -BitwiseOp +BitwiseOp <- pre_op_white BitwiseOpTok post_op_white + / !pre_op_white BitwiseOpTok !post_op_white +BitwiseOpTok <- AMPERSAND / CARET / PIPE / KEYWORD_orelse / KEYWORD_catch Payload? -BitShiftOp +BitShiftOp <- pre_op_white BitShiftOpTok post_op_white + / !pre_op_white BitShiftOpTok !post_op_white +BitShiftOpTok <- LARROW2 / RARROW2 / LARROW2PIPE -AdditionOp +AdditionOp <- pre_op_white AdditionOpTok post_op_white + / !pre_op_white AdditionOpTok !post_op_white +AdditionOpTok <- PLUS / MINUS / PLUS2 @@ -282,7 +296,9 @@ AdditionOp / PLUSPIPE / MINUSPIPE -MultiplyOp +MultiplyOp <- pre_op_white MultiplyOpTok post_op_white + / !pre_op_white MultiplyOpTok !post_op_white +MultiplyOpTok <- PIPE2 / ASTERISK / SLASH @@ -438,6 +454,8 @@ line_comment line_string <- '\\\\' non_control_utf8* newline newline <- "\n" / "\r\n" / eof skip <- ([ \n\t\r] / line_comment)* +pre_op_white <- ([ \n\t\r] / line_comment)+ +post_op_white <- [ \n\t\r] skip CHAR_LITERAL <- skip ['] char_char* ['] diff --git a/lib/std/zig/parser_fuzz.zig b/lib/std/zig/parser_fuzz.zig index 39055df5ebe337cd6e00bfcdf81641264118d9b3..32ed24c6afc5d2537f429d1f3709f3f9f6ba3617 100644 --- a/lib/std/zig/parser_fuzz.zig +++ b/lib/std/zig/parser_fuzz.zig @@ -1,20 +1,35 @@ const std = @import("../std.zig"); +const Allocator = std.mem.Allocator; const Smith = std.testing.Smith; const oracle = @import("parser_generated_oracle.zig"); test "fuzz std.zig.Ast.parse() against generated oracle" { - try std.testing.fuzz({}, checkAgainstOracle, .{}); + try std.testing.fuzz({}, fuzzAgainstOracle, .{}); } -fn checkAgainstOracle(_: void, smith: *Smith) !void { +fn fuzzAgainstOracle(_: void, smith: *Smith) !void { var buffer: [1 << 14]u8 = undefined; const len = smith.slice(buffer[0 .. buffer.len - 1]); buffer[len] = 0; const source = buffer[0..len :0]; + try checkAgainstOracle(source); +} + +// Found using AFL++ +test "operator whitespace" { + try checkAgainstOracle( + \\test { + \\ _!= 0; + \\} + ); +} + +fn checkAgainstOracle(source: [:0]const u8) !void { var fba_buf: [1 << 18]u8 = undefined; var fba: std.heap.FixedBufferAllocator = .init(&fba_buf); + const ast = try std.zig.Ast.parse(fba.allocator(), source, .zig); errdefer logBadSource(source, ast); diff --git a/lib/std/zig/parser_generated_oracle.zig b/lib/std/zig/parser_generated_oracle.zig index 6e1b59019dd8dd2b78c925a4bde5e11bf3bf512a..bd7b581760b380e9ed08696a2896093c1bf8fa26 100644 --- a/lib/std/zig/parser_generated_oracle.zig +++ b/lib/std/zig/parser_generated_oracle.zig @@ -509,7 +509,7 @@ const Parser = struct { if (p.parseBoolAndExpr() and blk_1: { while (blk_3: { const pos_3 = p.i; - if (p.parseKEYWORD_or() and p.parseBoolAndExpr()) break :blk_3 true; + if (p.parseOrOp() and p.parseBoolAndExpr()) break :blk_3 true; p.i = pos_3; break :blk_3 false; }) {} @@ -525,7 +525,7 @@ const Parser = struct { if (p.parseCompareExpr() and blk_1: { while (blk_3: { const pos_3 = p.i; - if (p.parseKEYWORD_and() and p.parseCompareExpr()) break :blk_3 true; + if (p.parseAndOp() and p.parseCompareExpr()) break :blk_3 true; p.i = pos_3; break :blk_3 false; }) {} @@ -612,40 +612,6 @@ const Parser = struct { break :blk_0 false; }; } - pub fn parseBinOp(p: *Parser) bool { - return blk_0: { - const pos_0 = p.i; - if (p.parseExpr() and blk_1: { - while (blk_3: { - const pos_3 = p.i; - if (blk_4: { - var match_4 = false; - while (p.parsewhite()) { - match_4 = true; - } - break :blk_4 match_4; - } and blk_4: { - if (std.mem.startsWith(u8, p.source[p.i..], "or")) { - p.i += 2; - break :blk_4 true; - } - break :blk_4 false; - } and blk_4: { - var match_4 = false; - while (p.parsewhite()) { - match_4 = true; - } - break :blk_4 match_4; - } and p.parseBoolAndExpr()) break :blk_3 true; - p.i = pos_3; - break :blk_3 false; - }) {} - break :blk_1 true; - }) break :blk_0 true; - p.i = pos_0; - break :blk_0 false; - }; - } pub fn parsePrefixExpr(p: *Parser) bool { return blk_0: { const pos_0 = p.i; @@ -1461,7 +1427,67 @@ const Parser = struct { break :blk_0 false; }; } + pub fn parseOrOp(p: *Parser) bool { + return blk_0: { + const pos_0 = p.i; + if (p.parsepre_op_white() and p.parseKEYWORD_or() and p.parsepost_op_white()) break :blk_0 true; + p.i = pos_0; + if (blk_1: { + const pos_1 = p.i; + const match_1 = p.parsepre_op_white(); + p.i = pos_1; + break :blk_1 !match_1; + } and p.parseKEYWORD_or() and blk_1: { + const pos_1 = p.i; + const match_1 = p.parsepost_op_white(); + p.i = pos_1; + break :blk_1 !match_1; + }) break :blk_0 true; + p.i = pos_0; + break :blk_0 false; + }; + } + pub fn parseAndOp(p: *Parser) bool { + return blk_0: { + const pos_0 = p.i; + if (p.parsepre_op_white() and p.parseKEYWORD_and() and p.parsepost_op_white()) break :blk_0 true; + p.i = pos_0; + if (blk_1: { + const pos_1 = p.i; + const match_1 = p.parsepre_op_white(); + p.i = pos_1; + break :blk_1 !match_1; + } and p.parseKEYWORD_and() and blk_1: { + const pos_1 = p.i; + const match_1 = p.parsepost_op_white(); + p.i = pos_1; + break :blk_1 !match_1; + }) break :blk_0 true; + p.i = pos_0; + break :blk_0 false; + }; + } pub fn parseCompareOp(p: *Parser) bool { + return blk_0: { + const pos_0 = p.i; + if (p.parsepre_op_white() and p.parseCompareOpTok() and p.parsepost_op_white()) break :blk_0 true; + p.i = pos_0; + if (blk_1: { + const pos_1 = p.i; + const match_1 = p.parsepre_op_white(); + p.i = pos_1; + break :blk_1 !match_1; + } and p.parseCompareOpTok() and blk_1: { + const pos_1 = p.i; + const match_1 = p.parsepost_op_white(); + p.i = pos_1; + break :blk_1 !match_1; + }) break :blk_0 true; + p.i = pos_0; + break :blk_0 false; + }; + } + pub fn parseCompareOpTok(p: *Parser) bool { return blk_0: { const pos_0 = p.i; if (p.parseEQUALEQUAL()) break :blk_0 true; @@ -1480,6 +1506,26 @@ const Parser = struct { }; } pub fn parseBitwiseOp(p: *Parser) bool { + return blk_0: { + const pos_0 = p.i; + if (p.parsepre_op_white() and p.parseBitwiseOpTok() and p.parsepost_op_white()) break :blk_0 true; + p.i = pos_0; + if (blk_1: { + const pos_1 = p.i; + const match_1 = p.parsepre_op_white(); + p.i = pos_1; + break :blk_1 !match_1; + } and p.parseBitwiseOpTok() and blk_1: { + const pos_1 = p.i; + const match_1 = p.parsepost_op_white(); + p.i = pos_1; + break :blk_1 !match_1; + }) break :blk_0 true; + p.i = pos_0; + break :blk_0 false; + }; + } + pub fn parseBitwiseOpTok(p: *Parser) bool { return blk_0: { const pos_0 = p.i; if (p.parseAMPERSAND()) break :blk_0 true; @@ -1496,6 +1542,26 @@ const Parser = struct { }; } pub fn parseBitShiftOp(p: *Parser) bool { + return blk_0: { + const pos_0 = p.i; + if (p.parsepre_op_white() and p.parseBitShiftOpTok() and p.parsepost_op_white()) break :blk_0 true; + p.i = pos_0; + if (blk_1: { + const pos_1 = p.i; + const match_1 = p.parsepre_op_white(); + p.i = pos_1; + break :blk_1 !match_1; + } and p.parseBitShiftOpTok() and blk_1: { + const pos_1 = p.i; + const match_1 = p.parsepost_op_white(); + p.i = pos_1; + break :blk_1 !match_1; + }) break :blk_0 true; + p.i = pos_0; + break :blk_0 false; + }; + } + pub fn parseBitShiftOpTok(p: *Parser) bool { return blk_0: { const pos_0 = p.i; if (p.parseLARROW2()) break :blk_0 true; @@ -1508,6 +1574,26 @@ const Parser = struct { }; } pub fn parseAdditionOp(p: *Parser) bool { + return blk_0: { + const pos_0 = p.i; + if (p.parsepre_op_white() and p.parseAdditionOpTok() and p.parsepost_op_white()) break :blk_0 true; + p.i = pos_0; + if (blk_1: { + const pos_1 = p.i; + const match_1 = p.parsepre_op_white(); + p.i = pos_1; + break :blk_1 !match_1; + } and p.parseAdditionOpTok() and blk_1: { + const pos_1 = p.i; + const match_1 = p.parsepost_op_white(); + p.i = pos_1; + break :blk_1 !match_1; + }) break :blk_0 true; + p.i = pos_0; + break :blk_0 false; + }; + } + pub fn parseAdditionOpTok(p: *Parser) bool { return blk_0: { const pos_0 = p.i; if (p.parsePLUS()) break :blk_0 true; @@ -1528,6 +1614,26 @@ const Parser = struct { }; } pub fn parseMultiplyOp(p: *Parser) bool { + return blk_0: { + const pos_0 = p.i; + if (p.parsepre_op_white() and p.parseMultiplyOpTok() and p.parsepost_op_white()) break :blk_0 true; + p.i = pos_0; + if (blk_1: { + const pos_1 = p.i; + const match_1 = p.parsepre_op_white(); + p.i = pos_1; + break :blk_1 !match_1; + } and p.parseMultiplyOpTok() and blk_1: { + const pos_1 = p.i; + const match_1 = p.parsepost_op_white(); + p.i = pos_1; + break :blk_1 !match_1; + }) break :blk_0 true; + p.i = pos_0; + break :blk_0 false; + }; + } + pub fn parseMultiplyOpTok(p: *Parser) bool { return blk_0: { const pos_0 = p.i; if (p.parsePIPE2()) break :blk_0 true; @@ -2378,6 +2484,55 @@ const Parser = struct { break :blk_0 false; }; } + pub fn parsepre_op_white(p: *Parser) bool { + return blk_0: { + const pos_0 = p.i; + if (blk_1: { + var match_1 = false; + while (blk_3: { + const pos_3 = p.i; + if ((p.i < p.source.len and switch (p.source[p.i]) { + ' '...' ', + '\n'...'\n', + '\t'...'\t', + '\r'...'\r', + => blk_4: { + p.i += 1; + break :blk_4 true; + }, + else => false, + })) break :blk_3 true; + p.i = pos_3; + if (p.parseline_comment()) break :blk_3 true; + p.i = pos_3; + break :blk_3 false; + }) { + match_1 = true; + } + break :blk_1 match_1; + }) break :blk_0 true; + p.i = pos_0; + break :blk_0 false; + }; + } + pub fn parsepost_op_white(p: *Parser) bool { + return blk_0: { + const pos_0 = p.i; + if ((p.i < p.source.len and switch (p.source[p.i]) { + ' '...' ', + '\n'...'\n', + '\t'...'\t', + '\r'...'\r', + => blk_1: { + p.i += 1; + break :blk_1 true; + }, + else => false, + }) and p.parseskip()) break :blk_0 true; + p.i = pos_0; + break :blk_0 false; + }; + } pub fn parseCHAR_LITERAL(p: *Parser) bool { return blk_0: { const pos_0 = p.i; -- 2.54.0