diff --git a/doc/langref/grammar.peg b/doc/langref/grammar.peg index ccb86d02fa4ee5983062dd0ce735a752694459d8..bb602287eb8bef8498d55b6334680e233c68af75 100644 --- a/doc/langref/grammar.peg +++ b/doc/langref/grammar.peg @@ -450,11 +450,14 @@ ExprPrefix <- ASTERISK # *** Tokens *** +# https://en.wikipedia.org/wiki/Byte_order_mark +byte_order_mark <- '\357\273\277' + # Unfortunately, there is not a standard way to match the start of the file in PEG. # This rule definition is compatible with the original peg(1) tool but is not # portable. It is however trivial to implement an equivalent rule in hand written # parsers and other PEG tooling should have similar mechanisms. -sof <- &{ (yy->__pos == 0) } +sof <- &{ (yy->__pos == 0) } byte_order_mark? eof <- !. ox80_oxBF <- [\200-\277] @@ -536,7 +539,7 @@ line_string <- '\\\\' non_control_utf8* newline # the newline terminating a multiline string literal or doc comment visible # to the pre_op_white non-terminal. newline <- &("\n" / "\r\n" / eof) -skip <- ([ \n\t\r] / line_comment)* +skip <- sof? ([ \n\t\r] / line_comment)* skip_require_newline <- [ \t\r]* ([\n] / line_comment) skip pre_op_white <- ([ \n\t\r] / line_comment)+ post_op_white <- [ \n\t\r] skip diff --git a/lib/std/zig/parser_fuzz.zig b/lib/std/zig/parser_fuzz.zig index 6250d1bb837030f658100b71a96a7a05cb94e970..acb5bb59ba362da8148cfc1f76dc408bbaf02d65 100644 --- a/lib/std/zig/parser_fuzz.zig +++ b/lib/std/zig/parser_fuzz.zig @@ -119,6 +119,14 @@ test "catch capture whitespace" { try checkAgainstOracle("test{0 catch |h|0;}"); } +// Found using AFL++ +test "byte order mark" { + // https://en.wikipedia.org/wiki/Byte_order_mark + try checkAgainstOracle("\xef\xbb\xbf"); + try checkAgainstOracle("\xef\xbb\xbf///\n0"); +} + +>>>>>>> bc46a69fba (grammar: handle byte order mark) fn checkAgainstOracle(source: [:0]const u8) !void { var fba_buf: [1 << 18]u8 = undefined; var fba: std.heap.FixedBufferAllocator = .init(&fba_buf); diff --git a/lib/std/zig/parser_generated_oracle.zig b/lib/std/zig/parser_generated_oracle.zig index e20f0c2b347ce36be29d4b8957809d503eaf17ab..41a86a9ee2e3cd8698bba712e3846ce492a2d38f 100644 --- a/lib/std/zig/parser_generated_oracle.zig +++ b/lib/std/zig/parser_generated_oracle.zig @@ -2327,10 +2327,24 @@ const Parser = struct { break :blk_0 false; }; } + pub fn parsebyte_order_mark(p: *Parser) Error!bool { + return blk_0: { + const pos_0 = p.i; + if (blk_1: { + if (std.mem.startsWith(u8, p.source[p.i..], "\xef\xbb\xbf")) { + p.i += 3; + break :blk_1 true; + } + break :blk_1 false; + }) break :blk_0 true; + p.i = pos_0; + break :blk_0 false; + }; + } pub fn parsesof(p: *Parser) Error!bool { return blk_0: { const pos_0 = p.i; - if ((p.i == 0)) break :blk_0 true; + if ((p.i == 0) and (try p.parsebyte_order_mark() or true)) break :blk_0 true; p.i = pos_0; break :blk_0 false; }; @@ -2854,7 +2868,7 @@ const Parser = struct { pub fn parseskip(p: *Parser) Error!bool { return blk_0: { const pos_0 = p.i; - if (blk_1: { + if ((try p.parsesof() or true) and blk_1: { var i_1: usize = 0; while (blk_3: { const pos_3 = p.i;