| author |  Veikka Tuominen <git@vexu.eu> 2020-05-13 17:36:06+03:00 |
| committer | Veikka Tuominen <git@vexu.eu> 2020-05-13 17:36:06+03:00 |
| log | cefc04348e79f85118ad3c7b7fd15a4a5d635d50 |
| tree | 09384b471d1fa34206843bda62591b11f3eac5b7 |
| parent | be392777b763028a02866342b12915a1e69ebdf1 |
| signature |  Commit is signed but in an unrecognized format. |
continue parsing on invalid and token
3 files changed, 22 insertions(+), 15 deletions(-)
lib
lib/std/zig/ast.zig+4-3
| ... | ... | @@ -166,6 +166,7 @@ pub const Error = union(enum) { |
| 166 | 166 | ExpectedSuffixOp: ExpectedSuffixOp, |
| 167 | 167 | DeclBetweenFields: DeclBetweenFields, |
| 168 | 168 | MissingComma: MissingComma, |
| 169 | InvalidAnd: InvalidAnd, | |
| 169 | 170 | |
| 170 | 171 | pub fn render(self: *const Error, tokens: *Tree.TokenList, stream: var) !void { |
| 171 | 172 | switch (self.*) { |
| ... | ... | @@ -215,6 +216,7 @@ pub const Error = union(enum) { |
| 215 | 216 | .ExpectedSuffixOp => |*x| return x.render(tokens, stream), |
| 216 | 217 | .DeclBetweenFields => |*x| return x.render(tokens, stream), |
| 217 | 218 | .MissingComma => |*x| return x.render(tokens, stream), |
| 219 | .InvalidAnd => |*x| return x.render(tokens, stream), | |
| 218 | 220 | } |
| 219 | 221 | } |
| 220 | 222 | |
| ... | ... | @@ -266,6 +268,7 @@ pub const Error = union(enum) { |
| 266 | 268 | .ExpectedSuffixOp => |x| return x.token, |
| 267 | 269 | .DeclBetweenFields => |x| return x.token, |
| 268 | 270 | .MissingComma => |x| return x.token, |
| 271 | .InvalidAnd => |x| return x.token, | |
| 269 | 272 | } |
| 270 | 273 | } |
| 271 | 274 | |
| ... | ... | @@ -312,6 +315,7 @@ pub const Error = union(enum) { |
| 312 | 315 | pub const ExtraAllowZeroQualifier = SimpleError("Extra allowzero qualifier"); |
| 313 | 316 | pub const DeclBetweenFields = SimpleError("Declarations are not allowed between container fields"); |
| 314 | 317 | pub const MissingComma = SimpleError("Expected comma between items"); |
| 318 | pub const InvalidAnd = SimpleError("`&&` is invalid. Note that `and` is boolean AND."); | |
| 315 | 319 | |
| 316 | 320 | pub const ExpectedCall = struct { |
| 317 | 321 | node: *Node, |
| ... | ... | @@ -339,9 +343,6 @@ pub const Error = union(enum) { |
| 339 | 343 | pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void { |
| 340 | 344 | const found_token = tokens.at(self.token); |
| 341 | 345 | switch (found_token.id) { |
| 342 | .Invalid_ampersands => { | |
| 343 | return stream.print("`&&` is invalid. Note that `and` is boolean AND.", .{}); | |
| 344 | }, | |
| 345 | 346 | .Invalid => { |
| 346 | 347 | return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()}); |
| 347 | 348 | }, |
lib/std/zig/parse.zig+15-10
| ... | ... | @@ -2824,21 +2824,16 @@ fn ListParseFn(comptime L: type, comptime nodeParseFn: var) ParseFn(L) { |
| 2824 | 2824 | while (try nodeParseFn(arena, it, tree)) |node| { |
| 2825 | 2825 | try list.push(node); |
| 2826 | 2826 | |
| 2827 | const token = nextToken(it); | |
| 2828 | switch (token.ptr.id) { | |
| 2829 | .Comma => {}, | |
| 2827 | switch (it.peek().?.id) { | |
| 2828 | .Comma => _ = nextToken(it), | |
| 2830 | 2829 | // all possible delimiters |
| 2831 | .Colon, .RParen, .RBrace, .RBracket => { | |
| 2832 | putBackToken(it, token.index); | |
| 2833 | break; | |
| 2834 | }, | |
| 2830 | .Colon, .RParen, .RBrace, .RBracket => break, | |
| 2835 | 2831 | else => { |
| 2836 | 2832 | // this is likely just a missing comma, |
| 2837 | 2833 | // continue parsing this list and give an error |
| 2838 | 2834 | try tree.errors.push(.{ |
| 2839 | .MissingComma = .{ .token = token.index }, | |
| 2835 | .MissingComma = .{ .token = it.index }, | |
| 2840 | 2836 | }); |
| 2841 | putBackToken(it, token.index); | |
| 2842 | 2837 | }, |
| 2843 | 2838 | } |
| 2844 | 2839 | } |
| ... | ... | @@ -2850,7 +2845,17 @@ fn ListParseFn(comptime L: type, comptime nodeParseFn: var) ParseFn(L) { |
| 2850 | 2845 | fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) NodeParseFn { |
| 2851 | 2846 | return struct { |
| 2852 | 2847 | pub fn parse(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?*Node { |
| 2853 | const op_token = eatToken(it, token) orelse return null; | |
| 2848 | const op_token = if (token == .Keyword_and) switch (it.peek().?.id) { | |
| 2849 | .Keyword_and => nextToken(it).index, | |
| 2850 | .Invalid_ampersands => blk: { | |
| 2851 | try tree.errors.push(.{ | |
| 2852 | .InvalidAnd = .{ .token = it.index }, | |
| 2853 | }); | |
| 2854 | break :blk nextToken(it).index; | |
| 2855 | }, | |
| 2856 | else => return null, | |
| 2857 | } else eatToken(it, token) orelse return null; | |
| 2858 | ||
| 2854 | 2859 | const node = try arena.create(Node.InfixOp); |
| 2855 | 2860 | node.* = .{ |
| 2856 | 2861 | .op_token = op_token, |
lib/std/zig/parser_test.zig+3-2
| ... | ... | @@ -21,14 +21,15 @@ test "zig fmt: fault tolerant parsing" { |
| 21 | 21 | \\ 2 => {} |
| 22 | 22 | \\ 3 => {} |
| 23 | 23 | \\ else => { |
| 24 | \\ inline; | |
| 24 | \\ foo && bar +; | |
| 25 | 25 | \\ } |
| 26 | 26 | \\ } |
| 27 | 27 | \\} |
| 28 | 28 | , &[_]Error{ |
| 29 | 29 | .MissingComma, |
| 30 | 30 | .MissingComma, |
| 31 | .ExpectedInlinable, | |
| 31 | .InvalidAnd, | |
| 32 | .InvalidToken, | |
| 32 | 33 | }); |
| 33 | 34 | } |
| 34 | 35 |