authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-13 17:36:06+03:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-13 17:36:06+03:00
logcefc04348e79f85118ad3c7b7fd15a4a5d635d50
tree09384b471d1fa34206843bda62591b11f3eac5b7
parentbe392777b763028a02866342b12915a1e69ebdf1
signaturelock-open Commit is signed but in an unrecognized format.

continue parsing on invalid and token

3 files changed, 22 insertions(+), 15 deletions(-)

lib/std/zig/ast.zig+4-3
...@@ -166,6 +166,7 @@ pub const Error = union(enum) {...@@ -166,6 +166,7 @@ pub const Error = union(enum) {
166 ExpectedSuffixOp: ExpectedSuffixOp,166 ExpectedSuffixOp: ExpectedSuffixOp,
167 DeclBetweenFields: DeclBetweenFields,167 DeclBetweenFields: DeclBetweenFields,
168 MissingComma: MissingComma,168 MissingComma: MissingComma,
169 InvalidAnd: InvalidAnd,
169170
170 pub fn render(self: *const Error, tokens: *Tree.TokenList, stream: var) !void {171 pub fn render(self: *const Error, tokens: *Tree.TokenList, stream: var) !void {
171 switch (self.*) {172 switch (self.*) {
...@@ -215,6 +216,7 @@ pub const Error = union(enum) {...@@ -215,6 +216,7 @@ pub const Error = union(enum) {
215 .ExpectedSuffixOp => |*x| return x.render(tokens, stream),216 .ExpectedSuffixOp => |*x| return x.render(tokens, stream),
216 .DeclBetweenFields => |*x| return x.render(tokens, stream),217 .DeclBetweenFields => |*x| return x.render(tokens, stream),
217 .MissingComma => |*x| return x.render(tokens, stream),218 .MissingComma => |*x| return x.render(tokens, stream),
219 .InvalidAnd => |*x| return x.render(tokens, stream),
218 }220 }
219 }221 }
220222
...@@ -266,6 +268,7 @@ pub const Error = union(enum) {...@@ -266,6 +268,7 @@ pub const Error = union(enum) {
266 .ExpectedSuffixOp => |x| return x.token,268 .ExpectedSuffixOp => |x| return x.token,
267 .DeclBetweenFields => |x| return x.token,269 .DeclBetweenFields => |x| return x.token,
268 .MissingComma => |x| return x.token,270 .MissingComma => |x| return x.token,
271 .InvalidAnd => |x| return x.token,
269 }272 }
270 }273 }
271274
...@@ -312,6 +315,7 @@ pub const Error = union(enum) {...@@ -312,6 +315,7 @@ pub const Error = union(enum) {
312 pub const ExtraAllowZeroQualifier = SimpleError("Extra allowzero qualifier");315 pub const ExtraAllowZeroQualifier = SimpleError("Extra allowzero qualifier");
313 pub const DeclBetweenFields = SimpleError("Declarations are not allowed between container fields");316 pub const DeclBetweenFields = SimpleError("Declarations are not allowed between container fields");
314 pub const MissingComma = SimpleError("Expected comma between items");317 pub const MissingComma = SimpleError("Expected comma between items");
318 pub const InvalidAnd = SimpleError("`&&` is invalid. Note that `and` is boolean AND.");
315319
316 pub const ExpectedCall = struct {320 pub const ExpectedCall = struct {
317 node: *Node,321 node: *Node,
...@@ -339,9 +343,6 @@ pub const Error = union(enum) {...@@ -339,9 +343,6 @@ pub const Error = union(enum) {
339 pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void {343 pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void {
340 const found_token = tokens.at(self.token);344 const found_token = tokens.at(self.token);
341 switch (found_token.id) {345 switch (found_token.id) {
342 .Invalid_ampersands => {
343 return stream.print("`&&` is invalid. Note that `and` is boolean AND.", .{});
344 },
345 .Invalid => {346 .Invalid => {
346 return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()});347 return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()});
347 },348 },
lib/std/zig/parse.zig+15-10
...@@ -2824,21 +2824,16 @@ fn ListParseFn(comptime L: type, comptime nodeParseFn: var) ParseFn(L) {...@@ -2824,21 +2824,16 @@ fn ListParseFn(comptime L: type, comptime nodeParseFn: var) ParseFn(L) {
2824 while (try nodeParseFn(arena, it, tree)) |node| {2824 while (try nodeParseFn(arena, it, tree)) |node| {
2825 try list.push(node);2825 try list.push(node);
28262826
2827 const token = nextToken(it);2827 switch (it.peek().?.id) {
2828 switch (token.ptr.id) {2828 .Comma => _ = nextToken(it),
2829 .Comma => {},
2830 // all possible delimiters2829 // all possible delimiters
2831 .Colon, .RParen, .RBrace, .RBracket => {2830 .Colon, .RParen, .RBrace, .RBracket => break,
2832 putBackToken(it, token.index);
2833 break;
2834 },
2835 else => {2831 else => {
2836 // this is likely just a missing comma,2832 // this is likely just a missing comma,
2837 // continue parsing this list and give an error2833 // continue parsing this list and give an error
2838 try tree.errors.push(.{2834 try tree.errors.push(.{
2839 .MissingComma = .{ .token = token.index },2835 .MissingComma = .{ .token = it.index },
2840 });2836 });
2841 putBackToken(it, token.index);
2842 },2837 },
2843 }2838 }
2844 }2839 }
...@@ -2850,7 +2845,17 @@ fn ListParseFn(comptime L: type, comptime nodeParseFn: var) ParseFn(L) {...@@ -2850,7 +2845,17 @@ fn ListParseFn(comptime L: type, comptime nodeParseFn: var) ParseFn(L) {
2850fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) NodeParseFn {2845fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) NodeParseFn {
2851 return struct {2846 return struct {
2852 pub fn parse(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?*Node {2847 pub fn parse(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?*Node {
2853 const op_token = eatToken(it, token) orelse return null;2848 const op_token = if (token == .Keyword_and) switch (it.peek().?.id) {
2849 .Keyword_and => nextToken(it).index,
2850 .Invalid_ampersands => blk: {
2851 try tree.errors.push(.{
2852 .InvalidAnd = .{ .token = it.index },
2853 });
2854 break :blk nextToken(it).index;
2855 },
2856 else => return null,
2857 } else eatToken(it, token) orelse return null;
2858
2854 const node = try arena.create(Node.InfixOp);2859 const node = try arena.create(Node.InfixOp);
2855 node.* = .{2860 node.* = .{
2856 .op_token = op_token,2861 .op_token = op_token,
lib/std/zig/parser_test.zig+3-2
...@@ -21,14 +21,15 @@ test "zig fmt: fault tolerant parsing" {...@@ -21,14 +21,15 @@ test "zig fmt: fault tolerant parsing" {
21 \\ 2 => {}21 \\ 2 => {}
22 \\ 3 => {}22 \\ 3 => {}
23 \\ else => {23 \\ else => {
24 \\ inline;24 \\ foo && bar +;
25 \\ }25 \\ }
26 \\ }26 \\ }
27 \\}27 \\}
28 , &[_]Error{28 , &[_]Error{
29 .MissingComma,29 .MissingComma,
30 .MissingComma,30 .MissingComma,
31 .ExpectedInlinable,31 .InvalidAnd,
32 .InvalidToken,
32 });33 });
33}34}
3435