authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-13 20:42:18+03:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-13 20:42:18+03:00
log23c5ff94e9dda07694762fcc829dbac006bb00a1
tree06cb4d00e9d80cfb0903696a28fac7e980096b9c
parentafab095b6115e189ea2c5e0947a21e55a82c9f99
signaturelock-open Commit is signed but in an unrecognized format.

improve recovery on top level declarations

4 files changed, 127 insertions(+), 69 deletions(-)

lib/std/zig/ast.zig+2
...@@ -893,6 +893,7 @@ pub const Node = struct {...@@ -893,6 +893,7 @@ pub const Node = struct {
893 pub const ReturnType = union(enum) {893 pub const ReturnType = union(enum) {
894 Explicit: *Node,894 Explicit: *Node,
895 InferErrorSet: *Node,895 InferErrorSet: *Node,
896 Invalid,
896 };897 };
897898
898 pub fn iterate(self: *FnProto, index: usize) ?*Node {899 pub fn iterate(self: *FnProto, index: usize) ?*Node {
...@@ -942,6 +943,7 @@ pub const Node = struct {...@@ -942,6 +943,7 @@ pub const Node = struct {
942 if (self.body_node) |body_node| return body_node.lastToken();943 if (self.body_node) |body_node| return body_node.lastToken();
943 switch (self.return_type) {944 switch (self.return_type) {
944 .Explicit, .InferErrorSet => |node| return node.lastToken(),945 .Explicit, .InferErrorSet => |node| return node.lastToken(),
946 .Invalid => unreachable,
945 }947 }
946 }948 }
947 };949 };
lib/std/zig/parse.zig+91-66
...@@ -130,15 +130,7 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All...@@ -130,15 +130,7 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
130130
131 const visib_token = eatToken(it, .Keyword_pub);131 const visib_token = eatToken(it, .Keyword_pub);
132132
133 if (parseTopLevelDecl(arena, it, tree) catch |err| switch (err) {133 if (try parseTopLevelDecl(arena, it, tree)) |node| {
134 error.OutOfMemory => return error.OutOfMemory,
135 error.ParseError => {
136 // attempt to recover by finding a semicolon
137 // TODO if this was a function with a body we should use findEndOfBlock
138 findToken(it, .Semicolon);
139 continue;
140 },
141 }) |node| {
142 if (field_state == .seen) {134 if (field_state == .seen) {
143 field_state = .{ .end = visib_token orelse node.firstToken() };135 field_state = .{ .end = visib_token orelse node.firstToken() };
144 }136 }
...@@ -217,43 +209,49 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All...@@ -217,43 +209,49 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
217 return list;209 return list;
218}210}
219211
220/// Attempts to find a closing brace, assumes the opening brace was found.212/// Attempts to find a closing brace.
221fn findEndOfBlock(it: *TokenIterator) void {213fn findEndOfBlock(it: *TokenIterator) void {
222 var count: u32 = 1;214 var count: u32 = 0;
223 while (it.next()) |tok| switch (tok.id) {215 while (true) {
224 .LBrace => count += 1,216 const tok = nextToken(it);
225 .RBrace => {217 switch (tok.ptr.id) {
226 count -= 1;218 .LBrace => count += 1,
227 if (count == 0) return;219 .RBrace => {
228 },220 if (count <= 1) return;
229 .Eof => {221 count -= 1;
230 _ = it.prev();222 },
231 return;223 .Eof => {
232 },224 putBackToken(it, tok.index);
233 else => {},225 return;
234 };226 },
227 else => {},
228 }
229 }
235}230}
236231
237/// Attempts to find `wanted` token, keeps track of parentheses.232/// Attempts to find `wanted` token, keeps track of parentheses.
238fn findToken(it: *TokenIterator, wanted: Token.Id) void {233fn findToken(it: *TokenIterator, wanted: Token.Id) void {
239 var count: u32 = 0;234 var count: u32 = 0;
240 while (it.next()) |tok| switch (tok.id) {235 while (true) {
241 .LParen, .LBracket, .LBrace => count += 1,236 const tok = nextToken(it);
242 .RParen, .RBracket, .RBrace => {237 switch (tok.ptr.id) {
243 if (count == 0) {238 .LParen, .LBracket, .LBrace => count += 1,
244 _ = it.prev();239 .RParen, .RBracket, .RBrace => {
240 if (count == 0) {
241 putBackToken(it, tok.index);
242 return;
243 }
244 count -= 1;
245 },
246 .Eof => {
247 putBackToken(it, tok.index);
245 return;248 return;
246 }249 },
247 count -= 1;250 else => {
248 },251 if (tok.ptr.id == wanted and count == 0) return;
249 .Eof => {252 },
250 _ = it.prev();253 }
251 return;254 }
252 },
253 else => {
254 if (tok.id == wanted and count == 0) return;
255 },
256 };
257}255}
258256
259/// Eat a multiline container doc comment257/// Eat a multiline container doc comment
...@@ -317,7 +315,7 @@ fn parseTopLevelComptime(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*...@@ -317,7 +315,7 @@ fn parseTopLevelComptime(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*
317/// <- (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block)315/// <- (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block)
318/// / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? VarDecl316/// / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? VarDecl
319/// / KEYWORD_usingnamespace Expr SEMICOLON317/// / KEYWORD_usingnamespace Expr SEMICOLON
320fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {318fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error!?*Node {
321 var lib_name: ?*Node = null;319 var lib_name: ?*Node = null;
322 const extern_export_inline_token = blk: {320 const extern_export_inline_token = blk: {
323 if (eatToken(it, .Keyword_export)) |token| break :blk token;321 if (eatToken(it, .Keyword_export)) |token| break :blk token;
...@@ -330,12 +328,26 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -330,12 +328,26 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
330 break :blk null;328 break :blk null;
331 };329 };
332330
333 if (try parseFnProto(arena, it, tree)) |node| {331 if (parseFnProto(arena, it, tree) catch |err| switch (err) {
332 error.OutOfMemory => return error.OutOfMemory,
333 error.ParseError => {
334 // this fn will likely have a body so we
335 // use findEndOfBlock instead of findToken.
336 findEndOfBlock(it);
337 return null;
338 },
339 }) |node| {
334 const fn_node = node.cast(Node.FnProto).?;340 const fn_node = node.cast(Node.FnProto).?;
335 fn_node.*.extern_export_inline_token = extern_export_inline_token;341 fn_node.*.extern_export_inline_token = extern_export_inline_token;
336 fn_node.*.lib_name = lib_name;342 fn_node.*.lib_name = lib_name;
337 if (eatToken(it, .Semicolon)) |_| return node;343 if (eatToken(it, .Semicolon)) |_| return node;
338 if (try parseBlock(arena, it, tree)) |body_node| {344 if (parseBlock(arena, it, tree) catch |err| switch (err) {
345 error.OutOfMemory => return error.OutOfMemory,
346 // since parseBlock only return error.ParseError on
347 // a missing '}' we can assume this function was
348 // supposed to end here.
349 error.ParseError => null,
350 }) |body_node| {
339 fn_node.body_node = body_node;351 fn_node.body_node = body_node;
340 return node;352 return node;
341 }353 }
...@@ -356,7 +368,14 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -356,7 +368,14 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
356368
357 const thread_local_token = eatToken(it, .Keyword_threadlocal);369 const thread_local_token = eatToken(it, .Keyword_threadlocal);
358370
359 if (try parseVarDecl(arena, it, tree)) |node| {371 if (parseVarDecl(arena, it, tree) catch |err| switch (err) {
372 error.OutOfMemory => return error.OutOfMemory,
373 error.ParseError => {
374 // try to skip to next decl
375 findToken(it, .Semicolon);
376 return null;
377 },
378 }) |node| {
360 var var_decl = node.cast(Node.VarDecl).?;379 var var_decl = node.cast(Node.VarDecl).?;
361 var_decl.*.thread_local_token = thread_local_token;380 var_decl.*.thread_local_token = thread_local_token;
362 var_decl.*.comptime_token = null;381 var_decl.*.comptime_token = null;
...@@ -369,7 +388,8 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -369,7 +388,8 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
369 try tree.errors.push(.{388 try tree.errors.push(.{
370 .ExpectedVarDecl = .{ .token = it.index },389 .ExpectedVarDecl = .{ .token = it.index },
371 });390 });
372 return error.ParseError;391 // ignore this, try to find next decl by skipping the next block
392 findEndOfBlock(it);
373 }393 }
374394
375 if (extern_export_inline_token) |token| {395 if (extern_export_inline_token) |token| {
...@@ -379,16 +399,14 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -379,16 +399,14 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
379 return null;399 return null;
380 }400 }
381401
382 const use_node = (try parseUse(arena, it, tree)) orelse return null;402 return parseUse(arena, it, tree) catch |err| switch (err) {
383 const expr_node = try expectNode(arena, it, tree, parseExpr, .{403 error.OutOfMemory => return error.OutOfMemory,
384 .ExpectedExpr = .{ .token = it.index },404 error.ParseError => {
385 });405 // try to skip to next decl
386 const semicolon_token = try expectToken(it, tree, .Semicolon);406 findToken(it, .Semicolon);
387 const use_node_raw = use_node.cast(Node.Use).?;407 return null;
388 use_node_raw.*.expr = expr_node;408 },
389 use_node_raw.*.semicolon_token = semicolon_token;409 };
390
391 return use_node;
392}410}
393411
394/// FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? LinkSection? EXCLAMATIONMARK? (KEYWORD_var / TypeExpr)412/// FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? LinkSection? EXCLAMATIONMARK? (KEYWORD_var / TypeExpr)
...@@ -422,18 +440,23 @@ fn parseFnProto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {...@@ -422,18 +440,23 @@ fn parseFnProto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
422 const exclamation_token = eatToken(it, .Bang);440 const exclamation_token = eatToken(it, .Bang);
423441
424 const return_type_expr = (try parseVarType(arena, it, tree)) orelse442 const return_type_expr = (try parseVarType(arena, it, tree)) orelse
425 try expectNode(arena, it, tree, parseTypeExpr, .{443 (try parseTypeExpr(arena, it, tree)) orelse blk: {
426 .ExpectedReturnType = .{ .token = it.index },444 try tree.errors.push(.{
427 });445 .ExpectedReturnType = .{ .token = it.index },
446 });
447 // most likely the user forgot to specify the return type.
448 // Mark return type as invalid and try to continue.
449 break :blk null;
450 };
428451
429 const return_type: Node.FnProto.ReturnType = if (exclamation_token != null)452 // TODO https://github.com/ziglang/zig/issues/3750
430 .{453 const R = Node.FnProto.ReturnType;
431 .InferErrorSet = return_type_expr,454 const return_type = if (return_type_expr == null)
432 }455 R{ .Invalid = {} }
456 else if (exclamation_token != null)
457 R{ .InferErrorSet = return_type_expr.? }
433 else458 else
434 .{459 R{ .Explicit = return_type_expr.? };
435 .Explicit = return_type_expr,
436 };
437460
438 const var_args_token = if (params.len > 0)461 const var_args_token = if (params.len > 0)
439 params.at(params.len - 1).*.cast(Node.ParamDecl).?.var_args_token462 params.at(params.len - 1).*.cast(Node.ParamDecl).?.var_args_token
...@@ -2992,8 +3015,10 @@ fn parseUse(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {...@@ -2992,8 +3015,10 @@ fn parseUse(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
2992 .doc_comments = null,3015 .doc_comments = null,
2993 .visib_token = null,3016 .visib_token = null,
2994 .use_token = token,3017 .use_token = token,
2995 .expr = undefined, // set by caller3018 .expr = try expectNode(arena, it, tree, parseExpr, .{
2996 .semicolon_token = undefined, // set by caller3019 .ExpectedExpr = .{ .token = it.index },
3020 }),
3021 .semicolon_token = try expectToken(it, tree, .Semicolon),
2997 };3022 };
2998 return &node.base;3023 return &node.base;
2999}3024}
lib/std/zig/parser_test.zig+30-1
...@@ -1,4 +1,4 @@...@@ -1,4 +1,4 @@
1test "zig fmt: fault tolerant parsing" {1test "recovery: top level" {
2 try testError(2 try testError(
3 \\test "" {inline}3 \\test "" {inline}
4 \\test "" {inline}4 \\test "" {inline}
...@@ -6,6 +6,9 @@ test "zig fmt: fault tolerant parsing" {...@@ -6,6 +6,9 @@ test "zig fmt: fault tolerant parsing" {
6 .ExpectedInlinable,6 .ExpectedInlinable,
7 .ExpectedInlinable,7 .ExpectedInlinable,
8 });8 });
9}
10
11test "recovery: block statements" {
9 try testError(12 try testError(
10 \\test "" {13 \\test "" {
11 \\ foo + +;14 \\ foo + +;
...@@ -15,6 +18,9 @@ test "zig fmt: fault tolerant parsing" {...@@ -15,6 +18,9 @@ test "zig fmt: fault tolerant parsing" {
15 .InvalidToken,18 .InvalidToken,
16 .ExpectedInlinable,19 .ExpectedInlinable,
17 });20 });
21}
22
23test "recovery: missing comma" {
18 try testError(24 try testError(
19 \\test "" {25 \\test "" {
20 \\ switch (foo) {26 \\ switch (foo) {
...@@ -33,6 +39,29 @@ test "zig fmt: fault tolerant parsing" {...@@ -33,6 +39,29 @@ test "zig fmt: fault tolerant parsing" {
33 });39 });
34}40}
3541
42test "recovery: extra qualifier" {
43 try testError(
44 \\const a: *const const u8;
45 \\test ""
46 , &[_]Error{
47 .ExtraConstQualifier,
48 .ExpectedLBrace,
49 });
50}
51
52test "recovery: missing return type" {
53 try testError(
54 \\fn foo() {
55 \\ a && b;
56 \\}
57 \\test ""
58 , &[_]Error{
59 .ExpectedReturnType,
60 .InvalidAnd,
61 .ExpectedLBrace,
62 });
63}
64
36test "zig fmt: top-level fields" {65test "zig fmt: top-level fields" {
37 try testCanonical(66 try testCanonical(
38 \\a: did_you_know,67 \\a: did_you_know,
lib/std/zig/render.zig+4-2
...@@ -1444,6 +1444,7 @@ fn renderExpression(...@@ -1444,6 +1444,7 @@ fn renderExpression(
1444 else switch (fn_proto.return_type) {1444 else switch (fn_proto.return_type) {
1445 .Explicit => |node| node.firstToken(),1445 .Explicit => |node| node.firstToken(),
1446 .InferErrorSet => |node| tree.prevToken(node.firstToken()),1446 .InferErrorSet => |node| tree.prevToken(node.firstToken()),
1447 .Invalid => unreachable,
1447 });1448 });
1448 assert(tree.tokens.at(rparen).id == .RParen);1449 assert(tree.tokens.at(rparen).id == .RParen);
14491450
...@@ -1518,13 +1519,14 @@ fn renderExpression(...@@ -1518,13 +1519,14 @@ fn renderExpression(
1518 }1519 }
15191520
1520 switch (fn_proto.return_type) {1521 switch (fn_proto.return_type) {
1521 ast.Node.FnProto.ReturnType.Explicit => |node| {1522 .Explicit => |node| {
1522 return renderExpression(allocator, stream, tree, indent, start_col, node, space);1523 return renderExpression(allocator, stream, tree, indent, start_col, node, space);
1523 },1524 },
1524 ast.Node.FnProto.ReturnType.InferErrorSet => |node| {1525 .InferErrorSet => |node| {
1525 try renderToken(tree, stream, tree.prevToken(node.firstToken()), indent, start_col, Space.None); // !1526 try renderToken(tree, stream, tree.prevToken(node.firstToken()), indent, start_col, Space.None); // !
1526 return renderExpression(allocator, stream, tree, indent, start_col, node, space);1527 return renderExpression(allocator, stream, tree, indent, start_col, node, space);
1527 },1528 },
1529 .Invalid => unreachable,
1528 }1530 }
1529 },1531 },
15301532