authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-13 20:42:18+03:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-13 20:42:18+03:00
log23c5ff94e9dda07694762fcc829dbac006bb00a1
tree06cb4d00e9d80cfb0903696a28fac7e980096b9c
parentafab095b6115e189ea2c5e0947a21e55a82c9f99
signaturelock-open Commit is signed but in an unrecognized format.

improve recovery on top level declarations

4 files changed, 127 insertions(+), 69 deletions(-)

lib/std/zig/ast.zig+2
......@@ -893,6 +893,7 @@ pub const Node = struct {
893893 pub const ReturnType = union(enum) {
894894 Explicit: *Node,
895895 InferErrorSet: *Node,
896 Invalid,
896897 };
897898
898899 pub fn iterate(self: *FnProto, index: usize) ?*Node {
......@@ -942,6 +943,7 @@ pub const Node = struct {
942943 if (self.body_node) |body_node| return body_node.lastToken();
943944 switch (self.return_type) {
944945 .Explicit, .InferErrorSet => |node| return node.lastToken(),
946 .Invalid => unreachable,
945947 }
946948 }
947949 };
lib/std/zig/parse.zig+91-66
......@@ -130,15 +130,7 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
130130
131131 const visib_token = eatToken(it, .Keyword_pub);
132132
133 if (parseTopLevelDecl(arena, it, tree) catch |err| switch (err) {
134 error.OutOfMemory => return error.OutOfMemory,
135 error.ParseError => {
136 // attempt to recover by finding a semicolon
137 // TODO if this was a function with a body we should use findEndOfBlock
138 findToken(it, .Semicolon);
139 continue;
140 },
141 }) |node| {
133 if (try parseTopLevelDecl(arena, it, tree)) |node| {
142134 if (field_state == .seen) {
143135 field_state = .{ .end = visib_token orelse node.firstToken() };
144136 }
......@@ -217,43 +209,49 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
217209 return list;
218210}
219211
220/// Attempts to find a closing brace, assumes the opening brace was found.
212/// Attempts to find a closing brace.
221213fn findEndOfBlock(it: *TokenIterator) void {
222 var count: u32 = 1;
223 while (it.next()) |tok| switch (tok.id) {
224 .LBrace => count += 1,
225 .RBrace => {
226 count -= 1;
227 if (count == 0) return;
228 },
229 .Eof => {
230 _ = it.prev();
231 return;
232 },
233 else => {},
234 };
214 var count: u32 = 0;
215 while (true) {
216 const tok = nextToken(it);
217 switch (tok.ptr.id) {
218 .LBrace => count += 1,
219 .RBrace => {
220 if (count <= 1) return;
221 count -= 1;
222 },
223 .Eof => {
224 putBackToken(it, tok.index);
225 return;
226 },
227 else => {},
228 }
229 }
235230}
236231
237232/// Attempts to find `wanted` token, keeps track of parentheses.
238233fn findToken(it: *TokenIterator, wanted: Token.Id) void {
239234 var count: u32 = 0;
240 while (it.next()) |tok| switch (tok.id) {
241 .LParen, .LBracket, .LBrace => count += 1,
242 .RParen, .RBracket, .RBrace => {
243 if (count == 0) {
244 _ = it.prev();
235 while (true) {
236 const tok = nextToken(it);
237 switch (tok.ptr.id) {
238 .LParen, .LBracket, .LBrace => count += 1,
239 .RParen, .RBracket, .RBrace => {
240 if (count == 0) {
241 putBackToken(it, tok.index);
242 return;
243 }
244 count -= 1;
245 },
246 .Eof => {
247 putBackToken(it, tok.index);
245248 return;
246 }
247 count -= 1;
248 },
249 .Eof => {
250 _ = it.prev();
251 return;
252 },
253 else => {
254 if (tok.id == wanted and count == 0) return;
255 },
256 };
249 },
250 else => {
251 if (tok.ptr.id == wanted and count == 0) return;
252 },
253 }
254 }
257255}
258256
259257/// Eat a multiline container doc comment
......@@ -317,7 +315,7 @@ fn parseTopLevelComptime(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*
317315/// <- (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block)
318316/// / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? VarDecl
319317/// / KEYWORD_usingnamespace Expr SEMICOLON
320fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
318fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error!?*Node {
321319 var lib_name: ?*Node = null;
322320 const extern_export_inline_token = blk: {
323321 if (eatToken(it, .Keyword_export)) |token| break :blk token;
......@@ -330,12 +328,26 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
330328 break :blk null;
331329 };
332330
333 if (try parseFnProto(arena, it, tree)) |node| {
331 if (parseFnProto(arena, it, tree) catch |err| switch (err) {
332 error.OutOfMemory => return error.OutOfMemory,
333 error.ParseError => {
334 // this fn will likely have a body so we
335 // use findEndOfBlock instead of findToken.
336 findEndOfBlock(it);
337 return null;
338 },
339 }) |node| {
334340 const fn_node = node.cast(Node.FnProto).?;
335341 fn_node.*.extern_export_inline_token = extern_export_inline_token;
336342 fn_node.*.lib_name = lib_name;
337343 if (eatToken(it, .Semicolon)) |_| return node;
338 if (try parseBlock(arena, it, tree)) |body_node| {
344 if (parseBlock(arena, it, tree) catch |err| switch (err) {
345 error.OutOfMemory => return error.OutOfMemory,
346 // since parseBlock only return error.ParseError on
347 // a missing '}' we can assume this function was
348 // supposed to end here.
349 error.ParseError => null,
350 }) |body_node| {
339351 fn_node.body_node = body_node;
340352 return node;
341353 }
......@@ -356,7 +368,14 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
356368
357369 const thread_local_token = eatToken(it, .Keyword_threadlocal);
358370
359 if (try parseVarDecl(arena, it, tree)) |node| {
371 if (parseVarDecl(arena, it, tree) catch |err| switch (err) {
372 error.OutOfMemory => return error.OutOfMemory,
373 error.ParseError => {
374 // try to skip to next decl
375 findToken(it, .Semicolon);
376 return null;
377 },
378 }) |node| {
360379 var var_decl = node.cast(Node.VarDecl).?;
361380 var_decl.*.thread_local_token = thread_local_token;
362381 var_decl.*.comptime_token = null;
......@@ -369,7 +388,8 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
369388 try tree.errors.push(.{
370389 .ExpectedVarDecl = .{ .token = it.index },
371390 });
372 return error.ParseError;
391 // ignore this, try to find next decl by skipping the next block
392 findEndOfBlock(it);
373393 }
374394
375395 if (extern_export_inline_token) |token| {
......@@ -379,16 +399,14 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
379399 return null;
380400 }
381401
382 const use_node = (try parseUse(arena, it, tree)) orelse return null;
383 const expr_node = try expectNode(arena, it, tree, parseExpr, .{
384 .ExpectedExpr = .{ .token = it.index },
385 });
386 const semicolon_token = try expectToken(it, tree, .Semicolon);
387 const use_node_raw = use_node.cast(Node.Use).?;
388 use_node_raw.*.expr = expr_node;
389 use_node_raw.*.semicolon_token = semicolon_token;
390
391 return use_node;
402 return parseUse(arena, it, tree) catch |err| switch (err) {
403 error.OutOfMemory => return error.OutOfMemory,
404 error.ParseError => {
405 // try to skip to next decl
406 findToken(it, .Semicolon);
407 return null;
408 },
409 };
392410}
393411
394412/// FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? LinkSection? EXCLAMATIONMARK? (KEYWORD_var / TypeExpr)
......@@ -422,18 +440,23 @@ fn parseFnProto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
422440 const exclamation_token = eatToken(it, .Bang);
423441
424442 const return_type_expr = (try parseVarType(arena, it, tree)) orelse
425 try expectNode(arena, it, tree, parseTypeExpr, .{
426 .ExpectedReturnType = .{ .token = it.index },
427 });
443 (try parseTypeExpr(arena, it, tree)) orelse blk: {
444 try tree.errors.push(.{
445 .ExpectedReturnType = .{ .token = it.index },
446 });
447 // most likely the user forgot to specify the return type.
448 // Mark return type as invalid and try to continue.
449 break :blk null;
450 };
428451
429 const return_type: Node.FnProto.ReturnType = if (exclamation_token != null)
430 .{
431 .InferErrorSet = return_type_expr,
432 }
452 // TODO https://github.com/ziglang/zig/issues/3750
453 const R = Node.FnProto.ReturnType;
454 const return_type = if (return_type_expr == null)
455 R{ .Invalid = {} }
456 else if (exclamation_token != null)
457 R{ .InferErrorSet = return_type_expr.? }
433458 else
434 .{
435 .Explicit = return_type_expr,
436 };
459 R{ .Explicit = return_type_expr.? };
437460
438461 const var_args_token = if (params.len > 0)
439462 params.at(params.len - 1).*.cast(Node.ParamDecl).?.var_args_token
......@@ -2992,8 +3015,10 @@ fn parseUse(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
29923015 .doc_comments = null,
29933016 .visib_token = null,
29943017 .use_token = token,
2995 .expr = undefined, // set by caller
2996 .semicolon_token = undefined, // set by caller
3018 .expr = try expectNode(arena, it, tree, parseExpr, .{
3019 .ExpectedExpr = .{ .token = it.index },
3020 }),
3021 .semicolon_token = try expectToken(it, tree, .Semicolon),
29973022 };
29983023 return &node.base;
29993024}
lib/std/zig/parser_test.zig+30-1
......@@ -1,4 +1,4 @@
1test "zig fmt: fault tolerant parsing" {
1test "recovery: top level" {
22 try testError(
33 \\test "" {inline}
44 \\test "" {inline}
......@@ -6,6 +6,9 @@ test "zig fmt: fault tolerant parsing" {
66 .ExpectedInlinable,
77 .ExpectedInlinable,
88 });
9}
10
11test "recovery: block statements" {
912 try testError(
1013 \\test "" {
1114 \\ foo + +;
......@@ -15,6 +18,9 @@ test "zig fmt: fault tolerant parsing" {
1518 .InvalidToken,
1619 .ExpectedInlinable,
1720 });
21}
22
23test "recovery: missing comma" {
1824 try testError(
1925 \\test "" {
2026 \\ switch (foo) {
......@@ -33,6 +39,29 @@ test "zig fmt: fault tolerant parsing" {
3339 });
3440}
3541
42test "recovery: extra qualifier" {
43 try testError(
44 \\const a: *const const u8;
45 \\test ""
46 , &[_]Error{
47 .ExtraConstQualifier,
48 .ExpectedLBrace,
49 });
50}
51
52test "recovery: missing return type" {
53 try testError(
54 \\fn foo() {
55 \\ a && b;
56 \\}
57 \\test ""
58 , &[_]Error{
59 .ExpectedReturnType,
60 .InvalidAnd,
61 .ExpectedLBrace,
62 });
63}
64
3665test "zig fmt: top-level fields" {
3766 try testCanonical(
3867 \\a: did_you_know,
lib/std/zig/render.zig+4-2
......@@ -1444,6 +1444,7 @@ fn renderExpression(
14441444 else switch (fn_proto.return_type) {
14451445 .Explicit => |node| node.firstToken(),
14461446 .InferErrorSet => |node| tree.prevToken(node.firstToken()),
1447 .Invalid => unreachable,
14471448 });
14481449 assert(tree.tokens.at(rparen).id == .RParen);
14491450
......@@ -1518,13 +1519,14 @@ fn renderExpression(
15181519 }
15191520
15201521 switch (fn_proto.return_type) {
1521 ast.Node.FnProto.ReturnType.Explicit => |node| {
1522 .Explicit => |node| {
15221523 return renderExpression(allocator, stream, tree, indent, start_col, node, space);
15231524 },
1524 ast.Node.FnProto.ReturnType.InferErrorSet => |node| {
1525 .InferErrorSet => |node| {
15251526 try renderToken(tree, stream, tree.prevToken(node.firstToken()), indent, start_col, Space.None); // !
15261527 return renderExpression(allocator, stream, tree, indent, start_col, node, space);
15271528 },
1529 .Invalid => unreachable,
15281530 }
15291531 },
15301532