authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-14 12:09:40+03:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-14 12:09:40+03:00
loga32e240540895af536fdeb98db448ee745def59a
tree1472bddfdee4110ec8c01d2462d825209697e2e4
parent89f2923a8aad5119b1506c3b8e08464e132c228e
signaturelock-open Commit is signed but in an unrecognized format.

improve recovery from invalid container members

Instead of trying to find the end of the block or the next comma/semicolon we no try to find the next token that can start a container member.

2 files changed, 99 insertions(+), 20 deletions(-)

lib/std/zig/parse.zig+78-20
...@@ -59,10 +59,13 @@ fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error...@@ -59,10 +59,13 @@ fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error
59 node.* = .{59 node.* = .{
60 .decls = try parseContainerMembers(arena, it, tree),60 .decls = try parseContainerMembers(arena, it, tree),
61 .eof_token = eatToken(it, .Eof) orelse blk: {61 .eof_token = eatToken(it, .Eof) orelse blk: {
62 // parseContainerMembers will try to skip as much
63 // invalid tokens as it can so this can only be a '}'
64 const tok = eatToken(it, .RBrace).?;
62 try tree.errors.push(.{65 try tree.errors.push(.{
63 .ExpectedContainerMembers = .{ .token = it.index },66 .ExpectedContainerMembers = .{ .token = tok },
64 });67 });
65 break :blk undefined;68 break :blk tok;
66 },69 },
67 };70 };
68 return node;71 return node;
...@@ -101,7 +104,7 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All...@@ -101,7 +104,7 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
101 if (parseTestDecl(arena, it, tree) catch |err| switch (err) {104 if (parseTestDecl(arena, it, tree) catch |err| switch (err) {
102 error.OutOfMemory => return error.OutOfMemory,105 error.OutOfMemory => return error.OutOfMemory,
103 error.ParseError => {106 error.ParseError => {
104 findEndOfBlock(it);107 findNextContainerMember(it);
105 continue;108 continue;
106 },109 },
107 }) |node| {110 }) |node| {
...@@ -116,7 +119,7 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All...@@ -116,7 +119,7 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
116 if (parseTopLevelComptime(arena, it, tree) catch |err| switch (err) {119 if (parseTopLevelComptime(arena, it, tree) catch |err| switch (err) {
117 error.OutOfMemory => return error.OutOfMemory,120 error.OutOfMemory => return error.OutOfMemory,
118 error.ParseError => {121 error.ParseError => {
119 findEndOfBlock(it);122 findNextContainerMember(it);
120 continue;123 continue;
121 },124 },
122 }) |node| {125 }) |node| {
...@@ -178,8 +181,8 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All...@@ -178,8 +181,8 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
178 if (parseContainerField(arena, it, tree) catch |err| switch (err) {181 if (parseContainerField(arena, it, tree) catch |err| switch (err) {
179 error.OutOfMemory => return error.OutOfMemory,182 error.OutOfMemory => return error.OutOfMemory,
180 error.ParseError => {183 error.ParseError => {
181 // attempt to recover by finding a comma184 // attempt to recover
182 findToken(it, .Comma);185 findNextContainerMember(it);
183 continue;186 continue;
184 },187 },
185 }) |node| {188 }) |node| {
...@@ -198,7 +201,21 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All...@@ -198,7 +201,21 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
198 const field = node.cast(Node.ContainerField).?;201 const field = node.cast(Node.ContainerField).?;
199 field.doc_comments = doc_comments;202 field.doc_comments = doc_comments;
200 try list.push(node);203 try list.push(node);
201 const comma = eatToken(it, .Comma) orelse break;204 const comma = eatToken(it, .Comma) orelse {
205 // try to continue parsing
206 const index = it.index;
207 findNextContainerMember(it);
208 switch (it.peek().?.id) {
209 .Eof, .RBrace => break,
210 else => {
211 // add error and continue
212 try tree.errors.push(.{
213 .ExpectedToken = .{ .token = index, .expected_id = .Comma },
214 });
215 continue;
216 }
217 }
218 };
202 if (try parseAppendedDocComment(arena, it, tree, comma)) |appended_comment|219 if (try parseAppendedDocComment(arena, it, tree, comma)) |appended_comment|
203 field.doc_comments = appended_comment;220 field.doc_comments = appended_comment;
204 continue;221 continue;
...@@ -210,22 +227,63 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All...@@ -210,22 +227,63 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
210 .UnattachedDocComment = .{ .token = doc_comments.?.firstToken() },227 .UnattachedDocComment = .{ .token = doc_comments.?.firstToken() },
211 });228 });
212 }229 }
213 break;230
231 switch (it.peek().?.id) {
232 .Eof, .RBrace => break,
233 else => {
234 // this was likely not supposed to end yet,
235 // try to find the next declaration
236 const index = it.index;
237 findNextContainerMember(it);
238 try tree.errors.push(.{
239 .ExpectedContainerMembers = .{ .token = index },
240 });
241 },
242 }
214 }243 }
215244
216 return list;245 return list;
217}246}
218247
219/// Attempts to find a closing brace.248fn findNextContainerMember(it: *TokenIterator) void {
220fn findEndOfBlock(it: *TokenIterator) void {249 var level: u32 = 0;
221 var count: u32 = 0;
222 while (true) {250 while (true) {
223 const tok = nextToken(it);251 const tok = nextToken(it);
224 switch (tok.ptr.id) {252 switch (tok.ptr.id) {
225 .LBrace => count += 1,253 // any of these can start a new top level declaration
226 .RBrace => {254 .Keyword_test,
227 if (count <= 1) return;255 .Keyword_comptime,
228 count -= 1;256 .Keyword_pub,
257 .Keyword_export,
258 .Keyword_extern,
259 .Keyword_inline,
260 .Keyword_noinline,
261 .Keyword_usingnamespace,
262 .Keyword_threadlocal,
263 .Keyword_const,
264 .Keyword_var,
265 .Keyword_fn,
266 .Identifier,
267 => {
268 if (level == 0) {
269 putBackToken(it, tok.index);
270 return;
271 }
272 },
273 .Comma, .Semicolon => {
274 // this decl was likely meant to end here
275 if (level == 0) {
276 return;
277 }
278 },
279 .LParen, .LBracket, .LBrace => level += 1,
280 .RParen, .RBracket, .RBrace => {
281 if (level == 0) {
282 // end of container, exit
283 putBackToken(it, tok.index);
284 return;
285 }
286 level -= 1;
229 },287 },
230 .Eof => {288 .Eof => {
231 putBackToken(it, tok.index);289 putBackToken(it, tok.index);
...@@ -338,9 +396,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?...@@ -338,9 +396,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?
338 if (parseFnProto(arena, it, tree) catch |err| switch (err) {396 if (parseFnProto(arena, it, tree) catch |err| switch (err) {
339 error.OutOfMemory => return error.OutOfMemory,397 error.OutOfMemory => return error.OutOfMemory,
340 error.ParseError => {398 error.ParseError => {
341 // this fn will likely have a body so we399 findNextContainerMember(it);
342 // use findEndOfBlock instead of findToken.
343 findEndOfBlock(it);
344 return error.ParseError;400 return error.ParseError;
345 },401 },
346 }) |node| {402 }) |node| {
...@@ -381,7 +437,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?...@@ -381,7 +437,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?
381 error.OutOfMemory => return error.OutOfMemory,437 error.OutOfMemory => return error.OutOfMemory,
382 error.ParseError => {438 error.ParseError => {
383 // try to skip to next decl439 // try to skip to next decl
384 findToken(it, .Semicolon);440 findNextContainerMember(it);
385 return error.ParseError;441 return error.ParseError;
386 },442 },
387 }) |node| {443 }) |node| {
...@@ -413,7 +469,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?...@@ -413,7 +469,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?
413 error.OutOfMemory => return error.OutOfMemory,469 error.OutOfMemory => return error.OutOfMemory,
414 error.ParseError => {470 error.ParseError => {
415 // try to skip to next decl471 // try to skip to next decl
416 findToken(it, .Semicolon);472 findNextContainerMember(it);
417 return error.ParseError;473 return error.ParseError;
418 },474 },
419 };475 };
...@@ -3215,6 +3271,8 @@ fn expectToken(it: *TokenIterator, tree: *Tree, id: Token.Id) Error!TokenIndex {...@@ -3215,6 +3271,8 @@ fn expectToken(it: *TokenIterator, tree: *Tree, id: Token.Id) Error!TokenIndex {
3215 try tree.errors.push(.{3271 try tree.errors.push(.{
3216 .ExpectedToken = .{ .token = token.index, .expected_id = id },3272 .ExpectedToken = .{ .token = token.index, .expected_id = id },
3217 });3273 });
3274 // go back so that we can recover properly
3275 putBackToken(it, token.index);
3218 return error.ParseError;3276 return error.ParseError;
3219 }3277 }
3220 return token.index;3278 return token.index;
lib/std/zig/parser_test.zig+21
...@@ -119,6 +119,25 @@ test "recovery: missing semicolon" {...@@ -119,6 +119,25 @@ test "recovery: missing semicolon" {
119 });119 });
120}120}
121121
122test "recovery: invalid container members" {
123 try testError(
124 \\usingnamespace;
125 \\foo+
126 \\bar@,
127 \\while (a == 2) { test "" {}}
128 \\test "" {
129 \\ a && b
130 \\}
131 , &[_]Error{
132 .ExpectedExpr,
133 .ExpectedToken,
134 .ExpectedToken,
135 .ExpectedContainerMembers,
136 .InvalidAnd,
137 .ExpectedToken,
138 });
139}
140
122test "zig fmt: top-level fields" {141test "zig fmt: top-level fields" {
123 try testCanonical(142 try testCanonical(
124 \\a: did_you_know,143 \\a: did_you_know,
...@@ -2953,6 +2972,8 @@ test "zig fmt: extern without container keyword returns error" {...@@ -2953,6 +2972,8 @@ test "zig fmt: extern without container keyword returns error" {
2953 \\2972 \\
2954 , &[_]Error{2973 , &[_]Error{
2955 .ExpectedExpr,2974 .ExpectedExpr,
2975 .ExpectedVarDeclOrFn,
2976 .ExpectedContainerMembers,
2956 });2977 });
2957}2978}
29582979