authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-14 12:09:40+03:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-14 12:09:40+03:00
loga32e240540895af536fdeb98db448ee745def59a
tree1472bddfdee4110ec8c01d2462d825209697e2e4
parent89f2923a8aad5119b1506c3b8e08464e132c228e
signaturelock-open Commit is signed but in an unrecognized format.

improve recovery from invalid container members

Instead of trying to find the end of the block or the next comma/semicolon we no try to find the next token that can start a container member.

2 files changed, 99 insertions(+), 20 deletions(-)

lib/std/zig/parse.zig+78-20
......@@ -59,10 +59,13 @@ fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error
5959 node.* = .{
6060 .decls = try parseContainerMembers(arena, it, tree),
6161 .eof_token = eatToken(it, .Eof) orelse blk: {
62 // parseContainerMembers will try to skip as much
63 // invalid tokens as it can so this can only be a '}'
64 const tok = eatToken(it, .RBrace).?;
6265 try tree.errors.push(.{
63 .ExpectedContainerMembers = .{ .token = it.index },
66 .ExpectedContainerMembers = .{ .token = tok },
6467 });
65 break :blk undefined;
68 break :blk tok;
6669 },
6770 };
6871 return node;
......@@ -101,7 +104,7 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
101104 if (parseTestDecl(arena, it, tree) catch |err| switch (err) {
102105 error.OutOfMemory => return error.OutOfMemory,
103106 error.ParseError => {
104 findEndOfBlock(it);
107 findNextContainerMember(it);
105108 continue;
106109 },
107110 }) |node| {
......@@ -116,7 +119,7 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
116119 if (parseTopLevelComptime(arena, it, tree) catch |err| switch (err) {
117120 error.OutOfMemory => return error.OutOfMemory,
118121 error.ParseError => {
119 findEndOfBlock(it);
122 findNextContainerMember(it);
120123 continue;
121124 },
122125 }) |node| {
......@@ -178,8 +181,8 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
178181 if (parseContainerField(arena, it, tree) catch |err| switch (err) {
179182 error.OutOfMemory => return error.OutOfMemory,
180183 error.ParseError => {
181 // attempt to recover by finding a comma
182 findToken(it, .Comma);
184 // attempt to recover
185 findNextContainerMember(it);
183186 continue;
184187 },
185188 }) |node| {
......@@ -198,7 +201,21 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
198201 const field = node.cast(Node.ContainerField).?;
199202 field.doc_comments = doc_comments;
200203 try list.push(node);
201 const comma = eatToken(it, .Comma) orelse break;
204 const comma = eatToken(it, .Comma) orelse {
205 // try to continue parsing
206 const index = it.index;
207 findNextContainerMember(it);
208 switch (it.peek().?.id) {
209 .Eof, .RBrace => break,
210 else => {
211 // add error and continue
212 try tree.errors.push(.{
213 .ExpectedToken = .{ .token = index, .expected_id = .Comma },
214 });
215 continue;
216 }
217 }
218 };
202219 if (try parseAppendedDocComment(arena, it, tree, comma)) |appended_comment|
203220 field.doc_comments = appended_comment;
204221 continue;
......@@ -210,22 +227,63 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
210227 .UnattachedDocComment = .{ .token = doc_comments.?.firstToken() },
211228 });
212229 }
213 break;
230
231 switch (it.peek().?.id) {
232 .Eof, .RBrace => break,
233 else => {
234 // this was likely not supposed to end yet,
235 // try to find the next declaration
236 const index = it.index;
237 findNextContainerMember(it);
238 try tree.errors.push(.{
239 .ExpectedContainerMembers = .{ .token = index },
240 });
241 },
242 }
214243 }
215244
216245 return list;
217246}
218247
219/// Attempts to find a closing brace.
220fn findEndOfBlock(it: *TokenIterator) void {
221 var count: u32 = 0;
248fn findNextContainerMember(it: *TokenIterator) void {
249 var level: u32 = 0;
222250 while (true) {
223251 const tok = nextToken(it);
224252 switch (tok.ptr.id) {
225 .LBrace => count += 1,
226 .RBrace => {
227 if (count <= 1) return;
228 count -= 1;
253 // any of these can start a new top level declaration
254 .Keyword_test,
255 .Keyword_comptime,
256 .Keyword_pub,
257 .Keyword_export,
258 .Keyword_extern,
259 .Keyword_inline,
260 .Keyword_noinline,
261 .Keyword_usingnamespace,
262 .Keyword_threadlocal,
263 .Keyword_const,
264 .Keyword_var,
265 .Keyword_fn,
266 .Identifier,
267 => {
268 if (level == 0) {
269 putBackToken(it, tok.index);
270 return;
271 }
272 },
273 .Comma, .Semicolon => {
274 // this decl was likely meant to end here
275 if (level == 0) {
276 return;
277 }
278 },
279 .LParen, .LBracket, .LBrace => level += 1,
280 .RParen, .RBracket, .RBrace => {
281 if (level == 0) {
282 // end of container, exit
283 putBackToken(it, tok.index);
284 return;
285 }
286 level -= 1;
229287 },
230288 .Eof => {
231289 putBackToken(it, tok.index);
......@@ -338,9 +396,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?
338396 if (parseFnProto(arena, it, tree) catch |err| switch (err) {
339397 error.OutOfMemory => return error.OutOfMemory,
340398 error.ParseError => {
341 // this fn will likely have a body so we
342 // use findEndOfBlock instead of findToken.
343 findEndOfBlock(it);
399 findNextContainerMember(it);
344400 return error.ParseError;
345401 },
346402 }) |node| {
......@@ -381,7 +437,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?
381437 error.OutOfMemory => return error.OutOfMemory,
382438 error.ParseError => {
383439 // try to skip to next decl
384 findToken(it, .Semicolon);
440 findNextContainerMember(it);
385441 return error.ParseError;
386442 },
387443 }) |node| {
......@@ -413,7 +469,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?
413469 error.OutOfMemory => return error.OutOfMemory,
414470 error.ParseError => {
415471 // try to skip to next decl
416 findToken(it, .Semicolon);
472 findNextContainerMember(it);
417473 return error.ParseError;
418474 },
419475 };
......@@ -3215,6 +3271,8 @@ fn expectToken(it: *TokenIterator, tree: *Tree, id: Token.Id) Error!TokenIndex {
32153271 try tree.errors.push(.{
32163272 .ExpectedToken = .{ .token = token.index, .expected_id = id },
32173273 });
3274 // go back so that we can recover properly
3275 putBackToken(it, token.index);
32183276 return error.ParseError;
32193277 }
32203278 return token.index;
lib/std/zig/parser_test.zig+21
......@@ -119,6 +119,25 @@ test "recovery: missing semicolon" {
119119 });
120120}
121121
122test "recovery: invalid container members" {
123 try testError(
124 \\usingnamespace;
125 \\foo+
126 \\bar@,
127 \\while (a == 2) { test "" {}}
128 \\test "" {
129 \\ a && b
130 \\}
131 , &[_]Error{
132 .ExpectedExpr,
133 .ExpectedToken,
134 .ExpectedToken,
135 .ExpectedContainerMembers,
136 .InvalidAnd,
137 .ExpectedToken,
138 });
139}
140
122141test "zig fmt: top-level fields" {
123142 try testCanonical(
124143 \\a: did_you_know,
......@@ -2953,6 +2972,8 @@ test "zig fmt: extern without container keyword returns error" {
29532972 \\
29542973 , &[_]Error{
29552974 .ExpectedExpr,
2975 .ExpectedVarDeclOrFn,
2976 .ExpectedContainerMembers,
29562977 });
29572978}
29582979