authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-12 22:37:39+03:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-12 22:37:39+03:00
logdf22c7dfef7312474865c868a633c8e9bbaa63fa
treee789b77cd478b3ce45cc2b80ef38d0f03a67206c
parentfa57463bb9c09ee1e50012d7e50d120e1599fb81
signaturelock-open Commit is signed but in an unrecognized format.

std.zig attempt to continue parsing on error

2 files changed, 86 insertions(+), 23 deletions(-)

lib/std/zig/parse.zig+76-23
......@@ -48,31 +48,21 @@ pub fn parse(allocator: *Allocator, source: []const u8) Allocator.Error!*Tree {
4848
4949 while (it.peek().?.id == .LineComment) _ = it.next();
5050
51 tree.root_node = parseRoot(arena, &it, tree) catch |err| blk: {
52 switch (err) {
53 error.ParseError => {
54 assert(tree.errors.len != 0);
55 break :blk undefined;
56 },
57 error.OutOfMemory => {
58 return error.OutOfMemory;
59 },
60 }
61 };
51 tree.root_node = try parseRoot(arena, &it, tree);
6252
6353 return tree;
6454}
6555
6656/// Root <- skip ContainerMembers eof
67fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!*Node.Root {
57fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error!*Node.Root {
6858 const node = try arena.create(Node.Root);
6959 node.* = .{
7060 .decls = try parseContainerMembers(arena, it, tree),
71 .eof_token = eatToken(it, .Eof) orelse {
61 .eof_token = eatToken(it, .Eof) orelse blk: {
7262 try tree.errors.push(.{
7363 .ExpectedContainerMembers = .{ .token = it.index },
7464 });
75 return error.ParseError;
65 break :blk undefined;
7666 },
7767 };
7868 return node;
......@@ -85,7 +75,7 @@ fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!*Node.Roo
8575/// / KEYWORD_pub? ContainerField COMMA ContainerMembers
8676/// / KEYWORD_pub? ContainerField
8777/// /
88fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !Node.Root.DeclList {
78fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error!Node.Root.DeclList {
8979 var list = Node.Root.DeclList.init(arena);
9080
9181 var field_state: union(enum) {
......@@ -108,7 +98,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
10898
10999 const doc_comments = try parseDocComment(arena, it, tree);
110100
111 if (try parseTestDecl(arena, it, tree)) |node| {
101 if (parseTestDecl(arena, it, tree) catch |err| switch (err) {
102 error.OutOfMemory => return error.OutOfMemory,
103 error.ParseError => {
104 findEndOfBlock(it);
105 continue;
106 },
107 }) |node| {
112108 if (field_state == .seen) {
113109 field_state = .{ .end = node.firstToken() };
114110 }
......@@ -117,7 +113,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
117113 continue;
118114 }
119115
120 if (try parseTopLevelComptime(arena, it, tree)) |node| {
116 if (parseTopLevelComptime(arena, it, tree) catch |err| switch (err) {
117 error.OutOfMemory => return error.OutOfMemory,
118 error.ParseError => {
119 findEndOfBlock(it);
120 continue;
121 },
122 }) |node| {
121123 if (field_state == .seen) {
122124 field_state = .{ .end = node.firstToken() };
123125 }
......@@ -128,7 +130,15 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
128130
129131 const visib_token = eatToken(it, .Keyword_pub);
130132
131 if (try parseTopLevelDecl(arena, it, tree)) |node| {
133 if (parseTopLevelDecl(arena, it, tree) catch |err| switch (err) {
134 error.OutOfMemory => return error.OutOfMemory,
135 error.ParseError => {
136 // attempt to recover by finding a semicolon
137 // TODO if this was a function with a body we should use findEndOfBlock
138 findToken(it, .Semicolon);
139 continue;
140 },
141 }) |node| {
132142 if (field_state == .seen) {
133143 field_state = .{ .end = visib_token orelse node.firstToken() };
134144 }
......@@ -163,10 +173,17 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
163173 try tree.errors.push(.{
164174 .ExpectedPubItem = .{ .token = it.index },
165175 });
166 return error.ParseError;
176 // ignore this pub
167177 }
168178
169 if (try parseContainerField(arena, it, tree)) |node| {
179 if (parseContainerField(arena, it, tree) catch |err| switch (err) {
180 error.OutOfMemory => return error.OutOfMemory,
181 error.ParseError => {
182 // attempt to recover by finding a comma
183 findToken(it, .Comma);
184 continue;
185 },
186 }) |node| {
170187 switch (field_state) {
171188 .none => field_state = .seen,
172189 .err, .seen => {},
......@@ -200,8 +217,39 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
200217 return list;
201218}
202219
220/// Attempts to find a closing brace, assumes the opening brace was found.
221fn findEndOfBlock(it: *TokenIterator) void {
222 var count: u32 = 1;
223 while (it.next()) |tok| switch (tok.id) {
224 .LBrace => count += 1,
225 .RBrace => {
226 count -= 1;
227 if (count == 0) return;
228 },
229 else => {},
230 };
231}
232
233/// Attempts to find `wanted` token, keeps track of parentheses.
234fn findToken(it: *TokenIterator, wanted: Token.Id) void {
235 var count: u32 = 0;
236 while (it.next()) |tok| switch (tok.id) {
237 .LParen, .LBracket, .LBrace => count += 1,
238 .RParen, .RBracket, .RBrace => {
239 if (count == 0) {
240 _ = it.prev();
241 return;
242 }
243 count -= 1;
244 },
245 else => {
246 if (tok.id == wanted and count == 0) return;
247 },
248 };
249}
250
203251/// Eat a multiline container doc comment
204fn parseContainerDocComments(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
252fn parseContainerDocComments(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error!?*Node {
205253 var lines = Node.DocComment.LineList.init(arena);
206254 while (eatToken(it, .ContainerDocComment)) |line| {
207255 try lines.push(line);
......@@ -687,8 +735,13 @@ fn parseLoopStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Nod
687735 node.cast(Node.While).?.inline_token = inline_token;
688736 return node;
689737 }
738 if (inline_token == null) return null;
690739
691 return null;
740 // If we've seen "inline", there should have been a "for" or "while"
741 try tree.errors.push(.{
742 .ExpectedInlinable = .{ .token = it.index },
743 });
744 return error.ParseError;
692745}
693746
694747/// ForStatement
......@@ -2925,7 +2978,7 @@ fn parseDocComment(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node.D
29252978}
29262979
29272980/// Eat a single-line doc comment on the same line as another node
2928fn parseAppendedDocComment(arena: *Allocator, it: *TokenIterator, tree: *Tree, after_token: TokenIndex) !?*Node.DocComment {
2981fn parseAppendedDocComment(arena: *Allocator, it: *TokenIterator, tree: *Tree, after_token: TokenIndex) Allocator.Error!?*Node.DocComment {
29292982 const comment_token = eatToken(it, .DocComment) orelse return null;
29302983 if (tree.tokensOnSameLine(after_token, comment_token)) {
29312984 const node = try arena.create(Node.DocComment);
lib/std/zig/parser_test.zig+10
......@@ -1,3 +1,13 @@
1test "zig fmt: fault tolerant parsing" {
2 try testError(
3 \\test "" {inline}
4 \\test "" {inline}
5 , &[_]Error{
6 .ExpectedInlinable,
7 .ExpectedInlinable,
8 });
9}
10
111test "zig fmt: top-level fields" {
212 try testCanonical(
313 \\a: did_you_know,