authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-12 22:37:39+03:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-12 22:37:39+03:00
logdf22c7dfef7312474865c868a633c8e9bbaa63fa
treee789b77cd478b3ce45cc2b80ef38d0f03a67206c
parentfa57463bb9c09ee1e50012d7e50d120e1599fb81
signature Commit is signed but in an unrecognized format.

std.zig attempt to continue parsing on error

2 files changed, 86 insertions(+), 23 deletions(-)

lib/std/zig/parse.zig+76-23
...@@ -48,31 +48,21 @@ pub fn parse(allocator: *Allocator, source: []const u8) Allocator.Error!*Tree {...@@ -48,31 +48,21 @@ pub fn parse(allocator: *Allocator, source: []const u8) Allocator.Error!*Tree {
4848
49 while (it.peek().?.id == .LineComment) _ = it.next();49 while (it.peek().?.id == .LineComment) _ = it.next();
5050
51 tree.root_node = parseRoot(arena, &it, tree) catch |err| blk: {51 tree.root_node = try parseRoot(arena, &it, tree);
52 switch (err) {
53 error.ParseError => {
54 assert(tree.errors.len != 0);
55 break :blk undefined;
56 },
57 error.OutOfMemory => {
58 return error.OutOfMemory;
59 },
60 }
61 };
6252
63 return tree;53 return tree;
64}54}
6555
66/// Root <- skip ContainerMembers eof56/// Root <- skip ContainerMembers eof
67fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!*Node.Root {57fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error!*Node.Root {
68 const node = try arena.create(Node.Root);58 const node = try arena.create(Node.Root);
69 node.* = .{59 node.* = .{
70 .decls = try parseContainerMembers(arena, it, tree),60 .decls = try parseContainerMembers(arena, it, tree),
71 .eof_token = eatToken(it, .Eof) orelse {61 .eof_token = eatToken(it, .Eof) orelse blk: {
72 try tree.errors.push(.{62 try tree.errors.push(.{
73 .ExpectedContainerMembers = .{ .token = it.index },63 .ExpectedContainerMembers = .{ .token = it.index },
74 });64 });
75 return error.ParseError;65 break :blk undefined;
76 },66 },
77 };67 };
78 return node;68 return node;
...@@ -85,7 +75,7 @@ fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!*Node.Roo...@@ -85,7 +75,7 @@ fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!*Node.Roo
85/// / KEYWORD_pub? ContainerField COMMA ContainerMembers75/// / KEYWORD_pub? ContainerField COMMA ContainerMembers
86/// / KEYWORD_pub? ContainerField76/// / KEYWORD_pub? ContainerField
87/// /77/// /
88fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !Node.Root.DeclList {78fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error!Node.Root.DeclList {
89 var list = Node.Root.DeclList.init(arena);79 var list = Node.Root.DeclList.init(arena);
9080
91 var field_state: union(enum) {81 var field_state: union(enum) {
...@@ -108,7 +98,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No...@@ -108,7 +98,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
10898
109 const doc_comments = try parseDocComment(arena, it, tree);99 const doc_comments = try parseDocComment(arena, it, tree);
110100
111 if (try parseTestDecl(arena, it, tree)) |node| {101 if (parseTestDecl(arena, it, tree) catch |err| switch (err) {
102 error.OutOfMemory => return error.OutOfMemory,
103 error.ParseError => {
104 findEndOfBlock(it);
105 continue;
106 },
107 }) |node| {
112 if (field_state == .seen) {108 if (field_state == .seen) {
113 field_state = .{ .end = node.firstToken() };109 field_state = .{ .end = node.firstToken() };
114 }110 }
...@@ -117,7 +113,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No...@@ -117,7 +113,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
117 continue;113 continue;
118 }114 }
119115
120 if (try parseTopLevelComptime(arena, it, tree)) |node| {116 if (parseTopLevelComptime(arena, it, tree) catch |err| switch (err) {
117 error.OutOfMemory => return error.OutOfMemory,
118 error.ParseError => {
119 findEndOfBlock(it);
120 continue;
121 },
122 }) |node| {
121 if (field_state == .seen) {123 if (field_state == .seen) {
122 field_state = .{ .end = node.firstToken() };124 field_state = .{ .end = node.firstToken() };
123 }125 }
...@@ -128,7 +130,15 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No...@@ -128,7 +130,15 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
128130
129 const visib_token = eatToken(it, .Keyword_pub);131 const visib_token = eatToken(it, .Keyword_pub);
130132
131 if (try parseTopLevelDecl(arena, it, tree)) |node| {133 if (parseTopLevelDecl(arena, it, tree) catch |err| switch (err) {
134 error.OutOfMemory => return error.OutOfMemory,
135 error.ParseError => {
136 // attempt to recover by finding a semicolon
137 // TODO if this was a function with a body we should use findEndOfBlock
138 findToken(it, .Semicolon);
139 continue;
140 },
141 }) |node| {
132 if (field_state == .seen) {142 if (field_state == .seen) {
133 field_state = .{ .end = visib_token orelse node.firstToken() };143 field_state = .{ .end = visib_token orelse node.firstToken() };
134 }144 }
...@@ -163,10 +173,17 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No...@@ -163,10 +173,17 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
163 try tree.errors.push(.{173 try tree.errors.push(.{
164 .ExpectedPubItem = .{ .token = it.index },174 .ExpectedPubItem = .{ .token = it.index },
165 });175 });
166 return error.ParseError;176 // ignore this pub
167 }177 }
168178
169 if (try parseContainerField(arena, it, tree)) |node| {179 if (parseContainerField(arena, it, tree) catch |err| switch (err) {
180 error.OutOfMemory => return error.OutOfMemory,
181 error.ParseError => {
182 // attempt to recover by finding a comma
183 findToken(it, .Comma);
184 continue;
185 },
186 }) |node| {
170 switch (field_state) {187 switch (field_state) {
171 .none => field_state = .seen,188 .none => field_state = .seen,
172 .err, .seen => {},189 .err, .seen => {},
...@@ -200,8 +217,39 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No...@@ -200,8 +217,39 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
200 return list;217 return list;
201}218}
202219
220/// Attempts to find a closing brace, assumes the opening brace was found.
221fn findEndOfBlock(it: *TokenIterator) void {
222 var count: u32 = 1;
223 while (it.next()) |tok| switch (tok.id) {
224 .LBrace => count += 1,
225 .RBrace => {
226 count -= 1;
227 if (count == 0) return;
228 },
229 else => {},
230 };
231}
232
233/// Attempts to find `wanted` token, keeps track of parentheses.
234fn findToken(it: *TokenIterator, wanted: Token.Id) void {
235 var count: u32 = 0;
236 while (it.next()) |tok| switch (tok.id) {
237 .LParen, .LBracket, .LBrace => count += 1,
238 .RParen, .RBracket, .RBrace => {
239 if (count == 0) {
240 _ = it.prev();
241 return;
242 }
243 count -= 1;
244 },
245 else => {
246 if (tok.id == wanted and count == 0) return;
247 },
248 };
249}
250
203/// Eat a multiline container doc comment251/// Eat a multiline container doc comment
204fn parseContainerDocComments(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {252fn parseContainerDocComments(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error!?*Node {
205 var lines = Node.DocComment.LineList.init(arena);253 var lines = Node.DocComment.LineList.init(arena);
206 while (eatToken(it, .ContainerDocComment)) |line| {254 while (eatToken(it, .ContainerDocComment)) |line| {
207 try lines.push(line);255 try lines.push(line);
...@@ -687,8 +735,13 @@ fn parseLoopStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Nod...@@ -687,8 +735,13 @@ fn parseLoopStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Nod
687 node.cast(Node.While).?.inline_token = inline_token;735 node.cast(Node.While).?.inline_token = inline_token;
688 return node;736 return node;
689 }737 }
738 if (inline_token == null) return null;
690739
691 return null;740 // If we've seen "inline", there should have been a "for" or "while"
741 try tree.errors.push(.{
742 .ExpectedInlinable = .{ .token = it.index },
743 });
744 return error.ParseError;
692}745}
693746
694/// ForStatement747/// ForStatement
...@@ -2925,7 +2978,7 @@ fn parseDocComment(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node.D...@@ -2925,7 +2978,7 @@ fn parseDocComment(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node.D
2925}2978}
29262979
2927/// Eat a single-line doc comment on the same line as another node2980/// Eat a single-line doc comment on the same line as another node
2928fn parseAppendedDocComment(arena: *Allocator, it: *TokenIterator, tree: *Tree, after_token: TokenIndex) !?*Node.DocComment {2981fn parseAppendedDocComment(arena: *Allocator, it: *TokenIterator, tree: *Tree, after_token: TokenIndex) Allocator.Error!?*Node.DocComment {
2929 const comment_token = eatToken(it, .DocComment) orelse return null;2982 const comment_token = eatToken(it, .DocComment) orelse return null;
2930 if (tree.tokensOnSameLine(after_token, comment_token)) {2983 if (tree.tokensOnSameLine(after_token, comment_token)) {
2931 const node = try arena.create(Node.DocComment);2984 const node = try arena.create(Node.DocComment);
lib/std/zig/parser_test.zig+10
...@@ -1,3 +1,13 @@...@@ -1,3 +1,13 @@
1test "zig fmt: fault tolerant parsing" {
2 try testError(
3 \\test "" {inline}
4 \\test "" {inline}
5 , &[_]Error{
6 .ExpectedInlinable,
7 .ExpectedInlinable,
8 });
9}
10
1test "zig fmt: top-level fields" {11test "zig fmt: top-level fields" {
2 try testCanonical(12 try testCanonical(
3 \\a: did_you_know,13 \\a: did_you_know,