authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-02-21 17:37:10-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-02-21 17:37:10-07:00
log866f7dc7d68156d2cb1f3a7edad0882c67943726
treead86dfa4886147483297357e7ef01ac77c983aa1
parent15603f403c9ca91f7530798a5a7751cace284a28

parser: support more recovery test cases

3 files changed, 141 insertions(+), 116 deletions(-)

lib/std/zig/ast.zig+8
...@@ -205,6 +205,9 @@ pub const Tree = struct {...@@ -205,6 +205,9 @@ pub const Tree = struct {
205 token_tags[parse_error.token].symbol(),205 token_tags[parse_error.token].symbol(),
206 });206 });
207 },207 },
208 .expected_pub_item => {
209 return stream.writeAll("expected function or variable declaration after pub");
210 },
208 .expected_return_type => {211 .expected_return_type => {
209 return stream.print("expected return type expression, found '{s}'", .{212 return stream.print("expected return type expression, found '{s}'", .{
210 token_tags[parse_error.token].symbol(),213 token_tags[parse_error.token].symbol(),
...@@ -265,6 +268,9 @@ pub const Tree = struct {...@@ -265,6 +268,9 @@ pub const Tree = struct {
265 .invalid_align => {268 .invalid_align => {
266 return stream.writeAll("alignment not allowed on arrays");269 return stream.writeAll("alignment not allowed on arrays");
267 },270 },
271 .invalid_and => {
272 return stream.writeAll("`&&` is invalid; note that `and` is boolean AND");
273 },
268 .invalid_bit_range => {274 .invalid_bit_range => {
269 return stream.writeAll("bit range not allowed on slices and arrays");275 return stream.writeAll("bit range not allowed on slices and arrays");
270 },276 },
...@@ -2316,6 +2322,7 @@ pub const Error = struct {...@@ -2316,6 +2322,7 @@ pub const Error = struct {
2316 expected_param_list,2322 expected_param_list,
2317 expected_prefix_expr,2323 expected_prefix_expr,
2318 expected_primary_type_expr,2324 expected_primary_type_expr,
2325 expected_pub_item,
2319 expected_return_type,2326 expected_return_type,
2320 expected_semi_or_else,2327 expected_semi_or_else,
2321 expected_semi_or_lbrace,2328 expected_semi_or_lbrace,
...@@ -2330,6 +2337,7 @@ pub const Error = struct {...@@ -2330,6 +2337,7 @@ pub const Error = struct {
2330 extra_const_qualifier,2337 extra_const_qualifier,
2331 extra_volatile_qualifier,2338 extra_volatile_qualifier,
2332 invalid_align,2339 invalid_align,
2340 invalid_and,
2333 invalid_bit_range,2341 invalid_bit_range,
2334 invalid_token,2342 invalid_token,
2335 same_line_doc_comment,2343 same_line_doc_comment,
lib/std/zig/parse.zig+31-13
...@@ -423,7 +423,7 @@ const Parser = struct {...@@ -423,7 +423,7 @@ const Parser = struct {
423 while (true) {423 while (true) {
424 const tok = p.nextToken();424 const tok = p.nextToken();
425 switch (p.token_tags[tok]) {425 switch (p.token_tags[tok]) {
426 // any of these can start a new top level declaration426 // Any of these can start a new top level declaration.
427 .keyword_test,427 .keyword_test,
428 .keyword_comptime,428 .keyword_comptime,
429 .keyword_pub,429 .keyword_pub,
...@@ -436,13 +436,18 @@ const Parser = struct {...@@ -436,13 +436,18 @@ const Parser = struct {
436 .keyword_const,436 .keyword_const,
437 .keyword_var,437 .keyword_var,
438 .keyword_fn,438 .keyword_fn,
439 .identifier,
440 => {439 => {
441 if (level == 0) {440 if (level == 0) {
442 p.tok_i -= 1;441 p.tok_i -= 1;
443 return;442 return;
444 }443 }
445 },444 },
445 .identifier => {
446 if (p.token_tags[tok + 1] == .comma and level == 0) {
447 p.tok_i -= 1;
448 return;
449 }
450 },
446 .comma, .semicolon => {451 .comma, .semicolon => {
447 // this decl was likely meant to end here452 // this decl was likely meant to end here
448 if (level == 0) {453 if (level == 0) {
...@@ -531,10 +536,13 @@ const Parser = struct {...@@ -531,10 +536,13 @@ const Parser = struct {
531 fn expectTopLevelDecl(p: *Parser) !Node.Index {536 fn expectTopLevelDecl(p: *Parser) !Node.Index {
532 const extern_export_inline_token = p.nextToken();537 const extern_export_inline_token = p.nextToken();
533 var expect_fn: bool = false;538 var expect_fn: bool = false;
534 var exported: bool = false;539 var expect_var_or_fn: bool = false;
535 switch (p.token_tags[extern_export_inline_token]) {540 switch (p.token_tags[extern_export_inline_token]) {
536 .keyword_extern => _ = p.eatToken(.string_literal),541 .keyword_extern => {
537 .keyword_export => exported = true,542 _ = p.eatToken(.string_literal);
543 expect_var_or_fn = true;
544 },
545 .keyword_export => expect_var_or_fn = true,
538 .keyword_inline, .keyword_noinline => expect_fn = true,546 .keyword_inline, .keyword_noinline => expect_fn = true,
539 else => p.tok_i -= 1,547 else => p.tok_i -= 1,
540 }548 }
...@@ -580,11 +588,12 @@ const Parser = struct {...@@ -580,11 +588,12 @@ const Parser = struct {
580 if (thread_local_token != null) {588 if (thread_local_token != null) {
581 return p.fail(.expected_var_decl);589 return p.fail(.expected_var_decl);
582 }590 }
583591 if (expect_var_or_fn) {
584 if (exported) {
585 return p.fail(.expected_var_decl_or_fn);592 return p.fail(.expected_var_decl_or_fn);
586 }593 }
587594 if (p.token_tags[p.tok_i] != .keyword_usingnamespace) {
595 return p.fail(.expected_pub_item);
596 }
588 return p.expectUsingNamespace();597 return p.expectUsingNamespace();
589 }598 }
590599
...@@ -599,7 +608,7 @@ const Parser = struct {...@@ -599,7 +608,7 @@ const Parser = struct {
599 }608 }
600609
601 fn expectUsingNamespace(p: *Parser) !Node.Index {610 fn expectUsingNamespace(p: *Parser) !Node.Index {
602 const usingnamespace_token = try p.expectToken(.keyword_usingnamespace);611 const usingnamespace_token = p.assertToken(.keyword_usingnamespace);
603 const expr = try p.expectExpr();612 const expr = try p.expectExpr();
604 const semicolon_token = try p.expectToken(.semicolon);613 const semicolon_token = try p.expectToken(.semicolon);
605 return p.addNode(.{614 return p.addNode(.{
...@@ -1346,6 +1355,11 @@ const Parser = struct {...@@ -1346,6 +1355,11 @@ const Parser = struct {
1346 },1355 },
1347 });1356 });
1348 },1357 },
1358 .invalid_ampersands => {
1359 try p.warn(.invalid_and);
1360 p.tok_i += 1;
1361 return p.parseCompareExpr();
1362 },
1349 else => return res,1363 else => return res,
1350 }1364 }
1351 }1365 }
...@@ -2283,10 +2297,12 @@ const Parser = struct {...@@ -2283,10 +2297,12 @@ const Parser = struct {
2283 if (node == 0) break;2297 if (node == 0) break;
2284 res = node;2298 res = node;
2285 }2299 }
2286 const lparen = (try p.expectTokenRecoverable(.l_paren)) orelse {2300 const lparen = p.nextToken();
2301 if (p.token_tags[lparen] != .l_paren) {
2302 p.tok_i -= 1;
2287 try p.warn(.expected_param_list);2303 try p.warn(.expected_param_list);
2288 return res;2304 return res;
2289 };2305 }
2290 if (p.eatToken(.r_paren)) |_| {2306 if (p.eatToken(.r_paren)) |_| {
2291 return p.addNode(.{2307 return p.addNode(.{
2292 .tag = .async_call_one,2308 .tag = .async_call_one,
...@@ -3769,7 +3785,8 @@ const Parser = struct {...@@ -3769,7 +3785,8 @@ const Parser = struct {
3769 /// ExprList <- (Expr COMMA)* Expr?3785 /// ExprList <- (Expr COMMA)* Expr?
3770 fn parseBuiltinCall(p: *Parser) !Node.Index {3786 fn parseBuiltinCall(p: *Parser) !Node.Index {
3771 const builtin_token = p.assertToken(.builtin);3787 const builtin_token = p.assertToken(.builtin);
3772 _ = (try p.expectTokenRecoverable(.l_paren)) orelse {3788 if (p.token_tags[p.nextToken()] != .l_paren) {
3789 p.tok_i -= 1;
3773 try p.warn(.expected_param_list);3790 try p.warn(.expected_param_list);
3774 // Pretend this was an identifier so we can continue parsing.3791 // Pretend this was an identifier so we can continue parsing.
3775 return p.addNode(.{3792 return p.addNode(.{
...@@ -3780,7 +3797,7 @@ const Parser = struct {...@@ -3780,7 +3797,7 @@ const Parser = struct {
3780 .rhs = undefined,3797 .rhs = undefined,
3781 },3798 },
3782 });3799 });
3783 };3800 }
3784 if (p.eatToken(.r_paren)) |_| {3801 if (p.eatToken(.r_paren)) |_| {
3785 return p.addNode(.{3802 return p.addNode(.{
3786 .tag = .builtin_call_two,3803 .tag = .builtin_call_two,
...@@ -4015,6 +4032,7 @@ const Parser = struct {...@@ -4015,6 +4032,7 @@ const Parser = struct {
4015 fn expectToken(p: *Parser, tag: Token.Tag) Error!TokenIndex {4032 fn expectToken(p: *Parser, tag: Token.Tag) Error!TokenIndex {
4016 const token = p.nextToken();4033 const token = p.nextToken();
4017 if (p.token_tags[token] != tag) {4034 if (p.token_tags[token] != tag) {
4035 p.tok_i -= 1; // Go back so that we can recover properly.
4018 return p.failMsg(.{4036 return p.failMsg(.{
4019 .tag = .expected_token,4037 .tag = .expected_token,
4020 .token = token,4038 .token = token,
lib/std/zig/parser_test.zig+102-103
...@@ -3579,7 +3579,7 @@ test "zig fmt: file ends with struct field" {...@@ -3579,7 +3579,7 @@ test "zig fmt: file ends with struct field" {
3579// \\3579// \\
3580// , &[_]Error{3580// , &[_]Error{
3581// .expected_expr,3581// .expected_expr,
3582// .ExpectedVarDeclOrFn,3582// .expected_var_decl_or_fn,
3583// });3583// });
3584//}3584//}
35853585
...@@ -4070,24 +4070,24 @@ test "recovery: block statements" {...@@ -4070,24 +4070,24 @@ test "recovery: block statements" {
4070 });4070 });
4071}4071}
40724072
4073//test "recovery: missing comma" {4073test "recovery: missing comma" {
4074// try testError(4074 try testError(
4075// \\test "" {4075 \\test "" {
4076// \\ switch (foo) {4076 \\ switch (foo) {
4077// \\ 2 => {}4077 \\ 2 => {}
4078// \\ 3 => {}4078 \\ 3 => {}
4079// \\ else => {4079 \\ else => {
4080// \\ foo && bar +;4080 \\ foo && bar +;
4081// \\ }4081 \\ }
4082// \\ }4082 \\ }
4083// \\}4083 \\}
4084// , &[_]Error{4084 , &[_]Error{
4085// .expected_token,4085 .expected_token,
4086// .expected_token,4086 .expected_token,
4087// .invalid_and,4087 .invalid_and,
4088// .invalid_token,4088 .invalid_token,
4089// });4089 });
4090//}4090}
40914091
4092test "recovery: extra qualifier" {4092test "recovery: extra qualifier" {
4093 try testError(4093 try testError(
...@@ -4099,94 +4099,93 @@ test "recovery: extra qualifier" {...@@ -4099,94 +4099,93 @@ test "recovery: extra qualifier" {
4099 });4099 });
4100}4100}
41014101
4102//test "recovery: missing return type" {4102test "recovery: missing return type" {
4103// try testError(4103 try testError(
4104// \\fn foo() {4104 \\fn foo() {
4105// \\ a && b;4105 \\ a && b;
4106// \\}4106 \\}
4107// \\test ""4107 \\test ""
4108// , &[_]Error{4108 , &[_]Error{
4109// .ExpectedReturnType,4109 .expected_return_type,
4110// .invalid_and,4110 .invalid_and,
4111// .expected_block,4111 .expected_block,
4112// });4112 });
4113//}4113}
41144114
4115//test "recovery: continue after invalid decl" {4115test "recovery: continue after invalid decl" {
4116// try testError(4116 try testError(
4117// \\fn foo {4117 \\fn foo {
4118// \\ inline;4118 \\ inline;
4119// \\}4119 \\}
4120// \\pub test "" {4120 \\pub test "" {
4121// \\ async a && b;4121 \\ async a && b;
4122// \\}4122 \\}
4123// , &[_]Error{4123 , &[_]Error{
4124// .expected_token,4124 .expected_token,
4125// .ExpectedPubItem,4125 .expected_pub_item,
4126// .ExpectedParamList,4126 .expected_param_list,
4127// .invalid_and,4127 .invalid_and,
4128// });4128 });
4129// try testError(4129 try testError(
4130// \\threadlocal test "" {4130 \\threadlocal test "" {
4131// \\ @a && b;4131 \\ @a && b;
4132// \\}4132 \\}
4133// , &[_]Error{4133 , &[_]Error{
4134// .ExpectedVarDecl,4134 .expected_var_decl,
4135// .ExpectedParamList,4135 .expected_param_list,
4136// .invalid_and,4136 .invalid_and,
4137// });4137 });
4138//}4138}
41394139
4140//test "recovery: invalid extern/inline" {4140test "recovery: invalid extern/inline" {
4141// try testError(4141 try testError(
4142// \\inline test "" { a && b; }4142 \\inline test "" { a && b; }
4143// , &[_]Error{4143 , &[_]Error{
4144// .ExpectedFn,4144 .expected_fn,
4145// .invalid_and,4145 .invalid_and,
4146// });4146 });
4147// try testError(4147 try testError(
4148// \\extern "" test "" { a && b; }4148 \\extern "" test "" { a && b; }
4149// , &[_]Error{4149 , &[_]Error{
4150// .ExpectedVarDeclOrFn,4150 .expected_var_decl_or_fn,
4151// .invalid_and,4151 .invalid_and,
4152// });4152 });
4153//}4153}
41544154
4155//test "recovery: missing semicolon" {4155test "recovery: missing semicolon" {
4156// try testError(4156 try testError(
4157// \\test "" {4157 \\test "" {
4158// \\ comptime a && b4158 \\ comptime a && b
4159// \\ c && d4159 \\ c && d
4160// \\ @foo4160 \\ @foo
4161// \\}4161 \\}
4162// , &[_]Error{4162 , &[_]Error{
4163// .invalid_and,4163 .invalid_and,
4164// .expected_token,4164 .expected_token,
4165// .invalid_and,4165 .invalid_and,
4166// .expected_token,4166 .expected_token,
4167// .ExpectedParamList,4167 .expected_param_list,
4168// .expected_token,4168 .expected_token,
4169// });4169 });
4170//}4170}
41714171
4172//test "recovery: invalid container members" {4172test "recovery: invalid container members" {
4173// try testError(4173 try testError(
4174// \\usingnamespace;4174 \\usingnamespace;
4175// \\foo+4175 \\foo+
4176// \\bar@,4176 \\bar@,
4177// \\while (a == 2) { test "" {}}4177 \\while (a == 2) { test "" {}}
4178// \\test "" {4178 \\test "" {
4179// \\ a && b4179 \\ a && b
4180// \\}4180 \\}
4181// , &[_]Error{4181 , &[_]Error{
4182// .expected_expr,4182 .expected_expr,
4183// .expected_token,4183 .expected_token,
4184// .expected_token,4184 .expected_container_members,
4185// .expected_container_members,4185 .invalid_and,
4186// .invalid_and,4186 .expected_token,
4187// .expected_token,4187 });
4188// });4188}
4189//}
41904189
4191//test "recovery: invalid parameter" {4190//test "recovery: invalid parameter" {
4192// try testError(4191// try testError(