authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-02-21 17:37:10-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-02-21 17:37:10-07:00
log866f7dc7d68156d2cb1f3a7edad0882c67943726
treead86dfa4886147483297357e7ef01ac77c983aa1
parent15603f403c9ca91f7530798a5a7751cace284a28

parser: support more recovery test cases

3 files changed, 141 insertions(+), 116 deletions(-)

lib/std/zig/ast.zig+8
......@@ -205,6 +205,9 @@ pub const Tree = struct {
205205 token_tags[parse_error.token].symbol(),
206206 });
207207 },
208 .expected_pub_item => {
209 return stream.writeAll("expected function or variable declaration after pub");
210 },
208211 .expected_return_type => {
209212 return stream.print("expected return type expression, found '{s}'", .{
210213 token_tags[parse_error.token].symbol(),
......@@ -265,6 +268,9 @@ pub const Tree = struct {
265268 .invalid_align => {
266269 return stream.writeAll("alignment not allowed on arrays");
267270 },
271 .invalid_and => {
272 return stream.writeAll("`&&` is invalid; note that `and` is boolean AND");
273 },
268274 .invalid_bit_range => {
269275 return stream.writeAll("bit range not allowed on slices and arrays");
270276 },
......@@ -2316,6 +2322,7 @@ pub const Error = struct {
23162322 expected_param_list,
23172323 expected_prefix_expr,
23182324 expected_primary_type_expr,
2325 expected_pub_item,
23192326 expected_return_type,
23202327 expected_semi_or_else,
23212328 expected_semi_or_lbrace,
......@@ -2330,6 +2337,7 @@ pub const Error = struct {
23302337 extra_const_qualifier,
23312338 extra_volatile_qualifier,
23322339 invalid_align,
2340 invalid_and,
23332341 invalid_bit_range,
23342342 invalid_token,
23352343 same_line_doc_comment,
lib/std/zig/parse.zig+31-13
......@@ -423,7 +423,7 @@ const Parser = struct {
423423 while (true) {
424424 const tok = p.nextToken();
425425 switch (p.token_tags[tok]) {
426 // any of these can start a new top level declaration
426 // Any of these can start a new top level declaration.
427427 .keyword_test,
428428 .keyword_comptime,
429429 .keyword_pub,
......@@ -436,13 +436,18 @@ const Parser = struct {
436436 .keyword_const,
437437 .keyword_var,
438438 .keyword_fn,
439 .identifier,
440439 => {
441440 if (level == 0) {
442441 p.tok_i -= 1;
443442 return;
444443 }
445444 },
445 .identifier => {
446 if (p.token_tags[tok + 1] == .comma and level == 0) {
447 p.tok_i -= 1;
448 return;
449 }
450 },
446451 .comma, .semicolon => {
447452 // this decl was likely meant to end here
448453 if (level == 0) {
......@@ -531,10 +536,13 @@ const Parser = struct {
531536 fn expectTopLevelDecl(p: *Parser) !Node.Index {
532537 const extern_export_inline_token = p.nextToken();
533538 var expect_fn: bool = false;
534 var exported: bool = false;
539 var expect_var_or_fn: bool = false;
535540 switch (p.token_tags[extern_export_inline_token]) {
536 .keyword_extern => _ = p.eatToken(.string_literal),
537 .keyword_export => exported = true,
541 .keyword_extern => {
542 _ = p.eatToken(.string_literal);
543 expect_var_or_fn = true;
544 },
545 .keyword_export => expect_var_or_fn = true,
538546 .keyword_inline, .keyword_noinline => expect_fn = true,
539547 else => p.tok_i -= 1,
540548 }
......@@ -580,11 +588,12 @@ const Parser = struct {
580588 if (thread_local_token != null) {
581589 return p.fail(.expected_var_decl);
582590 }
583
584 if (exported) {
591 if (expect_var_or_fn) {
585592 return p.fail(.expected_var_decl_or_fn);
586593 }
587
594 if (p.token_tags[p.tok_i] != .keyword_usingnamespace) {
595 return p.fail(.expected_pub_item);
596 }
588597 return p.expectUsingNamespace();
589598 }
590599
......@@ -599,7 +608,7 @@ const Parser = struct {
599608 }
600609
601610 fn expectUsingNamespace(p: *Parser) !Node.Index {
602 const usingnamespace_token = try p.expectToken(.keyword_usingnamespace);
611 const usingnamespace_token = p.assertToken(.keyword_usingnamespace);
603612 const expr = try p.expectExpr();
604613 const semicolon_token = try p.expectToken(.semicolon);
605614 return p.addNode(.{
......@@ -1346,6 +1355,11 @@ const Parser = struct {
13461355 },
13471356 });
13481357 },
1358 .invalid_ampersands => {
1359 try p.warn(.invalid_and);
1360 p.tok_i += 1;
1361 return p.parseCompareExpr();
1362 },
13491363 else => return res,
13501364 }
13511365 }
......@@ -2283,10 +2297,12 @@ const Parser = struct {
22832297 if (node == 0) break;
22842298 res = node;
22852299 }
2286 const lparen = (try p.expectTokenRecoverable(.l_paren)) orelse {
2300 const lparen = p.nextToken();
2301 if (p.token_tags[lparen] != .l_paren) {
2302 p.tok_i -= 1;
22872303 try p.warn(.expected_param_list);
22882304 return res;
2289 };
2305 }
22902306 if (p.eatToken(.r_paren)) |_| {
22912307 return p.addNode(.{
22922308 .tag = .async_call_one,
......@@ -3769,7 +3785,8 @@ const Parser = struct {
37693785 /// ExprList <- (Expr COMMA)* Expr?
37703786 fn parseBuiltinCall(p: *Parser) !Node.Index {
37713787 const builtin_token = p.assertToken(.builtin);
3772 _ = (try p.expectTokenRecoverable(.l_paren)) orelse {
3788 if (p.token_tags[p.nextToken()] != .l_paren) {
3789 p.tok_i -= 1;
37733790 try p.warn(.expected_param_list);
37743791 // Pretend this was an identifier so we can continue parsing.
37753792 return p.addNode(.{
......@@ -3780,7 +3797,7 @@ const Parser = struct {
37803797 .rhs = undefined,
37813798 },
37823799 });
3783 };
3800 }
37843801 if (p.eatToken(.r_paren)) |_| {
37853802 return p.addNode(.{
37863803 .tag = .builtin_call_two,
......@@ -4015,6 +4032,7 @@ const Parser = struct {
40154032 fn expectToken(p: *Parser, tag: Token.Tag) Error!TokenIndex {
40164033 const token = p.nextToken();
40174034 if (p.token_tags[token] != tag) {
4035 p.tok_i -= 1; // Go back so that we can recover properly.
40184036 return p.failMsg(.{
40194037 .tag = .expected_token,
40204038 .token = token,
lib/std/zig/parser_test.zig+102-103
......@@ -3579,7 +3579,7 @@ test "zig fmt: file ends with struct field" {
35793579// \\
35803580// , &[_]Error{
35813581// .expected_expr,
3582// .ExpectedVarDeclOrFn,
3582// .expected_var_decl_or_fn,
35833583// });
35843584//}
35853585
......@@ -4070,24 +4070,24 @@ test "recovery: block statements" {
40704070 });
40714071}
40724072
4073//test "recovery: missing comma" {
4074// try testError(
4075// \\test "" {
4076// \\ switch (foo) {
4077// \\ 2 => {}
4078// \\ 3 => {}
4079// \\ else => {
4080// \\ foo && bar +;
4081// \\ }
4082// \\ }
4083// \\}
4084// , &[_]Error{
4085// .expected_token,
4086// .expected_token,
4087// .invalid_and,
4088// .invalid_token,
4089// });
4090//}
4073test "recovery: missing comma" {
4074 try testError(
4075 \\test "" {
4076 \\ switch (foo) {
4077 \\ 2 => {}
4078 \\ 3 => {}
4079 \\ else => {
4080 \\ foo && bar +;
4081 \\ }
4082 \\ }
4083 \\}
4084 , &[_]Error{
4085 .expected_token,
4086 .expected_token,
4087 .invalid_and,
4088 .invalid_token,
4089 });
4090}
40914091
40924092test "recovery: extra qualifier" {
40934093 try testError(
......@@ -4099,94 +4099,93 @@ test "recovery: extra qualifier" {
40994099 });
41004100}
41014101
4102//test "recovery: missing return type" {
4103// try testError(
4104// \\fn foo() {
4105// \\ a && b;
4106// \\}
4107// \\test ""
4108// , &[_]Error{
4109// .ExpectedReturnType,
4110// .invalid_and,
4111// .expected_block,
4112// });
4113//}
4102test "recovery: missing return type" {
4103 try testError(
4104 \\fn foo() {
4105 \\ a && b;
4106 \\}
4107 \\test ""
4108 , &[_]Error{
4109 .expected_return_type,
4110 .invalid_and,
4111 .expected_block,
4112 });
4113}
41144114
4115//test "recovery: continue after invalid decl" {
4116// try testError(
4117// \\fn foo {
4118// \\ inline;
4119// \\}
4120// \\pub test "" {
4121// \\ async a && b;
4122// \\}
4123// , &[_]Error{
4124// .expected_token,
4125// .ExpectedPubItem,
4126// .ExpectedParamList,
4127// .invalid_and,
4128// });
4129// try testError(
4130// \\threadlocal test "" {
4131// \\ @a && b;
4132// \\}
4133// , &[_]Error{
4134// .ExpectedVarDecl,
4135// .ExpectedParamList,
4136// .invalid_and,
4137// });
4138//}
4115test "recovery: continue after invalid decl" {
4116 try testError(
4117 \\fn foo {
4118 \\ inline;
4119 \\}
4120 \\pub test "" {
4121 \\ async a && b;
4122 \\}
4123 , &[_]Error{
4124 .expected_token,
4125 .expected_pub_item,
4126 .expected_param_list,
4127 .invalid_and,
4128 });
4129 try testError(
4130 \\threadlocal test "" {
4131 \\ @a && b;
4132 \\}
4133 , &[_]Error{
4134 .expected_var_decl,
4135 .expected_param_list,
4136 .invalid_and,
4137 });
4138}
41394139
4140//test "recovery: invalid extern/inline" {
4141// try testError(
4142// \\inline test "" { a && b; }
4143// , &[_]Error{
4144// .ExpectedFn,
4145// .invalid_and,
4146// });
4147// try testError(
4148// \\extern "" test "" { a && b; }
4149// , &[_]Error{
4150// .ExpectedVarDeclOrFn,
4151// .invalid_and,
4152// });
4153//}
4140test "recovery: invalid extern/inline" {
4141 try testError(
4142 \\inline test "" { a && b; }
4143 , &[_]Error{
4144 .expected_fn,
4145 .invalid_and,
4146 });
4147 try testError(
4148 \\extern "" test "" { a && b; }
4149 , &[_]Error{
4150 .expected_var_decl_or_fn,
4151 .invalid_and,
4152 });
4153}
41544154
4155//test "recovery: missing semicolon" {
4156// try testError(
4157// \\test "" {
4158// \\ comptime a && b
4159// \\ c && d
4160// \\ @foo
4161// \\}
4162// , &[_]Error{
4163// .invalid_and,
4164// .expected_token,
4165// .invalid_and,
4166// .expected_token,
4167// .ExpectedParamList,
4168// .expected_token,
4169// });
4170//}
4155test "recovery: missing semicolon" {
4156 try testError(
4157 \\test "" {
4158 \\ comptime a && b
4159 \\ c && d
4160 \\ @foo
4161 \\}
4162 , &[_]Error{
4163 .invalid_and,
4164 .expected_token,
4165 .invalid_and,
4166 .expected_token,
4167 .expected_param_list,
4168 .expected_token,
4169 });
4170}
41714171
4172//test "recovery: invalid container members" {
4173// try testError(
4174// \\usingnamespace;
4175// \\foo+
4176// \\bar@,
4177// \\while (a == 2) { test "" {}}
4178// \\test "" {
4179// \\ a && b
4180// \\}
4181// , &[_]Error{
4182// .expected_expr,
4183// .expected_token,
4184// .expected_token,
4185// .expected_container_members,
4186// .invalid_and,
4187// .expected_token,
4188// });
4189//}
4172test "recovery: invalid container members" {
4173 try testError(
4174 \\usingnamespace;
4175 \\foo+
4176 \\bar@,
4177 \\while (a == 2) { test "" {}}
4178 \\test "" {
4179 \\ a && b
4180 \\}
4181 , &[_]Error{
4182 .expected_expr,
4183 .expected_token,
4184 .expected_container_members,
4185 .invalid_and,
4186 .expected_token,
4187 });
4188}
41904189
41914190//test "recovery: invalid parameter" {
41924191// try testError(