| ... | ... | @@ -57,16 +57,10 @@ pub fn parse(allocator: *Allocator, source: []const u8) Allocator.Error!*Tree { |
| 57 | 57 | fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error!*Node.Root { |
| 58 | 58 | const node = try arena.create(Node.Root); |
| 59 | 59 | node.* = .{ |
| 60 | | .decls = try parseContainerMembers(arena, it, tree), |
| 61 | | .eof_token = eatToken(it, .Eof) orelse blk: { |
| 62 | | // parseContainerMembers will try to skip as much |
| 63 | | // invalid tokens as it can so this can only be a '}' |
| 64 | | const tok = eatToken(it, .RBrace).?; |
| 65 | | try tree.errors.push(.{ |
| 66 | | .ExpectedContainerMembers = .{ .token = tok }, |
| 67 | | }); |
| 68 | | break :blk tok; |
| 69 | | }, |
| 60 | .decls = try parseContainerMembers(arena, it, tree, true), |
| 61 | // parseContainerMembers will try to skip as much |
| 62 | // invalid tokens as it can so this can only be the EOF |
| 63 | .eof_token = eatToken(it, .Eof).?, |
| 70 | 64 | }; |
| 71 | 65 | return node; |
| 72 | 66 | } |
| ... | ... | @@ -75,10 +69,10 @@ fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error |
| 75 | 69 | /// <- TestDecl ContainerMembers |
| 76 | 70 | /// / TopLevelComptime ContainerMembers |
| 77 | 71 | /// / KEYWORD_pub? TopLevelDecl ContainerMembers |
| 78 | | /// / KEYWORD_pub? ContainerField COMMA ContainerMembers |
| 79 | | /// / KEYWORD_pub? ContainerField |
| 72 | /// / ContainerField COMMA ContainerMembers |
| 73 | /// / ContainerField |
| 80 | 74 | /// / |
| 81 | | fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !Node.Root.DeclList { |
| 75 | fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree, top_level: bool) !Node.Root.DeclList { |
| 82 | 76 | var list = Node.Root.DeclList.init(arena); |
| 83 | 77 | |
| 84 | 78 | var field_state: union(enum) { |
| ... | ... | @@ -205,9 +199,15 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No |
| 205 | 199 | // try to continue parsing |
| 206 | 200 | const index = it.index; |
| 207 | 201 | findNextContainerMember(it); |
| 208 | | switch (it.peek().?.id) { |
| 209 | | .Eof, .RBrace => break, |
| 202 | const next = it.peek().?.id; |
| 203 | switch (next) { |
| 204 | .Eof => break, |
| 210 | 205 | else => { |
| 206 | if (next == .RBrace) { |
| 207 | if (!top_level) break; |
| 208 | _ = nextToken(it); |
| 209 | } |
| 210 | |
| 211 | 211 | // add error and continue |
| 212 | 212 | try tree.errors.push(.{ |
| 213 | 213 | .ExpectedToken = .{ .token = index, .expected_id = .Comma }, |
| ... | ... | @@ -228,12 +228,18 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No |
| 228 | 228 | }); |
| 229 | 229 | } |
| 230 | 230 | |
| 231 | | switch (it.peek().?.id) { |
| 232 | | .Eof, .RBrace => break, |
| 231 | const next = it.peek().?.id; |
| 232 | switch (next) { |
| 233 | .Eof => break, |
| 233 | 234 | else => { |
| 235 | const index = it.index; |
| 236 | if (next == .RBrace) { |
| 237 | if (!top_level) break; |
| 238 | _ = nextToken(it); |
| 239 | } |
| 240 | |
| 234 | 241 | // this was likely not supposed to end yet, |
| 235 | 242 | // try to find the next declaration |
| 236 | | const index = it.index; |
| 237 | 243 | findNextContainerMember(it); |
| 238 | 244 | try tree.errors.push(.{ |
| 239 | 245 | .ExpectedContainerMembers = .{ .token = index }, |
| ... | ... | @@ -278,7 +284,10 @@ fn findNextContainerMember(it: *TokenIterator) void { |
| 278 | 284 | } |
| 279 | 285 | }, |
| 280 | 286 | .LParen, .LBracket, .LBrace => level += 1, |
| 281 | | .RParen, .RBracket, .RBrace => { |
| 287 | .RParen, .RBracket => { |
| 288 | if (level != 0) level -= 1; |
| 289 | }, |
| 290 | .RBrace => { |
| 282 | 291 | if (level == 0) { |
| 283 | 292 | // end of container, exit |
| 284 | 293 | putBackToken(it, tok.index); |
| ... | ... | @@ -402,20 +411,16 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node |
| 402 | 411 | fn_node.*.extern_export_inline_token = extern_export_inline_token; |
| 403 | 412 | fn_node.*.lib_name = lib_name; |
| 404 | 413 | if (eatToken(it, .Semicolon)) |_| return node; |
| 405 | | if (parseBlock(arena, it, tree) catch |err| switch (err) { |
| 406 | | error.OutOfMemory => return error.OutOfMemory, |
| 414 | |
| 415 | if (try expectNodeRecoverable(arena, it, tree, parseBlock, .{ |
| 407 | 416 | // since parseBlock only return error.ParseError on |
| 408 | 417 | // a missing '}' we can assume this function was |
| 409 | 418 | // supposed to end here. |
| 410 | | error.ParseError => return node, |
| 411 | | }) |body_node| { |
| 419 | .ExpectedSemiOrLBrace = .{ .token = it.index }, |
| 420 | })) |body_node| { |
| 412 | 421 | fn_node.body_node = body_node; |
| 413 | | return node; |
| 414 | 422 | } |
| 415 | | try tree.errors.push(.{ |
| 416 | | .ExpectedSemiOrLBrace = .{ .token = it.index }, |
| 417 | | }); |
| 418 | | return error.ParseError; |
| 423 | return node; |
| 419 | 424 | } |
| 420 | 425 | |
| 421 | 426 | if (extern_export_inline_token) |token| { |
| ... | ... | @@ -490,14 +495,11 @@ fn parseFnProto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { |
| 490 | 495 | const exclamation_token = eatToken(it, .Bang); |
| 491 | 496 | |
| 492 | 497 | const return_type_expr = (try parseVarType(arena, it, tree)) orelse |
| 493 | | (try parseTypeExpr(arena, it, tree)) orelse blk: { |
| 494 | | try tree.errors.push(.{ |
| 495 | | .ExpectedReturnType = .{ .token = it.index }, |
| 496 | | }); |
| 498 | try expectNodeRecoverable(arena, it, tree, parseTypeExpr, .{ |
| 497 | 499 | // most likely the user forgot to specify the return type. |
| 498 | 500 | // Mark return type as invalid and try to continue. |
| 499 | | break :blk null; |
| 500 | | }; |
| 501 | .ExpectedReturnType = .{ .token = it.index }, |
| 502 | }); |
| 501 | 503 | |
| 502 | 504 | // TODO https://github.com/ziglang/zig/issues/3750 |
| 503 | 505 | const R = Node.FnProto.ReturnType; |
| ... | ... | @@ -508,9 +510,10 @@ fn parseFnProto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { |
| 508 | 510 | else |
| 509 | 511 | R{ .Explicit = return_type_expr.? }; |
| 510 | 512 | |
| 511 | | const var_args_token = if (params.len > 0) |
| 512 | | params.at(params.len - 1).*.cast(Node.ParamDecl).?.var_args_token |
| 513 | | else |
| 513 | const var_args_token = if (params.len > 0) blk: { |
| 514 | const param_type = params.at(params.len - 1).*.cast(Node.ParamDecl).?.param_type; |
| 515 | break :blk if (param_type == .var_args) param_type.var_args else null; |
| 516 | } else |
| 514 | 517 | null; |
| 515 | 518 | |
| 516 | 519 | const fn_proto_node = try arena.create(Node.FnProto); |
| ... | ... | @@ -707,12 +710,7 @@ fn parseStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?*No |
| 707 | 710 | if (try parseLabeledStatement(arena, it, tree)) |node| return node; |
| 708 | 711 | if (try parseSwitchExpr(arena, it, tree)) |node| return node; |
| 709 | 712 | if (try parseAssignExpr(arena, it, tree)) |node| { |
| 710 | | _ = eatToken(it, .Semicolon) orelse { |
| 711 | | try tree.errors.push(.{ |
| 712 | | .ExpectedToken = .{ .token = it.index, .expected_id = .Semicolon }, |
| 713 | | }); |
| 714 | | // pretend we saw a semicolon and continue parsing |
| 715 | | }; |
| 713 | _ = try expectTokenRecoverable(it, tree, .Semicolon); |
| 716 | 714 | return node; |
| 717 | 715 | } |
| 718 | 716 | |
| ... | ... | @@ -727,16 +725,12 @@ fn parseIfStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node |
| 727 | 725 | const if_prefix = if_node.cast(Node.If).?; |
| 728 | 726 | |
| 729 | 727 | const block_expr = (try parseBlockExpr(arena, it, tree)); |
| 730 | | const assign_expr = if (block_expr == null) blk: { |
| 731 | | break :blk (try parseAssignExpr(arena, it, tree)) orelse null; |
| 732 | | } else null; |
| 733 | | |
| 734 | | if (block_expr == null and assign_expr == null) { |
| 735 | | try tree.errors.push(.{ |
| 728 | const assign_expr = if (block_expr == null) |
| 729 | try expectNode(arena, it, tree, parseAdditionExpr, .{ |
| 736 | 730 | .ExpectedBlockOrAssignment = .{ .token = it.index }, |
| 737 | | }); |
| 738 | | return error.ParseError; |
| 739 | | } |
| 731 | }) |
| 732 | else |
| 733 | null; |
| 740 | 734 | |
| 741 | 735 | const semicolon = if (assign_expr != null) eatToken(it, .Semicolon) else null; |
| 742 | 736 | |
| ... | ... | @@ -773,10 +767,9 @@ fn parseIfStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node |
| 773 | 767 | try tree.errors.push(.{ |
| 774 | 768 | .ExpectedSemiOrElse = .{ .token = it.index }, |
| 775 | 769 | }); |
| 776 | | return error.ParseError; |
| 777 | 770 | } |
| 778 | 771 | |
| 779 | | unreachable; |
| 772 | return if_node; |
| 780 | 773 | } |
| 781 | 774 | |
| 782 | 775 | /// LabeledStatement <- BlockLabel? (Block / LoopStatement) |
| ... | ... | @@ -882,7 +875,8 @@ fn parseForStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node |
| 882 | 875 | try tree.errors.push(.{ |
| 883 | 876 | .ExpectedSemiOrElse = .{ .token = it.index }, |
| 884 | 877 | }); |
| 885 | | return null; |
| 878 | |
| 879 | return node; |
| 886 | 880 | } |
| 887 | 881 | |
| 888 | 882 | return null; |
| ... | ... | @@ -944,7 +938,8 @@ fn parseWhileStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*No |
| 944 | 938 | try tree.errors.push(.{ |
| 945 | 939 | .ExpectedSemiOrElse = .{ .token = it.index }, |
| 946 | 940 | }); |
| 947 | | return null; |
| 941 | |
| 942 | return node; |
| 948 | 943 | } |
| 949 | 944 | |
| 950 | 945 | return null; |
| ... | ... | @@ -956,12 +951,7 @@ fn parseWhileStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*No |
| 956 | 951 | fn parseBlockExprStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { |
| 957 | 952 | if (try parseBlockExpr(arena, it, tree)) |node| return node; |
| 958 | 953 | if (try parseAssignExpr(arena, it, tree)) |node| { |
| 959 | | _ = eatToken(it, .Semicolon) orelse { |
| 960 | | try tree.errors.push(.{ |
| 961 | | .ExpectedToken = .{ .token = it.index, .expected_id = .Semicolon }, |
| 962 | | }); |
| 963 | | // pretend we saw a semicolon and continue parsing |
| 964 | | }; |
| 954 | _ = try expectTokenRecoverable(it, tree, .Semicolon); |
| 965 | 955 | return node; |
| 966 | 956 | } |
| 967 | 957 | return null; |
| ... | ... | @@ -1478,17 +1468,19 @@ fn parsePrimaryTypeExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*N |
| 1478 | 1468 | return &node.base; |
| 1479 | 1469 | } |
| 1480 | 1470 | if (eatToken(it, .Keyword_error)) |token| { |
| 1481 | | const period = try expectToken(it, tree, .Period); |
| 1482 | | const identifier = try expectNode(arena, it, tree, parseIdentifier, .{ |
| 1471 | const period = try expectTokenRecoverable(it, tree, .Period); |
| 1472 | const identifier = try expectNodeRecoverable(arena, it, tree, parseIdentifier, .{ |
| 1483 | 1473 | .ExpectedIdentifier = .{ .token = it.index }, |
| 1484 | 1474 | }); |
| 1485 | 1475 | const global_error_set = try createLiteral(arena, Node.ErrorType, token); |
| 1476 | if (period == null or identifier == null) return global_error_set; |
| 1477 | |
| 1486 | 1478 | const node = try arena.create(Node.InfixOp); |
| 1487 | 1479 | node.* = .{ |
| 1488 | | .op_token = period, |
| 1480 | .op_token = period.?, |
| 1489 | 1481 | .lhs = global_error_set, |
| 1490 | 1482 | .op = .Period, |
| 1491 | | .rhs = identifier, |
| 1483 | .rhs = identifier.?, |
| 1492 | 1484 | }; |
| 1493 | 1485 | return &node.base; |
| 1494 | 1486 | } |
| ... | ... | @@ -1948,15 +1940,8 @@ fn parseParamDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { |
| 1948 | 1940 | .comptime_token = comptime_token, |
| 1949 | 1941 | .noalias_token = noalias_token, |
| 1950 | 1942 | .name_token = name_token, |
| 1951 | | // TODO: These should be squished into a ParamType enum |
| 1952 | | .type_node = undefined, |
| 1953 | | .var_args_token = null, |
| 1943 | .param_type = param_type, |
| 1954 | 1944 | }; |
| 1955 | | switch (param_type) { |
| 1956 | | .VarType => |node| param_decl.type_node = node, |
| 1957 | | .TypeExpr => |node| param_decl.type_node = node, |
| 1958 | | .VarArgs => |token| param_decl.var_args_token = token, |
| 1959 | | } |
| 1960 | 1945 | return &param_decl.base; |
| 1961 | 1946 | } |
| 1962 | 1947 | |
| ... | ... | @@ -1964,20 +1949,15 @@ fn parseParamDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { |
| 1964 | 1949 | /// <- KEYWORD_var |
| 1965 | 1950 | /// / DOT3 |
| 1966 | 1951 | /// / TypeExpr |
| 1967 | | fn parseParamType(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?ParamType { |
| 1968 | | if (try parseVarType(arena, it, tree)) |node| return ParamType{ .VarType = node }; |
| 1969 | | if (eatToken(it, .Ellipsis3)) |token| return ParamType{ .VarArgs = token }; |
| 1970 | | if (try parseTypeExpr(arena, it, tree)) |node| return ParamType{ .TypeExpr = node }; |
| 1952 | fn parseParamType(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?Node.ParamDecl.ParamType { |
| 1953 | // TODO cast from tuple to error union is broken |
| 1954 | const P = Node.ParamDecl.ParamType; |
| 1955 | if (try parseVarType(arena, it, tree)) |node| return P{ .var_type = node }; |
| 1956 | if (eatToken(it, .Ellipsis3)) |token| return P{ .var_args = token }; |
| 1957 | if (try parseTypeExpr(arena, it, tree)) |node| return P{ .type_expr = node }; |
| 1971 | 1958 | return null; |
| 1972 | 1959 | } |
| 1973 | 1960 | |
| 1974 | | // TODO: Move to ast.Node.ParamDecl.ParamType |
| 1975 | | const ParamType = union(enum) { |
| 1976 | | VarType: *Node, |
| 1977 | | VarArgs: TokenIndex, |
| 1978 | | TypeExpr: *Node, |
| 1979 | | }; |
| 1980 | | |
| 1981 | 1961 | /// IfPrefix <- KEYWORD_if LPAREN Expr RPAREN PtrPayload? |
| 1982 | 1962 | fn parseIfPrefix(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { |
| 1983 | 1963 | const if_token = eatToken(it, .Keyword_if) orelse return null; |
| ... | ... | @@ -2778,7 +2758,7 @@ fn parsePtrTypeStart(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node |
| 2778 | 2758 | fn parseContainerDeclAuto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { |
| 2779 | 2759 | const node = (try parseContainerDeclType(arena, it, tree)) orelse return null; |
| 2780 | 2760 | const lbrace = try expectToken(it, tree, .LBrace); |
| 2781 | | const members = try parseContainerMembers(arena, it, tree); |
| 2761 | const members = try parseContainerMembers(arena, it, tree, false); |
| 2782 | 2762 | const rbrace = try expectToken(it, tree, .RBrace); |
| 2783 | 2763 | |
| 2784 | 2764 | const decl_type = node.cast(Node.ContainerDecl).?; |
| ... | ... | @@ -3250,6 +3230,11 @@ fn eatAnnotatedToken(it: *TokenIterator, id: Token.Id) ?AnnotatedToken { |
| 3250 | 3230 | } |
| 3251 | 3231 | |
| 3252 | 3232 | fn expectToken(it: *TokenIterator, tree: *Tree, id: Token.Id) Error!TokenIndex { |
| 3233 | return (try expectTokenRecoverable(it, tree, id)) orelse |
| 3234 | error.ParseError; |
| 3235 | } |
| 3236 | |
| 3237 | fn expectTokenRecoverable(it: *TokenIterator, tree: *Tree, id: Token.Id) !?TokenIndex { |
| 3253 | 3238 | const token = nextToken(it); |
| 3254 | 3239 | if (token.ptr.id != id) { |
| 3255 | 3240 | try tree.errors.push(.{ |
| ... | ... | @@ -3257,7 +3242,7 @@ fn expectToken(it: *TokenIterator, tree: *Tree, id: Token.Id) Error!TokenIndex { |
| 3257 | 3242 | }); |
| 3258 | 3243 | // go back so that we can recover properly |
| 3259 | 3244 | putBackToken(it, token.index); |
| 3260 | | return error.ParseError; |
| 3245 | return null; |
| 3261 | 3246 | } |
| 3262 | 3247 | return token.index; |
| 3263 | 3248 | } |
| ... | ... | @@ -3297,9 +3282,20 @@ fn expectNode( |
| 3297 | 3282 | parseFn: NodeParseFn, |
| 3298 | 3283 | err: AstError, // if parsing fails |
| 3299 | 3284 | ) Error!*Node { |
| 3285 | return (try expectNodeRecoverable(arena, it, tree, parseFn, err)) orelse |
| 3286 | return error.ParseError; |
| 3287 | } |
| 3288 | |
| 3289 | fn expectNodeRecoverable( |
| 3290 | arena: *Allocator, |
| 3291 | it: *TokenIterator, |
| 3292 | tree: *Tree, |
| 3293 | parseFn: NodeParseFn, |
| 3294 | err: AstError, // if parsing fails |
| 3295 | ) !?*Node { |
| 3300 | 3296 | return (try parseFn(arena, it, tree)) orelse { |
| 3301 | 3297 | try tree.errors.push(err); |
| 3302 | | return error.ParseError; |
| 3298 | return null; |
| 3303 | 3299 | }; |
| 3304 | 3300 | } |
| 3305 | 3301 | |