authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-05-07 23:54:35-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-05-07 23:54:35-04:00
logca27ce3bee16ebb611621f15830dd6bf74d65f9f
treeeb3c01eb95da13f282e0c1194d6ec8588713c101
parent0cb65b266aa20015f068e0460c74eb75a0b7f65c

std.zig.parser supports same-line comments on any token

3 files changed, 351 insertions(+), 273 deletions(-)

std/zig/ast.zig+9-3
......@@ -25,7 +25,10 @@ pub const Tree = struct {
2525 }
2626
2727 pub fn tokenSlice(self: &Tree, token_index: TokenIndex) []const u8 {
28 const token = self.tokens.at(token_index);
28 return self.tokenSlicePtr(self.tokens.at(token_index));
29 }
30
31 pub fn tokenSlicePtr(self: &Tree, token: &const Token) []const u8 {
2932 return self.source[token.start..token.end];
3033 }
3134
......@@ -36,14 +39,14 @@ pub const Tree = struct {
3639 line_end: usize,
3740 };
3841
39 pub fn tokenLocation(self: &Tree, start_index: usize, token_index: TokenIndex) Location {
42 pub fn tokenLocationPtr(self: &Tree, start_index: usize, token: &const Token) Location {
4043 var loc = Location {
4144 .line = 0,
4245 .column = 0,
4346 .line_start = start_index,
4447 .line_end = self.source.len,
4548 };
46 const token_start = self.tokens.at(token_index).start;
49 const token_start = token.start;
4750 for (self.source[start_index..]) |c, i| {
4851 if (i + start_index == token_start) {
4952 loc.line_end = i + start_index;
......@@ -61,6 +64,9 @@ pub const Tree = struct {
6164 return loc;
6265 }
6366
67 pub fn tokenLocation(self: &Tree, start_index: usize, token_index: TokenIndex) Location {
68 return self.tokenLocationPtr(start_index, self.tokens.at(token_index));
69 }
6470};
6571
6672pub const Error = union(enum) {
std/zig/parse.zig+288-216
......@@ -54,14 +54,15 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
5454
5555 switch (state) {
5656 State.TopLevel => {
57 while (try eatLineComment(arena, &tok_it)) |line_comment| {
57 while (try eatLineComment(arena, &tok_it, &tree)) |line_comment| {
5858 try root_node.decls.push(&line_comment.base);
5959 }
6060
61 const comments = try eatDocComments(arena, &tok_it);
61 const comments = try eatDocComments(arena, &tok_it, &tree);
6262
63 const token_index = tok_it.index;
64 const token_ptr = ??tok_it.next();
63 const token = nextToken(&tok_it, &tree);
64 const token_index = token.index;
65 const token_ptr = token.ptr;
6566 switch (token_ptr.id) {
6667 Token.Id.Keyword_test => {
6768 stack.push(State.TopLevel) catch unreachable;
......@@ -144,7 +145,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
144145 continue;
145146 },
146147 else => {
147 _ = tok_it.prev();
148 putBackToken(&tok_it, &tree);
148149 stack.push(State.TopLevel) catch unreachable;
149150 try stack.push(State {
150151 .TopLevelExtern = TopLevelDeclCtx {
......@@ -160,8 +161,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
160161 }
161162 },
162163 State.TopLevelExtern => |ctx| {
163 const token_index = tok_it.index;
164 const token_ptr = ??tok_it.next();
164 const token = nextToken(&tok_it, &tree);
165 const token_index = token.index;
166 const token_ptr = token.ptr;
165167 switch (token_ptr.id) {
166168 Token.Id.Keyword_export, Token.Id.Keyword_inline => {
167169 stack.push(State {
......@@ -194,7 +196,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
194196 continue;
195197 },
196198 else => {
197 _ = tok_it.prev();
199 putBackToken(&tok_it, &tree);
198200 stack.push(State { .TopLevelDecl = ctx }) catch unreachable;
199201 continue;
200202 }
......@@ -202,10 +204,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
202204 },
203205 State.TopLevelLibname => |ctx| {
204206 const lib_name = blk: {
205 const lib_name_token_index = tok_it.index;
206 const lib_name_token_ptr = ??tok_it.next();
207 break :blk (try parseStringLiteral(arena, &tok_it, lib_name_token_ptr, lib_name_token_index)) ?? {
208 _ = tok_it.prev();
207 const lib_name_token = nextToken(&tok_it, &tree);
208 const lib_name_token_index = lib_name_token.index;
209 const lib_name_token_ptr = lib_name_token.ptr;
210 break :blk (try parseStringLiteral(arena, &tok_it, lib_name_token_ptr, lib_name_token_index, &tree)) ?? {
211 putBackToken(&tok_it, &tree);
209212 break :blk null;
210213 };
211214 };
......@@ -222,8 +225,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
222225 continue;
223226 },
224227 State.TopLevelDecl => |ctx| {
225 const token_index = tok_it.index;
226 const token_ptr = ??tok_it.next();
228 const token = nextToken(&tok_it, &tree);
229 const token_index = token.index;
230 const token_ptr = token.ptr;
227231 switch (token_ptr.id) {
228232 Token.Id.Keyword_use => {
229233 if (ctx.extern_export_inline_token) |annotated_token| {
......@@ -345,7 +349,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
345349 }
346350 },
347351 State.TopLevelExternOrField => |ctx| {
348 if (eatToken(&tok_it, Token.Id.Identifier)) |identifier| {
352 if (eatToken(&tok_it, &tree, Token.Id.Identifier)) |identifier| {
349353 std.debug.assert(ctx.container_decl.kind == ast.Node.ContainerDecl.Kind.Struct);
350354 const node = try arena.construct(ast.Node.StructField {
351355 .base = ast.Node {
......@@ -379,10 +383,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
379383 },
380384
381385 State.FieldInitValue => |ctx| {
382 const eq_tok_index = tok_it.index;
383 const eq_tok_ptr = ??tok_it.next();
386 const eq_tok = nextToken(&tok_it, &tree);
387 const eq_tok_index = eq_tok.index;
388 const eq_tok_ptr = eq_tok.ptr;
384389 if (eq_tok_ptr.id != Token.Id.Equal) {
385 _ = tok_it.prev();
390 putBackToken(&tok_it, &tree);
386391 continue;
387392 }
388393 stack.push(State { .Expression = ctx }) catch unreachable;
......@@ -390,8 +395,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
390395 },
391396
392397 State.ContainerKind => |ctx| {
393 const token_index = tok_it.index;
394 const token_ptr = ??tok_it.next();
398 const token = nextToken(&tok_it, &tree);
399 const token_index = token.index;
400 const token_ptr = token.ptr;
395401 const node = try createToCtxNode(arena, ctx.opt_ctx, ast.Node.ContainerDecl,
396402 ast.Node.ContainerDecl {
397403 .base = undefined,
......@@ -421,7 +427,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
421427 },
422428
423429 State.ContainerInitArgStart => |container_decl| {
424 if (eatToken(&tok_it, Token.Id.LParen) == null) {
430 if (eatToken(&tok_it, &tree, Token.Id.LParen) == null) {
425431 continue;
426432 }
427433
......@@ -431,24 +437,26 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
431437 },
432438
433439 State.ContainerInitArg => |container_decl| {
434 const init_arg_token_index = tok_it.index;
435 const init_arg_token_ptr = ??tok_it.next();
440 const init_arg_token = nextToken(&tok_it, &tree);
441 const init_arg_token_index = init_arg_token.index;
442 const init_arg_token_ptr = init_arg_token.ptr;
436443 switch (init_arg_token_ptr.id) {
437444 Token.Id.Keyword_enum => {
438445 container_decl.init_arg_expr = ast.Node.ContainerDecl.InitArg {.Enum = null};
439 const lparen_tok_index = tok_it.index;
440 const lparen_tok_ptr = ??tok_it.next();
446 const lparen_tok = nextToken(&tok_it, &tree);
447 const lparen_tok_index = lparen_tok.index;
448 const lparen_tok_ptr = lparen_tok.ptr;
441449 if (lparen_tok_ptr.id == Token.Id.LParen) {
442450 try stack.push(State { .ExpectToken = Token.Id.RParen } );
443451 try stack.push(State { .Expression = OptionalCtx {
444452 .RequiredNull = &container_decl.init_arg_expr.Enum,
445453 } });
446454 } else {
447 _ = tok_it.prev();
455 putBackToken(&tok_it, &tree);
448456 }
449457 },
450458 else => {
451 _ = tok_it.prev();
459 putBackToken(&tok_it, &tree);
452460 container_decl.init_arg_expr = ast.Node.ContainerDecl.InitArg { .Type = undefined };
453461 stack.push(State { .Expression = OptionalCtx { .Required = &container_decl.init_arg_expr.Type } }) catch unreachable;
454462 },
......@@ -457,13 +465,14 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
457465 },
458466
459467 State.ContainerDecl => |container_decl| {
460 while (try eatLineComment(arena, &tok_it)) |line_comment| {
468 while (try eatLineComment(arena, &tok_it, &tree)) |line_comment| {
461469 try container_decl.fields_and_decls.push(&line_comment.base);
462470 }
463471
464 const comments = try eatDocComments(arena, &tok_it);
465 const token_index = tok_it.index;
466 const token_ptr = ??tok_it.next();
472 const comments = try eatDocComments(arena, &tok_it, &tree);
473 const token = nextToken(&tok_it, &tree);
474 const token_index = token.index;
475 const token_ptr = token.ptr;
467476 switch (token_ptr.id) {
468477 Token.Id.Identifier => {
469478 switch (container_decl.kind) {
......@@ -568,7 +577,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
568577 continue;
569578 },
570579 else => {
571 _ = tok_it.prev();
580 putBackToken(&tok_it, &tree);
572581 stack.push(State{ .ContainerDecl = container_decl }) catch unreachable;
573582 try stack.push(State {
574583 .TopLevelExtern = TopLevelDeclCtx {
......@@ -620,8 +629,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
620629 State.VarDeclAlign => |var_decl| {
621630 try stack.push(State { .VarDeclEq = var_decl });
622631
623 const next_token_index = tok_it.index;
624 const next_token_ptr = ??tok_it.next();
632 const next_token = nextToken(&tok_it, &tree);
633 const next_token_index = next_token.index;
634 const next_token_ptr = next_token.ptr;
625635 if (next_token_ptr.id == Token.Id.Keyword_align) {
626636 try stack.push(State { .ExpectToken = Token.Id.RParen });
627637 try stack.push(State { .Expression = OptionalCtx { .RequiredNull = &var_decl.align_node} });
......@@ -629,12 +639,13 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
629639 continue;
630640 }
631641
632 _ = tok_it.prev();
642 putBackToken(&tok_it, &tree);
633643 continue;
634644 },
635645 State.VarDeclEq => |var_decl| {
636 const token_index = tok_it.index;
637 const token_ptr = ??tok_it.next();
646 const token = nextToken(&tok_it, &tree);
647 const token_index = token.index;
648 const token_ptr = token.ptr;
638649 switch (token_ptr.id) {
639650 Token.Id.Equal => {
640651 var_decl.eq_token = token_index;
......@@ -662,8 +673,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
662673
663674
664675 State.FnDef => |fn_proto| {
665 const token_index = tok_it.index;
666 const token_ptr = ??tok_it.next();
676 const token = nextToken(&tok_it, &tree);
677 const token_index = token.index;
678 const token_ptr = token.ptr;
667679 switch(token_ptr.id) {
668680 Token.Id.LBrace => {
669681 const block = try arena.construct(ast.Node.Block {
......@@ -691,7 +703,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
691703 try stack.push(State { .ParamDecl = fn_proto });
692704 try stack.push(State { .ExpectToken = Token.Id.LParen });
693705
694 if (eatToken(&tok_it, Token.Id.Identifier)) |name_token| {
706 if (eatToken(&tok_it, &tree, Token.Id.Identifier)) |name_token| {
695707 fn_proto.name_token = name_token;
696708 }
697709 continue;
......@@ -699,7 +711,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
699711 State.FnProtoAlign => |fn_proto| {
700712 stack.push(State { .FnProtoReturnType = fn_proto }) catch unreachable;
701713
702 if (eatToken(&tok_it, Token.Id.Keyword_align)) |align_token| {
714 if (eatToken(&tok_it, &tree, Token.Id.Keyword_align)) |align_token| {
703715 try stack.push(State { .ExpectToken = Token.Id.RParen });
704716 try stack.push(State { .Expression = OptionalCtx { .RequiredNull = &fn_proto.align_expr } });
705717 try stack.push(State { .ExpectToken = Token.Id.LParen });
......@@ -707,8 +719,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
707719 continue;
708720 },
709721 State.FnProtoReturnType => |fn_proto| {
710 const token_index = tok_it.index;
711 const token_ptr = ??tok_it.next();
722 const token = nextToken(&tok_it, &tree);
723 const token_index = token.index;
724 const token_ptr = token.ptr;
712725 switch (token_ptr.id) {
713726 Token.Id.Bang => {
714727 fn_proto.return_type = ast.Node.FnProto.ReturnType { .InferErrorSet = undefined };
......@@ -732,7 +745,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
732745 }
733746 }
734747
735 _ = tok_it.prev();
748 putBackToken(&tok_it, &tree);
736749 fn_proto.return_type = ast.Node.FnProto.ReturnType { .Explicit = undefined };
737750 stack.push(State { .TypeExprBegin = OptionalCtx { .Required = &fn_proto.return_type.Explicit }, }) catch unreachable;
738751 continue;
......@@ -742,7 +755,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
742755
743756
744757 State.ParamDecl => |fn_proto| {
745 if (eatToken(&tok_it, Token.Id.RParen)) |_| {
758 if (eatToken(&tok_it, &tree, Token.Id.RParen)) |_| {
746759 continue;
747760 }
748761 const param_decl = try arena.construct(ast.Node.ParamDecl {
......@@ -766,9 +779,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
766779 continue;
767780 },
768781 State.ParamDeclAliasOrComptime => |param_decl| {
769 if (eatToken(&tok_it, Token.Id.Keyword_comptime)) |comptime_token| {
782 if (eatToken(&tok_it, &tree, Token.Id.Keyword_comptime)) |comptime_token| {
770783 param_decl.comptime_token = comptime_token;
771 } else if (eatToken(&tok_it, Token.Id.Keyword_noalias)) |noalias_token| {
784 } else if (eatToken(&tok_it, &tree, Token.Id.Keyword_noalias)) |noalias_token| {
772785 param_decl.noalias_token = noalias_token;
773786 }
774787 continue;
......@@ -776,17 +789,17 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
776789 State.ParamDeclName => |param_decl| {
777790 // TODO: Here, we eat two tokens in one state. This means that we can't have
778791 // comments between these two tokens.
779 if (eatToken(&tok_it, Token.Id.Identifier)) |ident_token| {
780 if (eatToken(&tok_it, Token.Id.Colon)) |_| {
792 if (eatToken(&tok_it, &tree, Token.Id.Identifier)) |ident_token| {
793 if (eatToken(&tok_it, &tree, Token.Id.Colon)) |_| {
781794 param_decl.name_token = ident_token;
782795 } else {
783 _ = tok_it.prev();
796 putBackToken(&tok_it, &tree);
784797 }
785798 }
786799 continue;
787800 },
788801 State.ParamDeclEnd => |ctx| {
789 if (eatToken(&tok_it, Token.Id.Ellipsis3)) |ellipsis3| {
802 if (eatToken(&tok_it, &tree, Token.Id.Ellipsis3)) |ellipsis3| {
790803 ctx.param_decl.var_args_token = ellipsis3;
791804 stack.push(State { .ExpectToken = Token.Id.RParen }) catch unreachable;
792805 continue;
......@@ -799,7 +812,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
799812 continue;
800813 },
801814 State.ParamDeclComma => |fn_proto| {
802 switch (expectCommaOrEnd(&tok_it, Token.Id.RParen)) {
815 switch (expectCommaOrEnd(&tok_it, &tree, Token.Id.RParen)) {
803816 ExpectCommaOrEndResult.end_token => |t| {
804817 if (t == null) {
805818 stack.push(State { .ParamDecl = fn_proto }) catch unreachable;
......@@ -814,7 +827,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
814827 },
815828
816829 State.MaybeLabeledExpression => |ctx| {
817 if (eatToken(&tok_it, Token.Id.Colon)) |_| {
830 if (eatToken(&tok_it, &tree, Token.Id.Colon)) |_| {
818831 stack.push(State {
819832 .LabeledExpression = LabelCtx {
820833 .label = ctx.label,
......@@ -828,8 +841,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
828841 continue;
829842 },
830843 State.LabeledExpression => |ctx| {
831 const token_index = tok_it.index;
832 const token_ptr = ??tok_it.next();
844 const token = nextToken(&tok_it, &tree);
845 const token_index = token.index;
846 const token_ptr = token.ptr;
833847 switch (token_ptr.id) {
834848 Token.Id.LBrace => {
835849 const block = try createToCtxNode(arena, ctx.opt_ctx, ast.Node.Block,
......@@ -899,14 +913,15 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
899913 return tree;
900914 }
901915
902 _ = tok_it.prev();
916 putBackToken(&tok_it, &tree);
903917 continue;
904918 },
905919 }
906920 },
907921 State.Inline => |ctx| {
908 const token_index = tok_it.index;
909 const token_ptr = ??tok_it.next();
922 const token = nextToken(&tok_it, &tree);
923 const token_index = token.index;
924 const token_ptr = token.ptr;
910925 switch (token_ptr.id) {
911926 Token.Id.Keyword_while => {
912927 stack.push(State {
......@@ -938,7 +953,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
938953 return tree;
939954 }
940955
941 _ = tok_it.prev();
956 putBackToken(&tok_it, &tree);
942957 continue;
943958 },
944959 }
......@@ -995,7 +1010,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
9951010 continue;
9961011 },
9971012 State.Else => |dest| {
998 if (eatToken(&tok_it, Token.Id.Keyword_else)) |else_token| {
1013 if (eatToken(&tok_it, &tree, Token.Id.Keyword_else)) |else_token| {
9991014 const node = try createNode(arena, ast.Node.Else,
10001015 ast.Node.Else {
10011016 .base = undefined,
......@@ -1016,19 +1031,20 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
10161031
10171032
10181033 State.Block => |block| {
1019 const token_index = tok_it.index;
1020 const token_ptr = ??tok_it.next();
1034 const token = nextToken(&tok_it, &tree);
1035 const token_index = token.index;
1036 const token_ptr = token.ptr;
10211037 switch (token_ptr.id) {
10221038 Token.Id.RBrace => {
10231039 block.rbrace = token_index;
10241040 continue;
10251041 },
10261042 else => {
1027 _ = tok_it.prev();
1043 putBackToken(&tok_it, &tree);
10281044 stack.push(State { .Block = block }) catch unreachable;
10291045
10301046 var any_comments = false;
1031 while (try eatLineComment(arena, &tok_it)) |line_comment| {
1047 while (try eatLineComment(arena, &tok_it, &tree)) |line_comment| {
10321048 try block.statements.push(&line_comment.base);
10331049 any_comments = true;
10341050 }
......@@ -1040,8 +1056,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
10401056 }
10411057 },
10421058 State.Statement => |block| {
1043 const token_index = tok_it.index;
1044 const token_ptr = ??tok_it.next();
1059 const token = nextToken(&tok_it, &tree);
1060 const token_index = token.index;
1061 const token_ptr = token.ptr;
10451062 switch (token_ptr.id) {
10461063 Token.Id.Keyword_comptime => {
10471064 stack.push(State {
......@@ -1100,7 +1117,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
11001117 continue;
11011118 },
11021119 else => {
1103 _ = tok_it.prev();
1120 putBackToken(&tok_it, &tree);
11041121 const statement = try block.statements.addOne();
11051122 try stack.push(State { .Semicolon = statement });
11061123 try stack.push(State { .AssignmentExpressionBegin = OptionalCtx{ .Required = statement } });
......@@ -1109,8 +1126,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
11091126 }
11101127 },
11111128 State.ComptimeStatement => |ctx| {
1112 const token_index = tok_it.index;
1113 const token_ptr = ??tok_it.next();
1129 const token = nextToken(&tok_it, &tree);
1130 const token_index = token.index;
1131 const token_ptr = token.ptr;
11141132 switch (token_ptr.id) {
11151133 Token.Id.Keyword_var, Token.Id.Keyword_const => {
11161134 stack.push(State {
......@@ -1127,8 +1145,8 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
11271145 continue;
11281146 },
11291147 else => {
1130 _ = tok_it.prev();
1131 _ = tok_it.prev();
1148 putBackToken(&tok_it, &tree);
1149 putBackToken(&tok_it, &tree);
11321150 const statement = try ctx.block.statements.addOne();
11331151 try stack.push(State { .Semicolon = statement });
11341152 try stack.push(State { .Expression = OptionalCtx { .Required = statement } });
......@@ -1146,10 +1164,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
11461164 },
11471165
11481166 State.AsmOutputItems => |items| {
1149 const lbracket_index = tok_it.index;
1150 const lbracket_ptr = ??tok_it.next();
1167 const lbracket = nextToken(&tok_it, &tree);
1168 const lbracket_index = lbracket.index;
1169 const lbracket_ptr = lbracket.ptr;
11511170 if (lbracket_ptr.id != Token.Id.LBracket) {
1152 _ = tok_it.prev();
1171 putBackToken(&tok_it, &tree);
11531172 continue;
11541173 }
11551174
......@@ -1174,8 +1193,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
11741193 continue;
11751194 },
11761195 State.AsmOutputReturnOrType => |node| {
1177 const token_index = tok_it.index;
1178 const token_ptr = ??tok_it.next();
1196 const token = nextToken(&tok_it, &tree);
1197 const token_index = token.index;
1198 const token_ptr = token.ptr;
11791199 switch (token_ptr.id) {
11801200 Token.Id.Identifier => {
11811201 node.kind = ast.Node.AsmOutput.Kind { .Variable = try createLiteral(arena, ast.Node.Identifier, token_index) };
......@@ -1197,10 +1217,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
11971217 }
11981218 },
11991219 State.AsmInputItems => |items| {
1200 const lbracket_index = tok_it.index;
1201 const lbracket_ptr = ??tok_it.next();
1220 const lbracket = nextToken(&tok_it, &tree);
1221 const lbracket_index = lbracket.index;
1222 const lbracket_ptr = lbracket.ptr;
12021223 if (lbracket_ptr.id != Token.Id.LBracket) {
1203 _ = tok_it.prev();
1224 putBackToken(&tok_it, &tree);
12041225 continue;
12051226 }
12061227
......@@ -1233,7 +1254,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
12331254
12341255
12351256 State.ExprListItemOrEnd => |list_state| {
1236 if (eatToken(&tok_it, list_state.end)) |token_index| {
1257 if (eatToken(&tok_it, &tree, list_state.end)) |token_index| {
12371258 *list_state.ptr = token_index;
12381259 continue;
12391260 }
......@@ -1243,7 +1264,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
12431264 continue;
12441265 },
12451266 State.ExprListCommaOrEnd => |list_state| {
1246 switch (expectCommaOrEnd(&tok_it, list_state.end)) {
1267 switch (expectCommaOrEnd(&tok_it, &tree, list_state.end)) {
12471268 ExpectCommaOrEndResult.end_token => |maybe_end| if (maybe_end) |end| {
12481269 *list_state.ptr = end;
12491270 continue;
......@@ -1258,11 +1279,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
12581279 }
12591280 },
12601281 State.FieldInitListItemOrEnd => |list_state| {
1261 while (try eatLineComment(arena, &tok_it)) |line_comment| {
1282 while (try eatLineComment(arena, &tok_it, &tree)) |line_comment| {
12621283 try list_state.list.push(&line_comment.base);
12631284 }
12641285
1265 if (eatToken(&tok_it, Token.Id.RBrace)) |rbrace| {
1286 if (eatToken(&tok_it, &tree, Token.Id.RBrace)) |rbrace| {
12661287 *list_state.ptr = rbrace;
12671288 continue;
12681289 }
......@@ -1295,7 +1316,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
12951316 continue;
12961317 },
12971318 State.FieldInitListCommaOrEnd => |list_state| {
1298 switch (expectCommaOrEnd(&tok_it, Token.Id.RBrace)) {
1319 switch (expectCommaOrEnd(&tok_it, &tree, Token.Id.RBrace)) {
12991320 ExpectCommaOrEndResult.end_token => |maybe_end| if (maybe_end) |end| {
13001321 *list_state.ptr = end;
13011322 continue;
......@@ -1310,7 +1331,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
13101331 }
13111332 },
13121333 State.FieldListCommaOrEnd => |container_decl| {
1313 switch (expectCommaOrEnd(&tok_it, Token.Id.RBrace)) {
1334 switch (expectCommaOrEnd(&tok_it, &tree, Token.Id.RBrace)) {
13141335 ExpectCommaOrEndResult.end_token => |maybe_end| if (maybe_end) |end| {
13151336 container_decl.rbrace_token = end;
13161337 continue;
......@@ -1325,11 +1346,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
13251346 }
13261347 },
13271348 State.ErrorTagListItemOrEnd => |list_state| {
1328 while (try eatLineComment(arena, &tok_it)) |line_comment| {
1349 while (try eatLineComment(arena, &tok_it, &tree)) |line_comment| {
13291350 try list_state.list.push(&line_comment.base);
13301351 }
13311352
1332 if (eatToken(&tok_it, Token.Id.RBrace)) |rbrace| {
1353 if (eatToken(&tok_it, &tree, Token.Id.RBrace)) |rbrace| {
13331354 *list_state.ptr = rbrace;
13341355 continue;
13351356 }
......@@ -1341,7 +1362,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
13411362 continue;
13421363 },
13431364 State.ErrorTagListCommaOrEnd => |list_state| {
1344 switch (expectCommaOrEnd(&tok_it, Token.Id.RBrace)) {
1365 switch (expectCommaOrEnd(&tok_it, &tree, Token.Id.RBrace)) {
13451366 ExpectCommaOrEndResult.end_token => |maybe_end| if (maybe_end) |end| {
13461367 *list_state.ptr = end;
13471368 continue;
......@@ -1356,16 +1377,16 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
13561377 }
13571378 },
13581379 State.SwitchCaseOrEnd => |list_state| {
1359 while (try eatLineComment(arena, &tok_it)) |line_comment| {
1380 while (try eatLineComment(arena, &tok_it, &tree)) |line_comment| {
13601381 try list_state.list.push(&line_comment.base);
13611382 }
13621383
1363 if (eatToken(&tok_it, Token.Id.RBrace)) |rbrace| {
1384 if (eatToken(&tok_it, &tree, Token.Id.RBrace)) |rbrace| {
13641385 *list_state.ptr = rbrace;
13651386 continue;
13661387 }
13671388
1368 const comments = try eatDocComments(arena, &tok_it);
1389 const comments = try eatDocComments(arena, &tok_it, &tree);
13691390 const node = try arena.construct(ast.Node.SwitchCase {
13701391 .base = ast.Node {
13711392 .id = ast.Node.Id.SwitchCase,
......@@ -1384,7 +1405,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
13841405 },
13851406
13861407 State.SwitchCaseCommaOrEnd => |list_state| {
1387 switch (expectCommaOrEnd(&tok_it, Token.Id.RParen)) {
1408 switch (expectCommaOrEnd(&tok_it, &tree, Token.Id.RParen)) {
13881409 ExpectCommaOrEndResult.end_token => |maybe_end| if (maybe_end) |end| {
13891410 *list_state.ptr = end;
13901411 continue;
......@@ -1400,8 +1421,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
14001421 },
14011422
14021423 State.SwitchCaseFirstItem => |case_items| {
1403 const token_index = tok_it.index;
1404 const token_ptr = ??tok_it.next();
1424 const token = nextToken(&tok_it, &tree);
1425 const token_index = token.index;
1426 const token_ptr = token.ptr;
14051427 if (token_ptr.id == Token.Id.Keyword_else) {
14061428 const else_node = try arena.construct(ast.Node.SwitchElse {
14071429 .base = ast.Node{ .id = ast.Node.Id.SwitchElse},
......@@ -1412,7 +1434,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
14121434 try stack.push(State { .ExpectToken = Token.Id.EqualAngleBracketRight });
14131435 continue;
14141436 } else {
1415 _ = tok_it.prev();
1437 putBackToken(&tok_it, &tree);
14161438 try stack.push(State { .SwitchCaseItem = case_items });
14171439 continue;
14181440 }
......@@ -1422,7 +1444,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
14221444 try stack.push(State { .RangeExpressionBegin = OptionalCtx { .Required = try case_items.addOne() } });
14231445 },
14241446 State.SwitchCaseItemCommaOrEnd => |case_items| {
1425 switch (expectCommaOrEnd(&tok_it, Token.Id.EqualAngleBracketRight)) {
1447 switch (expectCommaOrEnd(&tok_it, &tree, Token.Id.EqualAngleBracketRight)) {
14261448 ExpectCommaOrEndResult.end_token => |t| {
14271449 if (t == null) {
14281450 stack.push(State { .SwitchCaseItem = case_items }) catch unreachable;
......@@ -1445,7 +1467,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
14451467 continue;
14461468 },
14471469 State.AsyncAllocator => |async_node| {
1448 if (eatToken(&tok_it, Token.Id.AngleBracketLeft) == null) {
1470 if (eatToken(&tok_it, &tree, Token.Id.AngleBracketLeft) == null) {
14491471 continue;
14501472 }
14511473
......@@ -1491,7 +1513,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
14911513
14921514
14931515 State.ExternType => |ctx| {
1494 if (eatToken(&tok_it, Token.Id.Keyword_fn)) |fn_token| {
1516 if (eatToken(&tok_it, &tree, Token.Id.Keyword_fn)) |fn_token| {
14951517 const fn_proto = try arena.construct(ast.Node.FnProto {
14961518 .base = ast.Node {
14971519 .id = ast.Node.Id.FnProto,
......@@ -1525,8 +1547,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
15251547 continue;
15261548 },
15271549 State.SliceOrArrayAccess => |node| {
1528 const token_index = tok_it.index;
1529 const token_ptr = ??tok_it.next();
1550 const token = nextToken(&tok_it, &tree);
1551 const token_index = token.index;
1552 const token_ptr = token.ptr;
15301553 switch (token_ptr.id) {
15311554 Token.Id.Ellipsis2 => {
15321555 const start = node.op.ArrayAccess;
......@@ -1559,7 +1582,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
15591582 }
15601583 },
15611584 State.SliceOrArrayType => |node| {
1562 if (eatToken(&tok_it, Token.Id.RBracket)) |_| {
1585 if (eatToken(&tok_it, &tree, Token.Id.RBracket)) |_| {
15631586 node.op = ast.Node.PrefixOp.Op {
15641587 .SliceType = ast.Node.PrefixOp.AddrOfInfo {
15651588 .align_expr = null,
......@@ -1581,8 +1604,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
15811604 continue;
15821605 },
15831606 State.AddrOfModifiers => |addr_of_info| {
1584 const token_index = tok_it.index;
1585 const token_ptr = ??tok_it.next();
1607 const token = nextToken(&tok_it, &tree);
1608 const token_index = token.index;
1609 const token_ptr = token.ptr;
15861610 switch (token_ptr.id) {
15871611 Token.Id.Keyword_align => {
15881612 stack.push(state) catch unreachable;
......@@ -1620,7 +1644,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
16201644 continue;
16211645 },
16221646 else => {
1623 _ = tok_it.prev();
1647 putBackToken(&tok_it, &tree);
16241648 continue;
16251649 },
16261650 }
......@@ -1628,8 +1652,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
16281652
16291653
16301654 State.Payload => |opt_ctx| {
1631 const token_index = tok_it.index;
1632 const token_ptr = ??tok_it.next();
1655 const token = nextToken(&tok_it, &tree);
1656 const token_index = token.index;
1657 const token_ptr = token.ptr;
16331658 if (token_ptr.id != Token.Id.Pipe) {
16341659 if (opt_ctx != OptionalCtx.Optional) {
16351660 *(try tree.errors.addOne()) = Error {
......@@ -1641,7 +1666,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
16411666 return tree;
16421667 }
16431668
1644 _ = tok_it.prev();
1669 putBackToken(&tok_it, &tree);
16451670 continue;
16461671 }
16471672
......@@ -1664,8 +1689,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
16641689 continue;
16651690 },
16661691 State.PointerPayload => |opt_ctx| {
1667 const token_index = tok_it.index;
1668 const token_ptr = ??tok_it.next();
1692 const token = nextToken(&tok_it, &tree);
1693 const token_index = token.index;
1694 const token_ptr = token.ptr;
16691695 if (token_ptr.id != Token.Id.Pipe) {
16701696 if (opt_ctx != OptionalCtx.Optional) {
16711697 *(try tree.errors.addOne()) = Error {
......@@ -1677,7 +1703,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
16771703 return tree;
16781704 }
16791705
1680 _ = tok_it.prev();
1706 putBackToken(&tok_it, &tree);
16811707 continue;
16821708 }
16831709
......@@ -1707,8 +1733,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
17071733 continue;
17081734 },
17091735 State.PointerIndexPayload => |opt_ctx| {
1710 const token_index = tok_it.index;
1711 const token_ptr = ??tok_it.next();
1736 const token = nextToken(&tok_it, &tree);
1737 const token_index = token.index;
1738 const token_ptr = token.ptr;
17121739 if (token_ptr.id != Token.Id.Pipe) {
17131740 if (opt_ctx != OptionalCtx.Optional) {
17141741 *(try tree.errors.addOne()) = Error {
......@@ -1720,7 +1747,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
17201747 return tree;
17211748 }
17221749
1723 _ = tok_it.prev();
1750 putBackToken(&tok_it, &tree);
17241751 continue;
17251752 }
17261753
......@@ -1755,8 +1782,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
17551782
17561783
17571784 State.Expression => |opt_ctx| {
1758 const token_index = tok_it.index;
1759 const token_ptr = ??tok_it.next();
1785 const token = nextToken(&tok_it, &tree);
1786 const token_index = token.index;
1787 const token_ptr = token.ptr;
17601788 switch (token_ptr.id) {
17611789 Token.Id.Keyword_return, Token.Id.Keyword_break, Token.Id.Keyword_continue => {
17621790 const node = try createToCtxNode(arena, opt_ctx, ast.Node.ControlFlowExpression,
......@@ -1808,7 +1836,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
18081836 },
18091837 else => {
18101838 if (!try parseBlockExpr(&stack, arena, opt_ctx, token_ptr, token_index)) {
1811 _ = tok_it.prev();
1839 putBackToken(&tok_it, &tree);
18121840 stack.push(State { .UnwrapExpressionBegin = opt_ctx }) catch unreachable;
18131841 }
18141842 continue;
......@@ -1823,7 +1851,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
18231851 State.RangeExpressionEnd => |opt_ctx| {
18241852 const lhs = opt_ctx.get() ?? continue;
18251853
1826 if (eatToken(&tok_it, Token.Id.Ellipsis3)) |ellipsis3| {
1854 if (eatToken(&tok_it, &tree, Token.Id.Ellipsis3)) |ellipsis3| {
18271855 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
18281856 ast.Node.InfixOp {
18291857 .base = undefined,
......@@ -1846,8 +1874,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
18461874 State.AssignmentExpressionEnd => |opt_ctx| {
18471875 const lhs = opt_ctx.get() ?? continue;
18481876
1849 const token_index = tok_it.index;
1850 const token_ptr = ??tok_it.next();
1877 const token = nextToken(&tok_it, &tree);
1878 const token_index = token.index;
1879 const token_ptr = token.ptr;
18511880 if (tokenIdToAssignment(token_ptr.id)) |ass_id| {
18521881 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
18531882 ast.Node.InfixOp {
......@@ -1862,7 +1891,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
18621891 try stack.push(State { .Expression = OptionalCtx { .Required = &node.rhs } });
18631892 continue;
18641893 } else {
1865 _ = tok_it.prev();
1894 putBackToken(&tok_it, &tree);
18661895 continue;
18671896 }
18681897 },
......@@ -1876,8 +1905,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
18761905 State.UnwrapExpressionEnd => |opt_ctx| {
18771906 const lhs = opt_ctx.get() ?? continue;
18781907
1879 const token_index = tok_it.index;
1880 const token_ptr = ??tok_it.next();
1908 const token = nextToken(&tok_it, &tree);
1909 const token_index = token.index;
1910 const token_ptr = token.ptr;
18811911 if (tokenIdToUnwrapExpr(token_ptr.id)) |unwrap_id| {
18821912 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
18831913 ast.Node.InfixOp {
......@@ -1897,7 +1927,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
18971927 }
18981928 continue;
18991929 } else {
1900 _ = tok_it.prev();
1930 putBackToken(&tok_it, &tree);
19011931 continue;
19021932 }
19031933 },
......@@ -1911,7 +1941,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
19111941 State.BoolOrExpressionEnd => |opt_ctx| {
19121942 const lhs = opt_ctx.get() ?? continue;
19131943
1914 if (eatToken(&tok_it, Token.Id.Keyword_or)) |or_token| {
1944 if (eatToken(&tok_it, &tree, Token.Id.Keyword_or)) |or_token| {
19151945 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
19161946 ast.Node.InfixOp {
19171947 .base = undefined,
......@@ -1936,7 +1966,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
19361966 State.BoolAndExpressionEnd => |opt_ctx| {
19371967 const lhs = opt_ctx.get() ?? continue;
19381968
1939 if (eatToken(&tok_it, Token.Id.Keyword_and)) |and_token| {
1969 if (eatToken(&tok_it, &tree, Token.Id.Keyword_and)) |and_token| {
19401970 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
19411971 ast.Node.InfixOp {
19421972 .base = undefined,
......@@ -1961,8 +1991,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
19611991 State.ComparisonExpressionEnd => |opt_ctx| {
19621992 const lhs = opt_ctx.get() ?? continue;
19631993
1964 const token_index = tok_it.index;
1965 const token_ptr = ??tok_it.next();
1994 const token = nextToken(&tok_it, &tree);
1995 const token_index = token.index;
1996 const token_ptr = token.ptr;
19661997 if (tokenIdToComparison(token_ptr.id)) |comp_id| {
19671998 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
19681999 ast.Node.InfixOp {
......@@ -1977,7 +2008,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
19772008 try stack.push(State { .BinaryOrExpressionBegin = OptionalCtx { .Required = &node.rhs } });
19782009 continue;
19792010 } else {
1980 _ = tok_it.prev();
2011 putBackToken(&tok_it, &tree);
19812012 continue;
19822013 }
19832014 },
......@@ -1991,7 +2022,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
19912022 State.BinaryOrExpressionEnd => |opt_ctx| {
19922023 const lhs = opt_ctx.get() ?? continue;
19932024
1994 if (eatToken(&tok_it, Token.Id.Pipe)) |pipe| {
2025 if (eatToken(&tok_it, &tree, Token.Id.Pipe)) |pipe| {
19952026 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
19962027 ast.Node.InfixOp {
19972028 .base = undefined,
......@@ -2016,7 +2047,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
20162047 State.BinaryXorExpressionEnd => |opt_ctx| {
20172048 const lhs = opt_ctx.get() ?? continue;
20182049
2019 if (eatToken(&tok_it, Token.Id.Caret)) |caret| {
2050 if (eatToken(&tok_it, &tree, Token.Id.Caret)) |caret| {
20202051 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
20212052 ast.Node.InfixOp {
20222053 .base = undefined,
......@@ -2041,7 +2072,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
20412072 State.BinaryAndExpressionEnd => |opt_ctx| {
20422073 const lhs = opt_ctx.get() ?? continue;
20432074
2044 if (eatToken(&tok_it, Token.Id.Ampersand)) |ampersand| {
2075 if (eatToken(&tok_it, &tree, Token.Id.Ampersand)) |ampersand| {
20452076 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
20462077 ast.Node.InfixOp {
20472078 .base = undefined,
......@@ -2066,8 +2097,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
20662097 State.BitShiftExpressionEnd => |opt_ctx| {
20672098 const lhs = opt_ctx.get() ?? continue;
20682099
2069 const token_index = tok_it.index;
2070 const token_ptr = ??tok_it.next();
2100 const token = nextToken(&tok_it, &tree);
2101 const token_index = token.index;
2102 const token_ptr = token.ptr;
20712103 if (tokenIdToBitShift(token_ptr.id)) |bitshift_id| {
20722104 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
20732105 ast.Node.InfixOp {
......@@ -2082,7 +2114,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
20822114 try stack.push(State { .AdditionExpressionBegin = OptionalCtx { .Required = &node.rhs } });
20832115 continue;
20842116 } else {
2085 _ = tok_it.prev();
2117 putBackToken(&tok_it, &tree);
20862118 continue;
20872119 }
20882120 },
......@@ -2096,8 +2128,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
20962128 State.AdditionExpressionEnd => |opt_ctx| {
20972129 const lhs = opt_ctx.get() ?? continue;
20982130
2099 const token_index = tok_it.index;
2100 const token_ptr = ??tok_it.next();
2131 const token = nextToken(&tok_it, &tree);
2132 const token_index = token.index;
2133 const token_ptr = token.ptr;
21012134 if (tokenIdToAddition(token_ptr.id)) |add_id| {
21022135 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
21032136 ast.Node.InfixOp {
......@@ -2112,7 +2145,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
21122145 try stack.push(State { .MultiplyExpressionBegin = OptionalCtx { .Required = &node.rhs } });
21132146 continue;
21142147 } else {
2115 _ = tok_it.prev();
2148 putBackToken(&tok_it, &tree);
21162149 continue;
21172150 }
21182151 },
......@@ -2126,8 +2159,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
21262159 State.MultiplyExpressionEnd => |opt_ctx| {
21272160 const lhs = opt_ctx.get() ?? continue;
21282161
2129 const token_index = tok_it.index;
2130 const token_ptr = ??tok_it.next();
2162 const token = nextToken(&tok_it, &tree);
2163 const token_index = token.index;
2164 const token_ptr = token.ptr;
21312165 if (tokenIdToMultiply(token_ptr.id)) |mult_id| {
21322166 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
21332167 ast.Node.InfixOp {
......@@ -2142,7 +2176,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
21422176 try stack.push(State { .CurlySuffixExpressionBegin = OptionalCtx { .Required = &node.rhs } });
21432177 continue;
21442178 } else {
2145 _ = tok_it.prev();
2179 putBackToken(&tok_it, &tree);
21462180 continue;
21472181 }
21482182 },
......@@ -2210,7 +2244,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
22102244 State.TypeExprEnd => |opt_ctx| {
22112245 const lhs = opt_ctx.get() ?? continue;
22122246
2213 if (eatToken(&tok_it, Token.Id.Bang)) |bang| {
2247 if (eatToken(&tok_it, &tree, Token.Id.Bang)) |bang| {
22142248 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
22152249 ast.Node.InfixOp {
22162250 .base = undefined,
......@@ -2227,8 +2261,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
22272261 },
22282262
22292263 State.PrefixOpExpression => |opt_ctx| {
2230 const token_index = tok_it.index;
2231 const token_ptr = ??tok_it.next();
2264 const token = nextToken(&tok_it, &tree);
2265 const token_index = token.index;
2266 const token_ptr = token.ptr;
22322267 if (tokenIdToPrefixOp(token_ptr.id)) |prefix_id| {
22332268 var node = try createToCtxNode(arena, opt_ctx, ast.Node.PrefixOp,
22342269 ast.Node.PrefixOp {
......@@ -2259,14 +2294,14 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
22592294 }
22602295 continue;
22612296 } else {
2262 _ = tok_it.prev();
2297 putBackToken(&tok_it, &tree);
22632298 stack.push(State { .SuffixOpExpressionBegin = opt_ctx }) catch unreachable;
22642299 continue;
22652300 }
22662301 },
22672302
22682303 State.SuffixOpExpressionBegin => |opt_ctx| {
2269 if (eatToken(&tok_it, Token.Id.Keyword_async)) |async_token| {
2304 if (eatToken(&tok_it, &tree, Token.Id.Keyword_async)) |async_token| {
22702305 const async_node = try createNode(arena, ast.Node.AsyncAttribute,
22712306 ast.Node.AsyncAttribute {
22722307 .base = undefined,
......@@ -2295,8 +2330,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
22952330 State.SuffixOpExpressionEnd => |opt_ctx| {
22962331 const lhs = opt_ctx.get() ?? continue;
22972332
2298 const token_index = tok_it.index;
2299 const token_ptr = ??tok_it.next();
2333 const token = nextToken(&tok_it, &tree);
2334 const token_index = token.index;
2335 const token_ptr = token.ptr;
23002336 switch (token_ptr.id) {
23012337 Token.Id.LParen => {
23022338 const node = try createToCtxNode(arena, opt_ctx, ast.Node.SuffixOp,
......@@ -2353,50 +2389,49 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
23532389 continue;
23542390 },
23552391 else => {
2356 _ = tok_it.prev();
2392 putBackToken(&tok_it, &tree);
23572393 continue;
23582394 },
23592395 }
23602396 },
23612397
23622398 State.PrimaryExpression => |opt_ctx| {
2363 const token_index = tok_it.index;
2364 const token_ptr = ??tok_it.next();
2365 switch (token_ptr.id) {
2399 const token = nextToken(&tok_it, &tree);
2400 switch (token.ptr.id) {
23662401 Token.Id.IntegerLiteral => {
2367 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.StringLiteral, token_index);
2402 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.StringLiteral, token.index);
23682403 continue;
23692404 },
23702405 Token.Id.FloatLiteral => {
2371 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.FloatLiteral, token_index);
2406 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.FloatLiteral, token.index);
23722407 continue;
23732408 },
23742409 Token.Id.CharLiteral => {
2375 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.CharLiteral, token_index);
2410 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.CharLiteral, token.index);
23762411 continue;
23772412 },
23782413 Token.Id.Keyword_undefined => {
2379 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.UndefinedLiteral, token_index);
2414 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.UndefinedLiteral, token.index);
23802415 continue;
23812416 },
23822417 Token.Id.Keyword_true, Token.Id.Keyword_false => {
2383 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.BoolLiteral, token_index);
2418 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.BoolLiteral, token.index);
23842419 continue;
23852420 },
23862421 Token.Id.Keyword_null => {
2387 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.NullLiteral, token_index);
2422 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.NullLiteral, token.index);
23882423 continue;
23892424 },
23902425 Token.Id.Keyword_this => {
2391 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.ThisLiteral, token_index);
2426 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.ThisLiteral, token.index);
23922427 continue;
23932428 },
23942429 Token.Id.Keyword_var => {
2395 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.VarType, token_index);
2430 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.VarType, token.index);
23962431 continue;
23972432 },
23982433 Token.Id.Keyword_unreachable => {
2399 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.Unreachable, token_index);
2434 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.Unreachable, token.index);
24002435 continue;
24012436 },
24022437 Token.Id.Keyword_promise => {
......@@ -2404,14 +2439,15 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
24042439 .base = ast.Node {
24052440 .id = ast.Node.Id.PromiseType,
24062441 },
2407 .promise_token = token_index,
2442 .promise_token = token.index,
24082443 .result = null,
24092444 });
24102445 opt_ctx.store(&node.base);
2411 const next_token_index = tok_it.index;
2412 const next_token_ptr = ??tok_it.next();
2446 const next_token = nextToken(&tok_it, &tree);
2447 const next_token_index = next_token.index;
2448 const next_token_ptr = next_token.ptr;
24132449 if (next_token_ptr.id != Token.Id.Arrow) {
2414 _ = tok_it.prev();
2450 putBackToken(&tok_it, &tree);
24152451 continue;
24162452 }
24172453 node.result = ast.Node.PromiseType.Result {
......@@ -2423,14 +2459,14 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
24232459 continue;
24242460 },
24252461 Token.Id.StringLiteral, Token.Id.MultilineStringLiteralLine => {
2426 opt_ctx.store((try parseStringLiteral(arena, &tok_it, token_ptr, token_index)) ?? unreachable);
2462 opt_ctx.store((try parseStringLiteral(arena, &tok_it, token.ptr, token.index, &tree)) ?? unreachable);
24272463 continue;
24282464 },
24292465 Token.Id.LParen => {
24302466 const node = try createToCtxNode(arena, opt_ctx, ast.Node.GroupedExpression,
24312467 ast.Node.GroupedExpression {
24322468 .base = undefined,
2433 .lparen = token_index,
2469 .lparen = token.index,
24342470 .expr = undefined,
24352471 .rparen = undefined,
24362472 }
......@@ -2448,7 +2484,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
24482484 const node = try createToCtxNode(arena, opt_ctx, ast.Node.BuiltinCall,
24492485 ast.Node.BuiltinCall {
24502486 .base = undefined,
2451 .builtin_token = token_index,
2487 .builtin_token = token.index,
24522488 .params = ast.Node.BuiltinCall.ParamList.init(arena),
24532489 .rparen_token = undefined,
24542490 }
......@@ -2467,7 +2503,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
24672503 const node = try createToCtxNode(arena, opt_ctx, ast.Node.PrefixOp,
24682504 ast.Node.PrefixOp {
24692505 .base = undefined,
2470 .op_token = token_index,
2506 .op_token = token.index,
24712507 .op = undefined,
24722508 .rhs = undefined,
24732509 }
......@@ -2478,7 +2514,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
24782514 Token.Id.Keyword_error => {
24792515 stack.push(State {
24802516 .ErrorTypeOrSetDecl = ErrorTypeOrSetDeclCtx {
2481 .error_token = token_index,
2517 .error_token = token.index,
24822518 .opt_ctx = opt_ctx
24832519 }
24842520 }) catch unreachable;
......@@ -2488,7 +2524,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
24882524 stack.push(State {
24892525 .ContainerKind = ContainerKindCtx {
24902526 .opt_ctx = opt_ctx,
2491 .ltoken = token_index,
2527 .ltoken = token.index,
24922528 .layout = ast.Node.ContainerDecl.Layout.Packed,
24932529 },
24942530 }) catch unreachable;
......@@ -2498,18 +2534,18 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
24982534 stack.push(State {
24992535 .ExternType = ExternTypeCtx {
25002536 .opt_ctx = opt_ctx,
2501 .extern_token = token_index,
2537 .extern_token = token.index,
25022538 .comments = null,
25032539 },
25042540 }) catch unreachable;
25052541 continue;
25062542 },
25072543 Token.Id.Keyword_struct, Token.Id.Keyword_union, Token.Id.Keyword_enum => {
2508 _ = tok_it.prev();
2544 putBackToken(&tok_it, &tree);
25092545 stack.push(State {
25102546 .ContainerKind = ContainerKindCtx {
25112547 .opt_ctx = opt_ctx,
2512 .ltoken = token_index,
2548 .ltoken = token.index,
25132549 .layout = ast.Node.ContainerDecl.Layout.Auto,
25142550 },
25152551 }) catch unreachable;
......@@ -2518,7 +2554,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
25182554 Token.Id.Identifier => {
25192555 stack.push(State {
25202556 .MaybeLabeledExpression = MaybeLabeledExpressionCtx {
2521 .label = token_index,
2557 .label = token.index,
25222558 .opt_ctx = opt_ctx
25232559 }
25242560 }) catch unreachable;
......@@ -2532,7 +2568,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
25322568 .doc_comments = null,
25332569 .visib_token = null,
25342570 .name_token = null,
2535 .fn_token = token_index,
2571 .fn_token = token.index,
25362572 .params = ast.Node.FnProto.ParamList.init(arena),
25372573 .return_type = undefined,
25382574 .var_args_token = null,
......@@ -2560,7 +2596,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
25602596 .return_type = undefined,
25612597 .var_args_token = null,
25622598 .extern_export_inline_token = null,
2563 .cc_token = token_index,
2599 .cc_token = token.index,
25642600 .async_attr = null,
25652601 .body_node = null,
25662602 .lib_name = null,
......@@ -2580,7 +2616,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
25802616 const node = try createToCtxNode(arena, opt_ctx, ast.Node.Asm,
25812617 ast.Node.Asm {
25822618 .base = undefined,
2583 .asm_token = token_index,
2619 .asm_token = token.index,
25842620 .volatile_token = null,
25852621 .template = undefined,
25862622 .outputs = ast.Node.Asm.OutputList.init(arena),
......@@ -2614,18 +2650,18 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
26142650 stack.push(State {
26152651 .Inline = InlineCtx {
26162652 .label = null,
2617 .inline_token = token_index,
2653 .inline_token = token.index,
26182654 .opt_ctx = opt_ctx,
26192655 }
26202656 }) catch unreachable;
26212657 continue;
26222658 },
26232659 else => {
2624 if (!try parseBlockExpr(&stack, arena, opt_ctx, token_ptr, token_index)) {
2625 _ = tok_it.prev();
2660 if (!try parseBlockExpr(&stack, arena, opt_ctx, token.ptr, token.index)) {
2661 putBackToken(&tok_it, &tree);
26262662 if (opt_ctx != OptionalCtx.Optional) {
26272663 *(try tree.errors.addOne()) = Error {
2628 .ExpectedPrimaryExpr = Error.ExpectedPrimaryExpr { .token = token_index },
2664 .ExpectedPrimaryExpr = Error.ExpectedPrimaryExpr { .token = token.index },
26292665 };
26302666 return tree;
26312667 }
......@@ -2637,7 +2673,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
26372673
26382674
26392675 State.ErrorTypeOrSetDecl => |ctx| {
2640 if (eatToken(&tok_it, Token.Id.LBrace) == null) {
2676 if (eatToken(&tok_it, &tree, Token.Id.LBrace) == null) {
26412677 _ = try createToCtxLiteral(arena, ctx.opt_ctx, ast.Node.ErrorType, ctx.error_token);
26422678 continue;
26432679 }
......@@ -2661,11 +2697,12 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
26612697 continue;
26622698 },
26632699 State.StringLiteral => |opt_ctx| {
2664 const token_index = tok_it.index;
2665 const token_ptr = ??tok_it.next();
2700 const token = nextToken(&tok_it, &tree);
2701 const token_index = token.index;
2702 const token_ptr = token.ptr;
26662703 opt_ctx.store(
2667 (try parseStringLiteral(arena, &tok_it, token_ptr, token_index)) ?? {
2668 _ = tok_it.prev();
2704 (try parseStringLiteral(arena, &tok_it, token_ptr, token_index, &tree)) ?? {
2705 putBackToken(&tok_it, &tree);
26692706 if (opt_ctx != OptionalCtx.Optional) {
26702707 *(try tree.errors.addOne()) = Error {
26712708 .ExpectedPrimaryExpr = Error.ExpectedPrimaryExpr { .token = token_index },
......@@ -2679,14 +2716,15 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
26792716 },
26802717
26812718 State.Identifier => |opt_ctx| {
2682 if (eatToken(&tok_it, Token.Id.Identifier)) |ident_token| {
2719 if (eatToken(&tok_it, &tree, Token.Id.Identifier)) |ident_token| {
26832720 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.Identifier, ident_token);
26842721 continue;
26852722 }
26862723
26872724 if (opt_ctx != OptionalCtx.Optional) {
2688 const token_index = tok_it.index;
2689 const token_ptr = ??tok_it.next();
2725 const token = nextToken(&tok_it, &tree);
2726 const token_index = token.index;
2727 const token_ptr = token.ptr;
26902728 *(try tree.errors.addOne()) = Error {
26912729 .ExpectedToken = Error.ExpectedToken {
26922730 .token = token_index,
......@@ -2698,9 +2736,10 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
26982736 },
26992737
27002738 State.ErrorTag => |node_ptr| {
2701 const comments = try eatDocComments(arena, &tok_it);
2702 const ident_token_index = tok_it.index;
2703 const ident_token_ptr = ??tok_it.next();
2739 const comments = try eatDocComments(arena, &tok_it, &tree);
2740 const ident_token = nextToken(&tok_it, &tree);
2741 const ident_token_index = ident_token.index;
2742 const ident_token_ptr = ident_token.ptr;
27042743 if (ident_token_ptr.id != Token.Id.Identifier) {
27052744 *(try tree.errors.addOne()) = Error {
27062745 .ExpectedToken = Error.ExpectedToken {
......@@ -2723,8 +2762,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
27232762 },
27242763
27252764 State.ExpectToken => |token_id| {
2726 const token_index = tok_it.index;
2727 const token_ptr = ??tok_it.next();
2765 const token = nextToken(&tok_it, &tree);
2766 const token_index = token.index;
2767 const token_ptr = token.ptr;
27282768 if (token_ptr.id != token_id) {
27292769 *(try tree.errors.addOne()) = Error {
27302770 .ExpectedToken = Error.ExpectedToken {
......@@ -2737,8 +2777,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
27372777 continue;
27382778 },
27392779 State.ExpectTokenSave => |expect_token_save| {
2740 const token_index = tok_it.index;
2741 const token_ptr = ??tok_it.next();
2780 const token = nextToken(&tok_it, &tree);
2781 const token_index = token.index;
2782 const token_ptr = token.ptr;
27422783 if (token_ptr.id != expect_token_save.id) {
27432784 *(try tree.errors.addOne()) = Error {
27442785 .ExpectedToken = Error.ExpectedToken {
......@@ -2752,7 +2793,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
27522793 continue;
27532794 },
27542795 State.IfToken => |token_id| {
2755 if (eatToken(&tok_it, token_id)) |_| {
2796 if (eatToken(&tok_it, &tree, token_id)) |_| {
27562797 continue;
27572798 }
27582799
......@@ -2760,7 +2801,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
27602801 continue;
27612802 },
27622803 State.IfTokenSave => |if_token_save| {
2763 if (eatToken(&tok_it, if_token_save.id)) |token_index| {
2804 if (eatToken(&tok_it, &tree, if_token_save.id)) |token_index| {
27642805 *if_token_save.ptr = token_index;
27652806 continue;
27662807 }
......@@ -2769,7 +2810,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
27692810 continue;
27702811 },
27712812 State.OptionalTokenSave => |optional_token_save| {
2772 if (eatToken(&tok_it, optional_token_save.id)) |token_index| {
2813 if (eatToken(&tok_it, &tree, optional_token_save.id)) |token_index| {
27732814 *optional_token_save.ptr = token_index;
27742815 continue;
27752816 }
......@@ -3043,10 +3084,10 @@ const State = union(enum) {
30433084 OptionalTokenSave: OptionalTokenSave,
30443085};
30453086
3046fn eatDocComments(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterator) !?&ast.Node.DocComment {
3087fn eatDocComments(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterator, tree: &ast.Tree) !?&ast.Node.DocComment {
30473088 var result: ?&ast.Node.DocComment = null;
30483089 while (true) {
3049 if (eatToken(tok_it, Token.Id.DocComment)) |line_comment| {
3090 if (eatToken(tok_it, tree, Token.Id.DocComment)) |line_comment| {
30503091 const node = blk: {
30513092 if (result) |comment_node| {
30523093 break :blk comment_node;
......@@ -3069,8 +3110,8 @@ fn eatDocComments(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterator) !
30693110 return result;
30703111}
30713112
3072fn eatLineComment(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterator) !?&ast.Node.LineComment {
3073 const token = eatToken(tok_it, Token.Id.LineComment) ?? return null;
3113fn eatLineComment(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterator, tree: &ast.Tree) !?&ast.Node.LineComment {
3114 const token = eatToken(tok_it, tree, Token.Id.LineComment) ?? return null;
30743115 return try arena.construct(ast.Node.LineComment {
30753116 .base = ast.Node {
30763117 .id = ast.Node.Id.LineComment,
......@@ -3080,7 +3121,7 @@ fn eatLineComment(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterator) !
30803121}
30813122
30823123fn parseStringLiteral(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterator,
3083 token_ptr: &const Token, token_index: TokenIndex) !?&ast.Node
3124 token_ptr: &const Token, token_index: TokenIndex, tree: &ast.Tree) !?&ast.Node
30843125{
30853126 switch (token_ptr.id) {
30863127 Token.Id.StringLiteral => {
......@@ -3093,10 +3134,11 @@ fn parseStringLiteral(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterato
30933134 });
30943135 try node.lines.push(token_index);
30953136 while (true) {
3096 const multiline_str_index = tok_it.index;
3097 const multiline_str_ptr = ??tok_it.next();
3137 const multiline_str = nextToken(tok_it, tree);
3138 const multiline_str_index = multiline_str.index;
3139 const multiline_str_ptr = multiline_str.ptr;
30983140 if (multiline_str_ptr.id != Token.Id.MultilineStringLiteralLine) {
3099 _ = tok_it.prev();
3141 putBackToken(tok_it, tree);
31003142 break;
31013143 }
31023144
......@@ -3230,9 +3272,10 @@ const ExpectCommaOrEndResult = union(enum) {
32303272 parse_error: Error,
32313273};
32323274
3233fn expectCommaOrEnd(tok_it: &ast.Tree.TokenList.Iterator, end: @TagType(Token.Id)) ExpectCommaOrEndResult {
3234 const token_index = tok_it.index;
3235 const token_ptr = ??tok_it.next();
3275fn expectCommaOrEnd(tok_it: &ast.Tree.TokenList.Iterator, tree: &ast.Tree, end: @TagType(Token.Id)) ExpectCommaOrEndResult {
3276 const token = nextToken(tok_it, tree);
3277 const token_index = token.index;
3278 const token_ptr = token.ptr;
32363279 switch (token_ptr.id) {
32373280 Token.Id.Comma => return ExpectCommaOrEndResult { .end_token = null},
32383281 else => {
......@@ -3385,16 +3428,45 @@ fn createToCtxLiteral(arena: &mem.Allocator, opt_ctx: &const OptionalCtx, compti
33853428 return node;
33863429}
33873430
3388fn eatToken(tok_it: &ast.Tree.TokenList.Iterator, id: @TagType(Token.Id)) ?TokenIndex {
3389 const token_index = tok_it.index;
3390 const token_ptr = ??tok_it.next();
3391 if (token_ptr.id == id)
3392 return token_index;
3431fn eatToken(tok_it: &ast.Tree.TokenList.Iterator, tree: &ast.Tree, id: @TagType(Token.Id)) ?TokenIndex {
3432 const token = nextToken(tok_it, tree);
3433
3434 if (token.ptr.id == id)
3435 return token.index;
33933436
3394 _ = tok_it.prev();
3437 putBackToken(tok_it, tree);
33953438 return null;
33963439}
33973440
3441fn nextToken(tok_it: &ast.Tree.TokenList.Iterator, tree: &ast.Tree) AnnotatedToken {
3442 const result = AnnotatedToken {
3443 .index = tok_it.index,
3444 .ptr = ??tok_it.next(),
3445 };
3446 // possibly skip a following same line token
3447 const token = tok_it.next() ?? return result;
3448 if (token.id != Token.Id.LineComment) {
3449 putBackToken(tok_it, tree);
3450 return result;
3451 }
3452 const loc = tree.tokenLocationPtr(result.ptr.end, token);
3453 if (loc.line != 0) {
3454 putBackToken(tok_it, tree);
3455 }
3456 return result;
3457}
3458
3459fn putBackToken(tok_it: &ast.Tree.TokenList.Iterator, tree: &ast.Tree) void {
3460 const prev_tok = ??tok_it.prev();
3461 if (prev_tok.id == Token.Id.LineComment) {
3462 const minus2_tok = tok_it.prev() ?? return;
3463 const loc = tree.tokenLocationPtr(minus2_tok.end, prev_tok);
3464 if (loc.line != 0) {
3465 _ = tok_it.next();
3466 }
3467 }
3468}
3469
33983470const RenderAstFrame = struct {
33993471 node: &ast.Node,
34003472 indent: usize,
std/zig/parser_test.zig+54-54
......@@ -1,3 +1,48 @@
1//test "zig fmt: same-line comment after a statement" {
2// try testCanonical(
3// \\test "" {
4// \\ a = b;
5// \\ debug.assert(H.digest_size <= H.block_size); // HMAC makes this assumption
6// \\ a = b;
7// \\}
8// \\
9// );
10//}
11//
12//test "zig fmt: same-line comment after var decl in struct" {
13// try testCanonical(
14// \\pub const vfs_cap_data = extern struct {
15// \\ const Data = struct {}; // when on disk.
16// \\};
17// \\
18// );
19//}
20//
21//test "zig fmt: same-line comment after field decl" {
22// try testCanonical(
23// \\pub const dirent = extern struct {
24// \\ d_name: u8,
25// \\ d_name: u8, // comment 1
26// \\ d_name: u8,
27// \\ d_name: u8, // comment 2
28// \\ d_name: u8,
29// \\};
30// \\
31// );
32//}
33//
34//test "zig fmt: same-line comment after switch prong" {
35// try testCanonical(
36// \\test "" {
37// \\ switch (err) {
38// \\ error.PathAlreadyExists => {}, // comment 2
39// \\ else => return err, // comment 1
40// \\ }
41// \\}
42// \\
43// );
44//}
45//
146//test "zig fmt: same-line comment after non-block if expression" {
247// try testCanonical(
348// \\comptime {
......@@ -7,6 +52,15 @@
752// \\
853// );
954//}
55//
56//test "zig fmt: same-line comment on comptime expression" {
57// try testCanonical(
58// \\test "" {
59// \\ comptime assert(@typeId(T) == builtin.TypeId.Int); // must pass an integer to absInt
60// \\}
61// \\
62// );
63//}
1064
1165test "zig fmt: switch with empty body" {
1266 try testCanonical(
......@@ -17,15 +71,6 @@ test "zig fmt: switch with empty body" {
1771 );
1872}
1973
20//test "zig fmt: same-line comment on comptime expression" {
21// try testCanonical(
22// \\test "" {
23// \\ comptime assert(@typeId(T) == builtin.TypeId.Int); // must pass an integer to absInt
24// \\}
25// \\
26// );
27//}
28
2974test "zig fmt: float literal with exponent" {
3075 try testCanonical(
3176 \\pub const f64_true_min = 4.94065645841246544177e-324;
......@@ -152,18 +197,6 @@ test "zig fmt: comments before switch prong" {
152197 );
153198}
154199
155//test "zig fmt: same-line comment after switch prong" {
156// try testCanonical(
157// \\test "" {
158// \\ switch (err) {
159// \\ error.PathAlreadyExists => {}, // comment 2
160// \\ else => return err, // comment 1
161// \\ }
162// \\}
163// \\
164// );
165//}
166
167200test "zig fmt: comments before var decl in struct" {
168201 try testCanonical(
169202 \\pub const vfs_cap_data = extern struct {
......@@ -189,28 +222,6 @@ test "zig fmt: comments before var decl in struct" {
189222 );
190223}
191224
192//test "zig fmt: same-line comment after var decl in struct" {
193// try testCanonical(
194// \\pub const vfs_cap_data = extern struct {
195// \\ const Data = struct {}; // when on disk.
196// \\};
197// \\
198// );
199//}
200//
201//test "zig fmt: same-line comment after field decl" {
202// try testCanonical(
203// \\pub const dirent = extern struct {
204// \\ d_name: u8,
205// \\ d_name: u8, // comment 1
206// \\ d_name: u8,
207// \\ d_name: u8, // comment 2
208// \\ d_name: u8,
209// \\};
210// \\
211// );
212//}
213
214225test "zig fmt: array literal with 1 item on 1 line" {
215226 try testCanonical(
216227 \\var s = []const u64{0} ** 25;
......@@ -218,17 +229,6 @@ test "zig fmt: array literal with 1 item on 1 line" {
218229 );
219230}
220231
221//test "zig fmt: same-line comment after a statement" {
222// try testCanonical(
223// \\test "" {
224// \\ a = b;
225// \\ debug.assert(H.digest_size <= H.block_size); // HMAC makes this assumption
226// \\ a = b;
227// \\}
228// \\
229// );
230//}
231
232232test "zig fmt: comments before global variables" {
233233 try testCanonical(
234234 \\/// Foo copies keys and values before they go into the map, and