authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-11 13:38:06+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-11 13:38:06+02:00
log4b0556ebd4bdc2f3c05e85f3c003b6fc89f7ac0f
tree331b0669fb130a70cf49c69f7294ccf96ea761f8
parent6fb5ab1b523c58350321fdcbe7ba921300bef8af

std.zig.parser can now parse `std/heap.zig`:

related: #909 * Struct fields can now be pub * Parsing of double deref now works * Block expressions now have the right precedence

2 files changed, 180 insertions(+), 110 deletions(-)

std/zig/ast.zig+2
......@@ -360,6 +360,7 @@ pub const NodeContainerDecl = struct {
360360
361361pub const NodeStructField = struct {
362362 base: Node,
363 visib_token: ?Token,
363364 name_token: Token,
364365 type_expr: &Node,
365366
......@@ -373,6 +374,7 @@ pub const NodeStructField = struct {
373374 }
374375
375376 pub fn firstToken(self: &NodeStructField) Token {
377 if (self.visib_token) |visib_token| return visib_token;
376378 return self.name_token;
377379 }
378380
std/zig/parser.zig+178-110
......@@ -669,6 +669,7 @@ pub const Parser = struct {
669669 const node = try self.createAttachNode(arena, &container_decl.fields_and_decls, ast.NodeStructField,
670670 ast.NodeStructField {
671671 .base = undefined,
672 .visib_token = null,
672673 .name_token = token,
673674 .type_expr = undefined,
674675 }
......@@ -721,7 +722,42 @@ pub const Parser = struct {
721722 },
722723 }
723724 },
724 Token.Id.Keyword_pub, Token.Id.Keyword_export => {
725 Token.Id.Keyword_pub => {
726 if (self.eatToken(Token.Id.Identifier)) |identifier| {
727 switch (container_decl.kind) {
728 ast.NodeContainerDecl.Kind.Struct => {
729 const node = try self.createAttachNode(arena, &container_decl.fields_and_decls, ast.NodeStructField,
730 ast.NodeStructField {
731 .base = undefined,
732 .visib_token = token,
733 .name_token = identifier,
734 .type_expr = undefined,
735 }
736 );
737
738 stack.append(State { .FieldListCommaOrEnd = container_decl }) catch unreachable;
739 try stack.append(State { .Expression = DestPtr { .Field = &node.type_expr } });
740 try stack.append(State { .ExpectToken = Token.Id.Colon });
741 continue;
742 },
743 else => {
744 self.putBackToken(identifier);
745 }
746 }
747 }
748
749 stack.append(State{ .ContainerDecl = container_decl }) catch unreachable;
750 try stack.append(State {
751 .TopLevelExtern = TopLevelDeclCtx {
752 .decls = &container_decl.fields_and_decls,
753 .visib_token = token,
754 .extern_token = null,
755 .lib_name = null,
756 }
757 });
758 continue;
759 },
760 Token.Id.Keyword_export => {
725761 stack.append(State{ .ContainerDecl = container_decl }) catch unreachable;
726762 try stack.append(State {
727763 .TopLevelExtern = TopLevelDeclCtx {
......@@ -864,111 +900,11 @@ pub const Parser = struct {
864900 stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable;
865901 continue;
866902 },
867 Token.Id.Keyword_suspend => {
868 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSuspend,
869 ast.NodeSuspend {
870 .base = undefined,
871 .suspend_token = token,
872 .payload = null,
873 .body = null,
874 }
875 );
876
877 stack.append(State { .SuspendBody = node }) catch unreachable;
878 try stack.append(State { .Payload = &node.payload });
879 continue;
880 },
881 Token.Id.Keyword_if => {
882 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeIf,
883 ast.NodeIf {
884 .base = undefined,
885 .if_token = token,
886 .condition = undefined,
887 .payload = null,
888 .body = undefined,
889 .@"else" = null,
890 }
891 );
892
893 stack.append(State { .Else = &node.@"else" }) catch unreachable;
894 try stack.append(State { .Expression = DestPtr { .Field = &node.body } });
895 try stack.append(State { .PointerPayload = &node.payload });
896 try stack.append(State { .ExpectToken = Token.Id.RParen });
897 try stack.append(State { .Expression = DestPtr { .Field = &node.condition } });
898 try stack.append(State { .ExpectToken = Token.Id.LParen });
899 continue;
900 },
901 Token.Id.Keyword_while => {
902 stack.append(State {
903 .While = LoopCtx {
904 .label = null,
905 .inline_token = null,
906 .loop_token = token,
907 .dest_ptr = dest_ptr,
908 }
909 }) catch unreachable;
910 continue;
911 },
912 Token.Id.Keyword_for => {
913 stack.append(State {
914 .For = LoopCtx {
915 .label = null,
916 .inline_token = null,
917 .loop_token = token,
918 .dest_ptr = dest_ptr,
919 }
920 }) catch unreachable;
921 continue;
922 },
923 Token.Id.Keyword_switch => {
924 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSwitch,
925 ast.NodeSwitch {
926 .base = undefined,
927 .switch_token = token,
928 .expr = undefined,
929 .cases = ArrayList(&ast.NodeSwitchCase).init(arena),
930 .rbrace = undefined,
931 }
932 );
933
934 stack.append(State {
935 .SwitchCaseOrEnd = ListSave(&ast.NodeSwitchCase) {
936 .list = &node.cases,
937 .ptr = &node.rbrace,
938 },
939 }) catch unreachable;
940 try stack.append(State { .ExpectToken = Token.Id.LBrace });
941 try stack.append(State { .ExpectToken = Token.Id.RParen });
942 try stack.append(State { .Expression = DestPtr { .Field = &node.expr } });
943 try stack.append(State { .ExpectToken = Token.Id.LParen });
944 },
945 Token.Id.Keyword_comptime => {
946 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeComptime,
947 ast.NodeComptime {
948 .base = undefined,
949 .comptime_token = token,
950 .expr = undefined,
951 }
952 );
953 try stack.append(State { .Expression = DestPtr { .Field = &node.expr } });
954 continue;
955 },
956 Token.Id.LBrace => {
957 const block = try self.createToDestNode(arena, dest_ptr, ast.NodeBlock,
958 ast.NodeBlock {
959 .base = undefined,
960 .label = null,
961 .lbrace = token,
962 .statements = ArrayList(&ast.Node).init(arena),
963 .rbrace = undefined,
964 }
965 );
966 stack.append(State { .Block = block }) catch unreachable;
967 continue;
968 },
969903 else => {
970 self.putBackToken(token);
971 stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable;
904 if (!try self.parseBlockExpr(&stack, arena, dest_ptr, token)) {
905 self.putBackToken(token);
906 stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable;
907 }
972908 continue;
973909 }
974910 }
......@@ -1407,7 +1343,7 @@ pub const Parser = struct {
14071343 State.PrefixOpExpression => |dest_ptr| {
14081344 const token = self.getNextToken();
14091345 if (tokenIdToPrefixOp(token.id)) |prefix_id| {
1410 const node = try self.createToDestNode(arena, dest_ptr, ast.NodePrefixOp,
1346 var node = try self.createToDestNode(arena, dest_ptr, ast.NodePrefixOp,
14111347 ast.NodePrefixOp {
14121348 .base = undefined,
14131349 .op_token = token,
......@@ -1415,6 +1351,20 @@ pub const Parser = struct {
14151351 .rhs = undefined,
14161352 }
14171353 );
1354
1355 if (token.id == Token.Id.AsteriskAsterisk) {
1356 const child = try self.createNode(arena, ast.NodePrefixOp,
1357 ast.NodePrefixOp {
1358 .base = undefined,
1359 .op_token = token,
1360 .op = prefix_id,
1361 .rhs = undefined,
1362 }
1363 );
1364 node.rhs = &child.base;
1365 node = child;
1366 }
1367
14181368 stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.rhs } }) catch unreachable;
14191369 if (node.op == ast.NodePrefixOp.PrefixOp.AddrOf) {
14201370 try stack.append(State { .AddrOfModifiers = &node.op.AddrOf });
......@@ -1871,7 +1821,9 @@ pub const Parser = struct {
18711821 continue;
18721822 },
18731823 else => {
1874 try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id));
1824 if (!try self.parseBlockExpr(&stack, arena, dest_ptr, token)) {
1825 try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id));
1826 }
18751827 continue;
18761828 }
18771829 }
......@@ -2790,6 +2742,117 @@ pub const Parser = struct {
27902742 }
27912743 }
27922744
2745 fn parseBlockExpr(self: &Parser, stack: &ArrayList(State), arena: &mem.Allocator, dest_ptr: &const DestPtr, token: &const Token) !bool {
2746 switch (token.id) {
2747 Token.Id.Keyword_suspend => {
2748 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSuspend,
2749 ast.NodeSuspend {
2750 .base = undefined,
2751 .suspend_token = *token,
2752 .payload = null,
2753 .body = null,
2754 }
2755 );
2756
2757 stack.append(State { .SuspendBody = node }) catch unreachable;
2758 try stack.append(State { .Payload = &node.payload });
2759 return true;
2760 },
2761 Token.Id.Keyword_if => {
2762 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeIf,
2763 ast.NodeIf {
2764 .base = undefined,
2765 .if_token = *token,
2766 .condition = undefined,
2767 .payload = null,
2768 .body = undefined,
2769 .@"else" = null,
2770 }
2771 );
2772
2773 stack.append(State { .Else = &node.@"else" }) catch unreachable;
2774 try stack.append(State { .Expression = DestPtr { .Field = &node.body } });
2775 try stack.append(State { .PointerPayload = &node.payload });
2776 try stack.append(State { .ExpectToken = Token.Id.RParen });
2777 try stack.append(State { .Expression = DestPtr { .Field = &node.condition } });
2778 try stack.append(State { .ExpectToken = Token.Id.LParen });
2779 return true;
2780 },
2781 Token.Id.Keyword_while => {
2782 stack.append(State {
2783 .While = LoopCtx {
2784 .label = null,
2785 .inline_token = null,
2786 .loop_token = *token,
2787 .dest_ptr = *dest_ptr,
2788 }
2789 }) catch unreachable;
2790 return true;
2791 },
2792 Token.Id.Keyword_for => {
2793 stack.append(State {
2794 .For = LoopCtx {
2795 .label = null,
2796 .inline_token = null,
2797 .loop_token = *token,
2798 .dest_ptr = *dest_ptr,
2799 }
2800 }) catch unreachable;
2801 return true;
2802 },
2803 Token.Id.Keyword_switch => {
2804 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSwitch,
2805 ast.NodeSwitch {
2806 .base = undefined,
2807 .switch_token = *token,
2808 .expr = undefined,
2809 .cases = ArrayList(&ast.NodeSwitchCase).init(arena),
2810 .rbrace = undefined,
2811 }
2812 );
2813
2814 stack.append(State {
2815 .SwitchCaseOrEnd = ListSave(&ast.NodeSwitchCase) {
2816 .list = &node.cases,
2817 .ptr = &node.rbrace,
2818 },
2819 }) catch unreachable;
2820 try stack.append(State { .ExpectToken = Token.Id.LBrace });
2821 try stack.append(State { .ExpectToken = Token.Id.RParen });
2822 try stack.append(State { .Expression = DestPtr { .Field = &node.expr } });
2823 try stack.append(State { .ExpectToken = Token.Id.LParen });
2824 return true;
2825 },
2826 Token.Id.Keyword_comptime => {
2827 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeComptime,
2828 ast.NodeComptime {
2829 .base = undefined,
2830 .comptime_token = *token,
2831 .expr = undefined,
2832 }
2833 );
2834 try stack.append(State { .Expression = DestPtr { .Field = &node.expr } });
2835 return true;
2836 },
2837 Token.Id.LBrace => {
2838 const block = try self.createToDestNode(arena, dest_ptr, ast.NodeBlock,
2839 ast.NodeBlock {
2840 .base = undefined,
2841 .label = null,
2842 .lbrace = *token,
2843 .statements = ArrayList(&ast.Node).init(arena),
2844 .rbrace = undefined,
2845 }
2846 );
2847 stack.append(State { .Block = block }) catch unreachable;
2848 return true;
2849 },
2850 else => {
2851 return false;
2852 }
2853 }
2854 }
2855
27932856 fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, maybe_ptr: ?&Token, state_after_comma: &const State) !void {
27942857 var token = self.getNextToken();
27952858 switch (token.id) {
......@@ -2881,7 +2944,7 @@ pub const Parser = struct {
28812944 Token.Id.Tilde => ast.NodePrefixOp.PrefixOp { .BitNot = void{} },
28822945 Token.Id.Minus => ast.NodePrefixOp.PrefixOp { .Negation = void{} },
28832946 Token.Id.MinusPercent => ast.NodePrefixOp.PrefixOp { .NegationWrap = void{} },
2884 Token.Id.Asterisk => ast.NodePrefixOp.PrefixOp { .Deref = void{} },
2947 Token.Id.Asterisk, Token.Id.AsteriskAsterisk => ast.NodePrefixOp.PrefixOp { .Deref = void{} },
28852948 Token.Id.Ampersand => ast.NodePrefixOp.PrefixOp {
28862949 .AddrOf = ast.NodePrefixOp.AddrOfInfo {
28872950 .align_expr = null,
......@@ -3126,6 +3189,9 @@ pub const Parser = struct {
31263189 },
31273190 ast.Node.Id.StructField => {
31283191 const field = @fieldParentPtr(ast.NodeStructField, "base", decl);
3192 if (field.visib_token) |visib_token| {
3193 try stream.print("{} ", self.tokenizer.getTokenSlice(visib_token));
3194 }
31293195 try stream.print("{}: ", self.tokenizer.getTokenSlice(field.name_token));
31303196 try stack.append(RenderState { .Expression = field.type_expr});
31313197 },
......@@ -4573,6 +4639,7 @@ test "zig fmt: struct declaration" {
45734639 \\const S = struct {
45744640 \\ const Self = this;
45754641 \\ f1: u8,
4642 \\ pub f3: u8,
45764643 \\
45774644 \\ fn method(self: &Self) Self {
45784645 \\ return *self;
......@@ -4583,14 +4650,14 @@ test "zig fmt: struct declaration" {
45834650 \\
45844651 \\const Ps = packed struct {
45854652 \\ a: u8,
4586 \\ b: u8,
4653 \\ pub b: u8,
45874654 \\
45884655 \\ c: u8
45894656 \\};
45904657 \\
45914658 \\const Es = extern struct {
45924659 \\ a: u8,
4593 \\ b: u8,
4660 \\ pub b: u8,
45944661 \\
45954662 \\ c: u8
45964663 \\};
......@@ -4895,6 +4962,7 @@ test "zig fmt: if" {
48954962 \\ }
48964963 \\
48974964 \\ const is_world_broken = if (10 < 0) true else false;
4965 \\ const some_number = 1 + if (10 < 0) 2 else 3;
48984966 \\
48994967 \\ const a: ?u8 = 10;
49004968 \\ const b: ?u8 = null;