authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-11 13:38:06+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-11 13:38:06+02:00
log4b0556ebd4bdc2f3c05e85f3c003b6fc89f7ac0f
tree331b0669fb130a70cf49c69f7294ccf96ea761f8
parent6fb5ab1b523c58350321fdcbe7ba921300bef8af

std.zig.parser can now parse `std/heap.zig`:

related: #909 * Struct fields can now be pub * Parsing of double deref now works * Block expressions now have the right precedence

2 files changed, 180 insertions(+), 110 deletions(-)

std/zig/ast.zig+2
...@@ -360,6 +360,7 @@ pub const NodeContainerDecl = struct {...@@ -360,6 +360,7 @@ pub const NodeContainerDecl = struct {
360360
361pub const NodeStructField = struct {361pub const NodeStructField = struct {
362 base: Node,362 base: Node,
363 visib_token: ?Token,
363 name_token: Token,364 name_token: Token,
364 type_expr: &Node,365 type_expr: &Node,
365366
...@@ -373,6 +374,7 @@ pub const NodeStructField = struct {...@@ -373,6 +374,7 @@ pub const NodeStructField = struct {
373 }374 }
374375
375 pub fn firstToken(self: &NodeStructField) Token {376 pub fn firstToken(self: &NodeStructField) Token {
377 if (self.visib_token) |visib_token| return visib_token;
376 return self.name_token;378 return self.name_token;
377 }379 }
378380
std/zig/parser.zig+178-110
...@@ -669,6 +669,7 @@ pub const Parser = struct {...@@ -669,6 +669,7 @@ pub const Parser = struct {
669 const node = try self.createAttachNode(arena, &container_decl.fields_and_decls, ast.NodeStructField,669 const node = try self.createAttachNode(arena, &container_decl.fields_and_decls, ast.NodeStructField,
670 ast.NodeStructField {670 ast.NodeStructField {
671 .base = undefined,671 .base = undefined,
672 .visib_token = null,
672 .name_token = token,673 .name_token = token,
673 .type_expr = undefined,674 .type_expr = undefined,
674 }675 }
...@@ -721,7 +722,42 @@ pub const Parser = struct {...@@ -721,7 +722,42 @@ pub const Parser = struct {
721 },722 },
722 }723 }
723 },724 },
724 Token.Id.Keyword_pub, Token.Id.Keyword_export => {725 Token.Id.Keyword_pub => {
726 if (self.eatToken(Token.Id.Identifier)) |identifier| {
727 switch (container_decl.kind) {
728 ast.NodeContainerDecl.Kind.Struct => {
729 const node = try self.createAttachNode(arena, &container_decl.fields_and_decls, ast.NodeStructField,
730 ast.NodeStructField {
731 .base = undefined,
732 .visib_token = token,
733 .name_token = identifier,
734 .type_expr = undefined,
735 }
736 );
737
738 stack.append(State { .FieldListCommaOrEnd = container_decl }) catch unreachable;
739 try stack.append(State { .Expression = DestPtr { .Field = &node.type_expr } });
740 try stack.append(State { .ExpectToken = Token.Id.Colon });
741 continue;
742 },
743 else => {
744 self.putBackToken(identifier);
745 }
746 }
747 }
748
749 stack.append(State{ .ContainerDecl = container_decl }) catch unreachable;
750 try stack.append(State {
751 .TopLevelExtern = TopLevelDeclCtx {
752 .decls = &container_decl.fields_and_decls,
753 .visib_token = token,
754 .extern_token = null,
755 .lib_name = null,
756 }
757 });
758 continue;
759 },
760 Token.Id.Keyword_export => {
725 stack.append(State{ .ContainerDecl = container_decl }) catch unreachable;761 stack.append(State{ .ContainerDecl = container_decl }) catch unreachable;
726 try stack.append(State {762 try stack.append(State {
727 .TopLevelExtern = TopLevelDeclCtx {763 .TopLevelExtern = TopLevelDeclCtx {
...@@ -864,111 +900,11 @@ pub const Parser = struct {...@@ -864,111 +900,11 @@ pub const Parser = struct {
864 stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable;900 stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable;
865 continue;901 continue;
866 },902 },
867 Token.Id.Keyword_suspend => {
868 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSuspend,
869 ast.NodeSuspend {
870 .base = undefined,
871 .suspend_token = token,
872 .payload = null,
873 .body = null,
874 }
875 );
876
877 stack.append(State { .SuspendBody = node }) catch unreachable;
878 try stack.append(State { .Payload = &node.payload });
879 continue;
880 },
881 Token.Id.Keyword_if => {
882 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeIf,
883 ast.NodeIf {
884 .base = undefined,
885 .if_token = token,
886 .condition = undefined,
887 .payload = null,
888 .body = undefined,
889 .@"else" = null,
890 }
891 );
892
893 stack.append(State { .Else = &node.@"else" }) catch unreachable;
894 try stack.append(State { .Expression = DestPtr { .Field = &node.body } });
895 try stack.append(State { .PointerPayload = &node.payload });
896 try stack.append(State { .ExpectToken = Token.Id.RParen });
897 try stack.append(State { .Expression = DestPtr { .Field = &node.condition } });
898 try stack.append(State { .ExpectToken = Token.Id.LParen });
899 continue;
900 },
901 Token.Id.Keyword_while => {
902 stack.append(State {
903 .While = LoopCtx {
904 .label = null,
905 .inline_token = null,
906 .loop_token = token,
907 .dest_ptr = dest_ptr,
908 }
909 }) catch unreachable;
910 continue;
911 },
912 Token.Id.Keyword_for => {
913 stack.append(State {
914 .For = LoopCtx {
915 .label = null,
916 .inline_token = null,
917 .loop_token = token,
918 .dest_ptr = dest_ptr,
919 }
920 }) catch unreachable;
921 continue;
922 },
923 Token.Id.Keyword_switch => {
924 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSwitch,
925 ast.NodeSwitch {
926 .base = undefined,
927 .switch_token = token,
928 .expr = undefined,
929 .cases = ArrayList(&ast.NodeSwitchCase).init(arena),
930 .rbrace = undefined,
931 }
932 );
933
934 stack.append(State {
935 .SwitchCaseOrEnd = ListSave(&ast.NodeSwitchCase) {
936 .list = &node.cases,
937 .ptr = &node.rbrace,
938 },
939 }) catch unreachable;
940 try stack.append(State { .ExpectToken = Token.Id.LBrace });
941 try stack.append(State { .ExpectToken = Token.Id.RParen });
942 try stack.append(State { .Expression = DestPtr { .Field = &node.expr } });
943 try stack.append(State { .ExpectToken = Token.Id.LParen });
944 },
945 Token.Id.Keyword_comptime => {
946 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeComptime,
947 ast.NodeComptime {
948 .base = undefined,
949 .comptime_token = token,
950 .expr = undefined,
951 }
952 );
953 try stack.append(State { .Expression = DestPtr { .Field = &node.expr } });
954 continue;
955 },
956 Token.Id.LBrace => {
957 const block = try self.createToDestNode(arena, dest_ptr, ast.NodeBlock,
958 ast.NodeBlock {
959 .base = undefined,
960 .label = null,
961 .lbrace = token,
962 .statements = ArrayList(&ast.Node).init(arena),
963 .rbrace = undefined,
964 }
965 );
966 stack.append(State { .Block = block }) catch unreachable;
967 continue;
968 },
969 else => {903 else => {
970 self.putBackToken(token);904 if (!try self.parseBlockExpr(&stack, arena, dest_ptr, token)) {
971 stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable;905 self.putBackToken(token);
906 stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable;
907 }
972 continue;908 continue;
973 }909 }
974 }910 }
...@@ -1407,7 +1343,7 @@ pub const Parser = struct {...@@ -1407,7 +1343,7 @@ pub const Parser = struct {
1407 State.PrefixOpExpression => |dest_ptr| {1343 State.PrefixOpExpression => |dest_ptr| {
1408 const token = self.getNextToken();1344 const token = self.getNextToken();
1409 if (tokenIdToPrefixOp(token.id)) |prefix_id| {1345 if (tokenIdToPrefixOp(token.id)) |prefix_id| {
1410 const node = try self.createToDestNode(arena, dest_ptr, ast.NodePrefixOp,1346 var node = try self.createToDestNode(arena, dest_ptr, ast.NodePrefixOp,
1411 ast.NodePrefixOp {1347 ast.NodePrefixOp {
1412 .base = undefined,1348 .base = undefined,
1413 .op_token = token,1349 .op_token = token,
...@@ -1415,6 +1351,20 @@ pub const Parser = struct {...@@ -1415,6 +1351,20 @@ pub const Parser = struct {
1415 .rhs = undefined,1351 .rhs = undefined,
1416 }1352 }
1417 );1353 );
1354
1355 if (token.id == Token.Id.AsteriskAsterisk) {
1356 const child = try self.createNode(arena, ast.NodePrefixOp,
1357 ast.NodePrefixOp {
1358 .base = undefined,
1359 .op_token = token,
1360 .op = prefix_id,
1361 .rhs = undefined,
1362 }
1363 );
1364 node.rhs = &child.base;
1365 node = child;
1366 }
1367
1418 stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.rhs } }) catch unreachable;1368 stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.rhs } }) catch unreachable;
1419 if (node.op == ast.NodePrefixOp.PrefixOp.AddrOf) {1369 if (node.op == ast.NodePrefixOp.PrefixOp.AddrOf) {
1420 try stack.append(State { .AddrOfModifiers = &node.op.AddrOf });1370 try stack.append(State { .AddrOfModifiers = &node.op.AddrOf });
...@@ -1871,7 +1821,9 @@ pub const Parser = struct {...@@ -1871,7 +1821,9 @@ pub const Parser = struct {
1871 continue;1821 continue;
1872 },1822 },
1873 else => {1823 else => {
1874 try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id));1824 if (!try self.parseBlockExpr(&stack, arena, dest_ptr, token)) {
1825 try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id));
1826 }
1875 continue;1827 continue;
1876 }1828 }
1877 }1829 }
...@@ -2790,6 +2742,117 @@ pub const Parser = struct {...@@ -2790,6 +2742,117 @@ pub const Parser = struct {
2790 }2742 }
2791 }2743 }
27922744
2745 fn parseBlockExpr(self: &Parser, stack: &ArrayList(State), arena: &mem.Allocator, dest_ptr: &const DestPtr, token: &const Token) !bool {
2746 switch (token.id) {
2747 Token.Id.Keyword_suspend => {
2748 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSuspend,
2749 ast.NodeSuspend {
2750 .base = undefined,
2751 .suspend_token = *token,
2752 .payload = null,
2753 .body = null,
2754 }
2755 );
2756
2757 stack.append(State { .SuspendBody = node }) catch unreachable;
2758 try stack.append(State { .Payload = &node.payload });
2759 return true;
2760 },
2761 Token.Id.Keyword_if => {
2762 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeIf,
2763 ast.NodeIf {
2764 .base = undefined,
2765 .if_token = *token,
2766 .condition = undefined,
2767 .payload = null,
2768 .body = undefined,
2769 .@"else" = null,
2770 }
2771 );
2772
2773 stack.append(State { .Else = &node.@"else" }) catch unreachable;
2774 try stack.append(State { .Expression = DestPtr { .Field = &node.body } });
2775 try stack.append(State { .PointerPayload = &node.payload });
2776 try stack.append(State { .ExpectToken = Token.Id.RParen });
2777 try stack.append(State { .Expression = DestPtr { .Field = &node.condition } });
2778 try stack.append(State { .ExpectToken = Token.Id.LParen });
2779 return true;
2780 },
2781 Token.Id.Keyword_while => {
2782 stack.append(State {
2783 .While = LoopCtx {
2784 .label = null,
2785 .inline_token = null,
2786 .loop_token = *token,
2787 .dest_ptr = *dest_ptr,
2788 }
2789 }) catch unreachable;
2790 return true;
2791 },
2792 Token.Id.Keyword_for => {
2793 stack.append(State {
2794 .For = LoopCtx {
2795 .label = null,
2796 .inline_token = null,
2797 .loop_token = *token,
2798 .dest_ptr = *dest_ptr,
2799 }
2800 }) catch unreachable;
2801 return true;
2802 },
2803 Token.Id.Keyword_switch => {
2804 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSwitch,
2805 ast.NodeSwitch {
2806 .base = undefined,
2807 .switch_token = *token,
2808 .expr = undefined,
2809 .cases = ArrayList(&ast.NodeSwitchCase).init(arena),
2810 .rbrace = undefined,
2811 }
2812 );
2813
2814 stack.append(State {
2815 .SwitchCaseOrEnd = ListSave(&ast.NodeSwitchCase) {
2816 .list = &node.cases,
2817 .ptr = &node.rbrace,
2818 },
2819 }) catch unreachable;
2820 try stack.append(State { .ExpectToken = Token.Id.LBrace });
2821 try stack.append(State { .ExpectToken = Token.Id.RParen });
2822 try stack.append(State { .Expression = DestPtr { .Field = &node.expr } });
2823 try stack.append(State { .ExpectToken = Token.Id.LParen });
2824 return true;
2825 },
2826 Token.Id.Keyword_comptime => {
2827 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeComptime,
2828 ast.NodeComptime {
2829 .base = undefined,
2830 .comptime_token = *token,
2831 .expr = undefined,
2832 }
2833 );
2834 try stack.append(State { .Expression = DestPtr { .Field = &node.expr } });
2835 return true;
2836 },
2837 Token.Id.LBrace => {
2838 const block = try self.createToDestNode(arena, dest_ptr, ast.NodeBlock,
2839 ast.NodeBlock {
2840 .base = undefined,
2841 .label = null,
2842 .lbrace = *token,
2843 .statements = ArrayList(&ast.Node).init(arena),
2844 .rbrace = undefined,
2845 }
2846 );
2847 stack.append(State { .Block = block }) catch unreachable;
2848 return true;
2849 },
2850 else => {
2851 return false;
2852 }
2853 }
2854 }
2855
2793 fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, maybe_ptr: ?&Token, state_after_comma: &const State) !void {2856 fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, maybe_ptr: ?&Token, state_after_comma: &const State) !void {
2794 var token = self.getNextToken();2857 var token = self.getNextToken();
2795 switch (token.id) {2858 switch (token.id) {
...@@ -2881,7 +2944,7 @@ pub const Parser = struct {...@@ -2881,7 +2944,7 @@ pub const Parser = struct {
2881 Token.Id.Tilde => ast.NodePrefixOp.PrefixOp { .BitNot = void{} },2944 Token.Id.Tilde => ast.NodePrefixOp.PrefixOp { .BitNot = void{} },
2882 Token.Id.Minus => ast.NodePrefixOp.PrefixOp { .Negation = void{} },2945 Token.Id.Minus => ast.NodePrefixOp.PrefixOp { .Negation = void{} },
2883 Token.Id.MinusPercent => ast.NodePrefixOp.PrefixOp { .NegationWrap = void{} },2946 Token.Id.MinusPercent => ast.NodePrefixOp.PrefixOp { .NegationWrap = void{} },
2884 Token.Id.Asterisk => ast.NodePrefixOp.PrefixOp { .Deref = void{} },2947 Token.Id.Asterisk, Token.Id.AsteriskAsterisk => ast.NodePrefixOp.PrefixOp { .Deref = void{} },
2885 Token.Id.Ampersand => ast.NodePrefixOp.PrefixOp {2948 Token.Id.Ampersand => ast.NodePrefixOp.PrefixOp {
2886 .AddrOf = ast.NodePrefixOp.AddrOfInfo {2949 .AddrOf = ast.NodePrefixOp.AddrOfInfo {
2887 .align_expr = null,2950 .align_expr = null,
...@@ -3126,6 +3189,9 @@ pub const Parser = struct {...@@ -3126,6 +3189,9 @@ pub const Parser = struct {
3126 },3189 },
3127 ast.Node.Id.StructField => {3190 ast.Node.Id.StructField => {
3128 const field = @fieldParentPtr(ast.NodeStructField, "base", decl);3191 const field = @fieldParentPtr(ast.NodeStructField, "base", decl);
3192 if (field.visib_token) |visib_token| {
3193 try stream.print("{} ", self.tokenizer.getTokenSlice(visib_token));
3194 }
3129 try stream.print("{}: ", self.tokenizer.getTokenSlice(field.name_token));3195 try stream.print("{}: ", self.tokenizer.getTokenSlice(field.name_token));
3130 try stack.append(RenderState { .Expression = field.type_expr});3196 try stack.append(RenderState { .Expression = field.type_expr});
3131 },3197 },
...@@ -4573,6 +4639,7 @@ test "zig fmt: struct declaration" {...@@ -4573,6 +4639,7 @@ test "zig fmt: struct declaration" {
4573 \\const S = struct {4639 \\const S = struct {
4574 \\ const Self = this;4640 \\ const Self = this;
4575 \\ f1: u8,4641 \\ f1: u8,
4642 \\ pub f3: u8,
4576 \\4643 \\
4577 \\ fn method(self: &Self) Self {4644 \\ fn method(self: &Self) Self {
4578 \\ return *self;4645 \\ return *self;
...@@ -4583,14 +4650,14 @@ test "zig fmt: struct declaration" {...@@ -4583,14 +4650,14 @@ test "zig fmt: struct declaration" {
4583 \\4650 \\
4584 \\const Ps = packed struct {4651 \\const Ps = packed struct {
4585 \\ a: u8,4652 \\ a: u8,
4586 \\ b: u8,4653 \\ pub b: u8,
4587 \\4654 \\
4588 \\ c: u84655 \\ c: u8
4589 \\};4656 \\};
4590 \\4657 \\
4591 \\const Es = extern struct {4658 \\const Es = extern struct {
4592 \\ a: u8,4659 \\ a: u8,
4593 \\ b: u8,4660 \\ pub b: u8,
4594 \\4661 \\
4595 \\ c: u84662 \\ c: u8
4596 \\};4663 \\};
...@@ -4895,6 +4962,7 @@ test "zig fmt: if" {...@@ -4895,6 +4962,7 @@ test "zig fmt: if" {
4895 \\ }4962 \\ }
4896 \\4963 \\
4897 \\ const is_world_broken = if (10 < 0) true else false;4964 \\ const is_world_broken = if (10 < 0) true else false;
4965 \\ const some_number = 1 + if (10 < 0) 2 else 3;
4898 \\4966 \\
4899 \\ const a: ?u8 = 10;4967 \\ const a: ?u8 = 10;
4900 \\ const b: ?u8 = null;4968 \\ const b: ?u8 = null;