authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-01 22:02:51+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-01 22:02:51+02:00
logb9093185f748293064e7eeaaa07e7479099420ed
treea45b7d49eeb362bef91d78eff9f5e87e21183c18
parentdf09c01f7f141a384010d17ab23db3b36316f5b6

std.zig.parser now parses slicing and array access

3 files changed, 170 insertions(+), 0 deletions(-)

std/zig/ast.zig+66
...@@ -30,6 +30,8 @@ pub const Node = struct {...@@ -30,6 +30,8 @@ pub const Node = struct {
30 ErrorType,30 ErrorType,
31 BuiltinCall,31 BuiltinCall,
32 Call,32 Call,
33 ArrayAccess,
34 SliceExpression,
33 LineComment,35 LineComment,
34 TestDecl,36 TestDecl,
35 };37 };
...@@ -57,6 +59,8 @@ pub const Node = struct {...@@ -57,6 +59,8 @@ pub const Node = struct {
57 Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index),59 Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index),
58 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),60 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
59 Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index),61 Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index),
62 Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).iterate(index),
63 Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).iterate(index),
60 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index),64 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index),
61 Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index),65 Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index),
62 };66 };
...@@ -85,6 +89,8 @@ pub const Node = struct {...@@ -85,6 +89,8 @@ pub const Node = struct {
85 Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(),89 Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(),
86 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),90 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),
87 Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(),91 Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(),
92 Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).firstToken(),
93 Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).firstToken(),
88 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(),94 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(),
89 Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(),95 Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(),
90 };96 };
...@@ -113,6 +119,8 @@ pub const Node = struct {...@@ -113,6 +119,8 @@ pub const Node = struct {
113 Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(),119 Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(),
114 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),120 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),
115 Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(),121 Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(),
122 Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).lastToken(),
123 Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).lastToken(),
116 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(),124 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(),
117 Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(),125 Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(),
118 };126 };
...@@ -598,6 +606,64 @@ pub const NodeCall = struct {...@@ -598,6 +606,64 @@ pub const NodeCall = struct {
598 }606 }
599};607};
600608
609pub const NodeArrayAccess = struct {
610 base: Node,
611 expr: &Node,
612 index: &Node,
613 rbracket_token: Token,
614
615 pub fn iterate(self: &NodeArrayAccess, index: usize) ?&Node {
616 var i = index;
617
618 if (i < 1) return self.expr;
619 i -= 1;
620
621 if (i < 1) return self.index;
622 i -= 1;
623
624 return null;
625 }
626
627 pub fn firstToken(self: &NodeArrayAccess) Token {
628 return self.expr.firstToken();
629 }
630
631 pub fn lastToken(self: &NodeArrayAccess) Token {
632 return self.rbracket_token;
633 }
634};
635
636pub const NodeSliceExpression = struct {
637 base: Node,
638 expr: &Node,
639 start: &Node,
640 end: ?&Node,
641 rbracket_token: Token,
642
643 pub fn iterate(self: &NodeSliceExpression, index: usize) ?&Node {
644 var i = index;
645
646 if (i < 1) return self.callee;
647 i -= 1;
648
649 if (i < 1) return self.start;
650 i -= 1;
651
652 if (i < 1) return self.end;
653 i -= 1;
654
655 return null;
656 }
657
658 pub fn firstToken(self: &NodeSliceExpression) Token {
659 return self.expr.firstToken();
660 }
661
662 pub fn lastToken(self: &NodeSliceExpression) Token {
663 return self.rbracket_token;
664 }
665};
666
601pub const NodeStringLiteral = struct {667pub const NodeStringLiteral = struct {
602 base: Node,668 base: Node,
603 token: Token,669 token: Token,
std/zig/parser.zig+92
...@@ -88,6 +88,7 @@ pub const Parser = struct {...@@ -88,6 +88,7 @@ pub const Parser = struct {
88 InfixOp: &ast.NodeInfixOp,88 InfixOp: &ast.NodeInfixOp,
89 PrefixOp: &ast.NodePrefixOp,89 PrefixOp: &ast.NodePrefixOp,
90 SuffixOp: &ast.Node,90 SuffixOp: &ast.Node,
91 SliceOrArrayAccess,
91 AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo,92 AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo,
92 TypeExpr: DestPtr,93 TypeExpr: DestPtr,
93 VarDecl: &ast.NodeVarDecl,94 VarDecl: &ast.NodeVarDecl,
...@@ -590,6 +591,11 @@ pub const Parser = struct {...@@ -590,6 +591,11 @@ pub const Parser = struct {
590 });591 });
591 continue;592 continue;
592593
594 } else if (token.id == Token.Id.LBracket) {
595 try stack.append(State.SliceOrArrayAccess);
596 try stack.append(State.ExpectOperand);
597 continue;
598
593 // TODO: Parse postfix operator599 // TODO: Parse postfix operator
594 } else {600 } else {
595 // no postfix/infix operator after this operand.601 // no postfix/infix operator after this operand.
...@@ -603,6 +609,53 @@ pub const Parser = struct {...@@ -603,6 +609,53 @@ pub const Parser = struct {
603 try dest_ptr.store(expression);609 try dest_ptr.store(expression);
604 break;610 break;
605 },611 },
612 State.SliceOrArrayAccess => {
613 var rbracket_or_ellipsis2_token = self.getNextToken();
614
615 switch (rbracket_or_ellipsis2_token.id) {
616 Token.Id.Ellipsis2 => {
617 const node = try arena.create(ast.NodeSliceExpression);
618 *node = ast.NodeSliceExpression {
619 .base = self.initNode(ast.Node.Id.SliceExpression),
620 .expr = undefined,
621 .start = expression,
622 .end = null,
623 .rbracket_token = undefined,
624 };
625
626 try stack.append(State { .SuffixOp = &node.base });
627 try stack.append(State.AfterOperand);
628
629 const rbracket_token = self.getNextToken();
630 if (rbracket_token.id != Token.Id.RBracket) {
631 self.putBackToken(rbracket_token);
632 try stack.append(State {
633 .ExpectTokenSave = ExpectTokenSave {
634 .id = Token.Id.RBracket,
635 .ptr = &node.rbracket_token,
636 }
637 });
638 try stack.append(State { .Expression = DestPtr { .NullableField = &node.end } });
639 } else {
640 node.rbracket_token = rbracket_token;
641 }
642 break;
643 },
644 Token.Id.RBracket => {
645 const node = try arena.create(ast.NodeArrayAccess);
646 *node = ast.NodeArrayAccess {
647 .base = self.initNode(ast.Node.Id.ArrayAccess),
648 .expr = undefined,
649 .index = expression,
650 .rbracket_token = token,
651 };
652 try stack.append(State { .SuffixOp = &node.base });
653 try stack.append(State.AfterOperand);
654 break;
655 },
656 else => return self.parseError(token, "expected ']' or '..', found {}", @tagName(token.id))
657 }
658 },
606 State.InfixOp => |infix_op| {659 State.InfixOp => |infix_op| {
607 infix_op.rhs = expression;660 infix_op.rhs = expression;
608 infix_op.lhs = popSuffixOp(&stack);661 infix_op.lhs = popSuffixOp(&stack);
...@@ -857,6 +910,7 @@ pub const Parser = struct {...@@ -857,6 +910,7 @@ pub const Parser = struct {
857 State.PrefixOp => unreachable,910 State.PrefixOp => unreachable,
858 State.SuffixOp => unreachable,911 State.SuffixOp => unreachable,
859 State.Operand => unreachable,912 State.Operand => unreachable,
913 State.SliceOrArrayAccess => unreachable,
860 }914 }
861 }915 }
862 }916 }
...@@ -874,6 +928,18 @@ pub const Parser = struct {...@@ -874,6 +928,18 @@ pub const Parser = struct {
874 left_leaf_ptr = &call.callee;928 left_leaf_ptr = &call.callee;
875 continue;929 continue;
876 },930 },
931 ast.Node.Id.ArrayAccess => {
932 const arr_access = @fieldParentPtr(ast.NodeArrayAccess, "base", suffix_op);
933 *left_leaf_ptr = &arr_access.base;
934 left_leaf_ptr = &arr_access.expr;
935 continue;
936 },
937 ast.Node.Id.SliceExpression => {
938 const slice_expr = @fieldParentPtr(ast.NodeSliceExpression, "base", suffix_op);
939 *left_leaf_ptr = &slice_expr.base;
940 left_leaf_ptr = &slice_expr.expr;
941 continue;
942 },
877 else => unreachable,943 else => unreachable,
878 }944 }
879 },945 },
...@@ -1594,6 +1660,24 @@ pub const Parser = struct {...@@ -1594,6 +1660,24 @@ pub const Parser = struct {
1594 try stack.append(RenderState { .Text = "("});1660 try stack.append(RenderState { .Text = "("});
1595 try stack.append(RenderState { .Expression = call.callee });1661 try stack.append(RenderState { .Expression = call.callee });
1596 },1662 },
1663 ast.Node.Id.ArrayAccess => {
1664 const arr_access = @fieldParentPtr(ast.NodeArrayAccess, "base", base);
1665 try stack.append(RenderState { .Text = "]"});
1666 try stack.append(RenderState { .Expression = arr_access.index});
1667 try stack.append(RenderState { .Text = "["});
1668 try stack.append(RenderState { .Expression = arr_access.expr });
1669 },
1670 ast.Node.Id.SliceExpression => {
1671 const slice_expr = @fieldParentPtr(ast.NodeSliceExpression, "base", base);
1672 try stack.append(RenderState { .Text = "]"});
1673 if (slice_expr.end) |end| {
1674 try stack.append(RenderState { .Expression = end});
1675 }
1676 try stack.append(RenderState { .Text = ".."});
1677 try stack.append(RenderState { .Expression = slice_expr.start});
1678 try stack.append(RenderState { .Text = "["});
1679 try stack.append(RenderState { .Expression = slice_expr.expr});
1680 },
1597 ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"),1681 ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"),
1598 ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"),1682 ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"),
15991683
...@@ -1978,6 +2062,14 @@ test "zig fmt: indexing" {...@@ -1978,6 +2062,14 @@ test "zig fmt: indexing" {
1978 \\ a[0 + 5];2062 \\ a[0 + 5];
1979 \\ a[0..];2063 \\ a[0..];
1980 \\ a[0..5];2064 \\ a[0..5];
2065 \\ a[a[0]];
2066 \\ a[a[0..]];
2067 \\ a[a[0..5]];
2068 \\ a[a[0]..];
2069 \\ a[a[0..5]..];
2070 \\ a[a[0]..a[0]];
2071 \\ a[a[0..5]..a[0]];
2072 \\ a[a[0..5]..a[0..5]];
1981 \\}2073 \\}
1982 \\2074 \\
1983 );2075 );
std/zig/tokenizer.zig+12
...@@ -91,6 +91,8 @@ pub const Token = struct {...@@ -91,6 +91,8 @@ pub const Token = struct {
91 PercentEqual,91 PercentEqual,
92 LBrace,92 LBrace,
93 RBrace,93 RBrace,
94 LBracket,
95 RBracket,
94 Period,96 Period,
95 Ellipsis2,97 Ellipsis2,
96 Ellipsis3,98 Ellipsis3,
...@@ -327,6 +329,16 @@ pub const Tokenizer = struct {...@@ -327,6 +329,16 @@ pub const Tokenizer = struct {
327 self.index += 1;329 self.index += 1;
328 break;330 break;
329 },331 },
332 '[' => {
333 result.id = Token.Id.LBracket;
334 self.index += 1;
335 break;
336 },
337 ']' => {
338 result.id = Token.Id.RBracket;
339 self.index += 1;
340 break;
341 },
330 ';' => {342 ';' => {
331 result.id = Token.Id.Semicolon;343 result.id = Token.Id.Semicolon;
332 self.index += 1;344 self.index += 1;