authorgravatar for brendan.hansknecht@gmail.comBrendan Hansknecht <brendan.hansknecht@gmail.com> 2019-10-31 21:53:46-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-11-06 14:03:21-05:00
logc1e8fdf812692edb9c7e496b00294aa749e81079
tree277731180a3c176352b80c1b7101f93e8765035d
parentdd4e9fb16b17579de1b54cdea90df82e41a50340

add token for parsing pointer dereference

8 files changed, 36 insertions(+), 7 deletions(-)

doc/docgen.zig+1
...@@ -917,6 +917,7 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok...@@ -917,6 +917,7 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok
917 .LBracket,917 .LBracket,
918 .RBracket,918 .RBracket,
919 .Period,919 .Period,
920 .PeriodAsterisk,
920 .Ellipsis2,921 .Ellipsis2,
921 .Ellipsis3,922 .Ellipsis3,
922 .Caret,923 .Caret,
lib/std/zig/parse.zig+4-3
...@@ -2363,6 +2363,10 @@ fn parseSuffixOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {...@@ -2363,6 +2363,10 @@ fn parseSuffixOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
2363 };2363 };
2364 }2364 }
23652365
2366 if (eatToken(it, .PeriodAsterisk)) |period_asterisk| {
2367 break :blk OpAndToken{ .op = Op{ .Deref = {} }, .token = period_asterisk };
2368 }
2369
2366 if (eatToken(it, .Period)) |period| {2370 if (eatToken(it, .Period)) |period| {
2367 if (try parseIdentifier(arena, it, tree)) |identifier| {2371 if (try parseIdentifier(arena, it, tree)) |identifier| {
2368 // TODO: It's a bit weird to return an InfixOp from the SuffixOp parser.2372 // TODO: It's a bit weird to return an InfixOp from the SuffixOp parser.
...@@ -2378,9 +2382,6 @@ fn parseSuffixOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {...@@ -2378,9 +2382,6 @@ fn parseSuffixOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
2378 };2382 };
2379 return &node.base;2383 return &node.base;
2380 }2384 }
2381 if (eatToken(it, .Asterisk)) |asterisk| {
2382 break :blk OpAndToken{ .op = Op{ .Deref = {} }, .token = asterisk };
2383 }
2384 if (eatToken(it, .QuestionMark)) |question_mark| {2385 if (eatToken(it, .QuestionMark)) |question_mark| {
2385 break :blk OpAndToken{ .op = Op{ .UnwrapOptional = {} }, .token = question_mark };2386 break :blk OpAndToken{ .op = Op{ .UnwrapOptional = {} }, .token = question_mark };
2386 }2387 }
lib/std/zig/render.zig+7-2
...@@ -617,10 +617,15 @@ fn renderExpression(...@@ -617,10 +617,15 @@ fn renderExpression(
617 return renderToken(tree, stream, rbracket, indent, start_col, space); // ]617 return renderToken(tree, stream, rbracket, indent, start_col, space); // ]
618 },618 },
619619
620 ast.Node.SuffixOp.Op.Deref, ast.Node.SuffixOp.Op.UnwrapOptional => {620 ast.Node.SuffixOp.Op.Deref => {
621 try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None);
622 return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); // .*
623 },
624
625 ast.Node.SuffixOp.Op.UnwrapOptional => {
621 try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None);626 try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None);
622 try renderToken(tree, stream, tree.prevToken(suffix_op.rtoken), indent, start_col, Space.None); // .627 try renderToken(tree, stream, tree.prevToken(suffix_op.rtoken), indent, start_col, Space.None); // .
623 return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); // * or ?628 return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); // ?
624 },629 },
625630
626 @TagType(ast.Node.SuffixOp.Op).Slice => |range| {631 @TagType(ast.Node.SuffixOp.Op).Slice => |range| {
lib/std/zig/tokenizer.zig+7
...@@ -103,6 +103,7 @@ pub const Token = struct {...@@ -103,6 +103,7 @@ pub const Token = struct {
103 LBracket,103 LBracket,
104 RBracket,104 RBracket,
105 Period,105 Period,
106 PeriodAsterisk,
106 Ellipsis2,107 Ellipsis2,
107 Ellipsis3,108 Ellipsis3,
108 Caret,109 Caret,
...@@ -231,6 +232,7 @@ pub const Token = struct {...@@ -231,6 +232,7 @@ pub const Token = struct {
231 .LBracket => "[",232 .LBracket => "[",
232 .RBracket => "]",233 .RBracket => "]",
233 .Period => ".",234 .Period => ".",
235 .PeriodAsterisk => ".*",
234 .Ellipsis2 => "..",236 .Ellipsis2 => "..",
235 .Ellipsis3 => "...",237 .Ellipsis3 => "...",
236 .Caret => "^",238 .Caret => "^",
...@@ -1033,6 +1035,11 @@ pub const Tokenizer = struct {...@@ -1033,6 +1035,11 @@ pub const Tokenizer = struct {
1033 '.' => {1035 '.' => {
1034 state = State.Period2;1036 state = State.Period2;
1035 },1037 },
1038 '*' => {
1039 result.id = Token.Id.PeriodAsterisk;
1040 self.index += 1;
1041 break;
1042 },
1036 else => {1043 else => {
1037 result.id = Token.Id.Period;1044 result.id = Token.Id.Period;
1038 break;1045 break;
src/parser.cpp+4-2
...@@ -2693,10 +2693,12 @@ static AstNode *ast_parse_suffix_op(ParseContext *pc) {...@@ -2693,10 +2693,12 @@ static AstNode *ast_parse_suffix_op(ParseContext *pc) {
2693 return res;2693 return res;
2694 }2694 }
26952695
2696 Token *dot_asterisk = eat_token_if(pc, TokenIdDotStar);
2697 if (dot_asterisk != nullptr)
2698 return ast_create_node(pc, NodeTypePtrDeref, dot_asterisk);
2699
2696 Token *dot = eat_token_if(pc, TokenIdDot);2700 Token *dot = eat_token_if(pc, TokenIdDot);
2697 if (dot != nullptr) {2701 if (dot != nullptr) {
2698 if (eat_token_if(pc, TokenIdStar) != nullptr)
2699 return ast_create_node(pc, NodeTypePtrDeref, dot);
2700 if (eat_token_if(pc, TokenIdQuestion) != nullptr)2702 if (eat_token_if(pc, TokenIdQuestion) != nullptr)
2701 return ast_create_node(pc, NodeTypeUnwrapOptional, dot);2703 return ast_create_node(pc, NodeTypeUnwrapOptional, dot);
27022704
src/tokenizer.cpp+6
...@@ -583,6 +583,11 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -583,6 +583,11 @@ void tokenize(Buf *buf, Tokenization *out) {
583 t.state = TokenizeStateSawDotDot;583 t.state = TokenizeStateSawDotDot;
584 set_token_id(&t, t.cur_tok, TokenIdEllipsis2);584 set_token_id(&t, t.cur_tok, TokenIdEllipsis2);
585 break;585 break;
586 case '*':
587 t.state = TokenizeStateStart;
588 set_token_id(&t, t.cur_tok, TokenIdDotStar);
589 end_token(&t);
590 break;
586 default:591 default:
587 t.pos -= 1;592 t.pos -= 1;
588 end_token(&t);593 end_token(&t);
...@@ -1603,6 +1608,7 @@ const char * token_name(TokenId id) {...@@ -1603,6 +1608,7 @@ const char * token_name(TokenId id) {
1603 case TokenIdDivEq: return "/=";1608 case TokenIdDivEq: return "/=";
1604 case TokenIdDocComment: return "DocComment";1609 case TokenIdDocComment: return "DocComment";
1605 case TokenIdDot: return ".";1610 case TokenIdDot: return ".";
1611 case TokenIdDotStar: return ".*";
1606 case TokenIdEllipsis2: return "..";1612 case TokenIdEllipsis2: return "..";
1607 case TokenIdEllipsis3: return "...";1613 case TokenIdEllipsis3: return "...";
1608 case TokenIdEof: return "EOF";1614 case TokenIdEof: return "EOF";
src/tokenizer.hpp+1
...@@ -44,6 +44,7 @@ enum TokenId {...@@ -44,6 +44,7 @@ enum TokenId {
44 TokenIdDivEq,44 TokenIdDivEq,
45 TokenIdDocComment,45 TokenIdDocComment,
46 TokenIdDot,46 TokenIdDot,
47 TokenIdDotStar,
47 TokenIdEllipsis2,48 TokenIdEllipsis2,
48 TokenIdEllipsis3,49 TokenIdEllipsis3,
49 TokenIdEof,50 TokenIdEof,
test/stage1/behavior/bugs/3468.zig created+6
...@@ -0,0 +1,6 @@
1// zig fmt: off
2test "pointer deref next to assignment" {
3 var a:i32=2;
4 var b=&a;
5 b.*=3;
6}