authorgravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2021-03-07 12:38:08+01:00
committergravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2021-03-08 01:37:28+01:00
logb988815bf0771dd86a345ce8ed8b0d3eb9d6f55e
treefed9a51f338b4ada0bbebf6314bba22482813361
parentd01bb211732f1d0aebf558c43825a52254d034ab

parser: fix parsing/rendering of a[b.. :c] slicing

The modification to the grammar in the comment is in line with the grammar in the zig-spec repo. Note: checking if the previous token is a colon is insufficent to tell if a block has a label, the identifier must be checked for as well. This can be seen in sentinel terminated slicing: `foo[0..1:{}]`

5 files changed, 43 insertions(+), 37 deletions(-)

lib/std/zig/ast.zig+6-3
...@@ -525,7 +525,9 @@ pub const Tree = struct {...@@ -525,7 +525,9 @@ pub const Tree = struct {
525 => {525 => {
526 // Look for a label.526 // Look for a label.
527 const lbrace = main_tokens[n];527 const lbrace = main_tokens[n];
528 if (token_tags[lbrace - 1] == .colon) {528 if (token_tags[lbrace - 1] == .colon and
529 token_tags[lbrace - 2] == .identifier)
530 {
529 end_offset += 2;531 end_offset += 2;
530 }532 }
531 return lbrace - end_offset;533 return lbrace - end_offset;
...@@ -989,13 +991,13 @@ pub const Tree = struct {...@@ -989,13 +991,13 @@ pub const Tree = struct {
989 },991 },
990 .slice => {992 .slice => {
991 const extra = tree.extraData(datas[n].rhs, Node.Slice);993 const extra = tree.extraData(datas[n].rhs, Node.Slice);
992 assert(extra.end != 0); // should have used SliceOpen994 assert(extra.end != 0); // should have used slice_open
993 end_offset += 1; // rbracket995 end_offset += 1; // rbracket
994 n = extra.end;996 n = extra.end;
995 },997 },
996 .slice_sentinel => {998 .slice_sentinel => {
997 const extra = tree.extraData(datas[n].rhs, Node.SliceSentinel);999 const extra = tree.extraData(datas[n].rhs, Node.SliceSentinel);
998 assert(extra.sentinel != 0); // should have used Slice1000 assert(extra.sentinel != 0); // should have used slice
999 end_offset += 1; // rbracket1001 end_offset += 1; // rbracket
1000 n = extra.sentinel;1002 n = extra.sentinel;
1001 },1003 },
...@@ -2925,6 +2927,7 @@ pub const Node = struct {...@@ -2925,6 +2927,7 @@ pub const Node = struct {
29252927
2926 pub const SliceSentinel = struct {2928 pub const SliceSentinel = struct {
2927 start: Index,2929 start: Index,
2930 /// May be 0 if the slice is "open"
2928 end: Index,2931 end: Index,
2929 sentinel: Index,2932 sentinel: Index,
2930 };2933 };
lib/std/zig/parse.zig+17-19
...@@ -3441,7 +3441,7 @@ const Parser = struct {...@@ -3441,7 +3441,7 @@ const Parser = struct {
3441 }3441 }
34423442
3443 /// SuffixOp3443 /// SuffixOp
3444 /// <- LBRACKET Expr (DOT2 (Expr (COLON Expr)?)?)? RBRACKET3444 /// <- LBRACKET Expr (DOT2 (Expr? (COLON Expr)?)?)? RBRACKET
3445 /// / DOT IDENTIFIER3445 /// / DOT IDENTIFIER
3446 /// / DOTASTERISK3446 /// / DOTASTERISK
3447 /// / DOTQUESTIONMARK3447 /// / DOTQUESTIONMARK
...@@ -3453,17 +3453,6 @@ const Parser = struct {...@@ -3453,17 +3453,6 @@ const Parser = struct {
34533453
3454 if (p.eatToken(.ellipsis2)) |_| {3454 if (p.eatToken(.ellipsis2)) |_| {
3455 const end_expr = try p.parseExpr();3455 const end_expr = try p.parseExpr();
3456 if (end_expr == 0) {
3457 _ = try p.expectToken(.r_bracket);
3458 return p.addNode(.{
3459 .tag = .slice_open,
3460 .main_token = lbracket,
3461 .data = .{
3462 .lhs = lhs,
3463 .rhs = index_expr,
3464 },
3465 });
3466 }
3467 if (p.eatToken(.colon)) |_| {3456 if (p.eatToken(.colon)) |_| {
3468 const sentinel = try p.parseExpr();3457 const sentinel = try p.parseExpr();
3469 _ = try p.expectToken(.r_bracket);3458 _ = try p.expectToken(.r_bracket);
...@@ -3479,20 +3468,29 @@ const Parser = struct {...@@ -3479,20 +3468,29 @@ const Parser = struct {
3479 }),3468 }),
3480 },3469 },
3481 });3470 });
3482 } else {3471 }
3483 _ = try p.expectToken(.r_bracket);3472 _ = try p.expectToken(.r_bracket);
3473 if (end_expr == 0) {
3484 return p.addNode(.{3474 return p.addNode(.{
3485 .tag = .slice,3475 .tag = .slice_open,
3486 .main_token = lbracket,3476 .main_token = lbracket,
3487 .data = .{3477 .data = .{
3488 .lhs = lhs,3478 .lhs = lhs,
3489 .rhs = try p.addExtra(Node.Slice{3479 .rhs = index_expr,
3490 .start = index_expr,
3491 .end = end_expr,
3492 }),
3493 },3480 },
3494 });3481 });
3495 }3482 }
3483 return p.addNode(.{
3484 .tag = .slice,
3485 .main_token = lbracket,
3486 .data = .{
3487 .lhs = lhs,
3488 .rhs = try p.addExtra(Node.Slice{
3489 .start = index_expr,
3490 .end = end_expr,
3491 }),
3492 },
3493 });
3496 }3494 }
3497 _ = try p.expectToken(.r_bracket);3495 _ = try p.expectToken(.r_bracket);
3498 return p.addNode(.{3496 return p.addNode(.{
lib/std/zig/parser_test.zig+2
...@@ -852,6 +852,7 @@ test "zig fmt: slices" {...@@ -852,6 +852,7 @@ test "zig fmt: slices" {
852 try testCanonical(852 try testCanonical(
853 \\const a = b[0..];853 \\const a = b[0..];
854 \\const c = d[0..1];854 \\const c = d[0..1];
855 \\const d = f[0.. :0];
855 \\const e = f[0..1 :0];856 \\const e = f[0..1 :0];
856 \\857 \\
857 );858 );
...@@ -861,6 +862,7 @@ test "zig fmt: slices with spaces in bounds" {...@@ -861,6 +862,7 @@ test "zig fmt: slices with spaces in bounds" {
861 try testCanonical(862 try testCanonical(
862 \\const a = b[0 + 0 ..];863 \\const a = b[0 + 0 ..];
863 \\const c = d[0 + 0 .. 1];864 \\const c = d[0 + 0 .. 1];
865 \\const c = d[0 + 0 .. :0];
864 \\const e = f[0 .. 1 + 1 :0];866 \\const e = f[0 .. 1 + 1 :0];
865 \\867 \\
866 );868 );
lib/std/zig/render.zig+15-14
...@@ -470,9 +470,9 @@ fn renderExpression(gpa: *Allocator, ais: *Ais, tree: ast.Tree, node: ast.Node.I...@@ -470,9 +470,9 @@ fn renderExpression(gpa: *Allocator, ais: *Ais, tree: ast.Tree, node: ast.Node.I
470 return renderToken(ais, tree, rbracket, space); // ]470 return renderToken(ais, tree, rbracket, space); // ]
471 },471 },
472472
473 .slice_open => return renderSlice(gpa, ais, tree, tree.sliceOpen(node), space),473 .slice_open => return renderSlice(gpa, ais, tree, node, tree.sliceOpen(node), space),
474 .slice => return renderSlice(gpa, ais, tree, tree.slice(node), space),474 .slice => return renderSlice(gpa, ais, tree, node, tree.slice(node), space),
475 .slice_sentinel => return renderSlice(gpa, ais, tree, tree.sliceSentinel(node), space),475 .slice_sentinel => return renderSlice(gpa, ais, tree, node, tree.sliceSentinel(node), space),
476476
477 .deref => {477 .deref => {
478 try renderExpression(gpa, ais, tree, datas[node].lhs, .none);478 try renderExpression(gpa, ais, tree, datas[node].lhs, .none);
...@@ -815,6 +815,7 @@ fn renderSlice(...@@ -815,6 +815,7 @@ fn renderSlice(
815 gpa: *Allocator,815 gpa: *Allocator,
816 ais: *Ais,816 ais: *Ais,
817 tree: ast.Tree,817 tree: ast.Tree,
818 slice_node: ast.Node.Index,
818 slice: ast.full.Slice,819 slice: ast.full.Slice,
819 space: Space,820 space: Space,
820) Error!void {821) Error!void {
...@@ -822,7 +823,9 @@ fn renderSlice(...@@ -822,7 +823,9 @@ fn renderSlice(
822 const after_start_space_bool = nodeCausesSliceOpSpace(node_tags[slice.ast.start]) or823 const after_start_space_bool = nodeCausesSliceOpSpace(node_tags[slice.ast.start]) or
823 if (slice.ast.end != 0) nodeCausesSliceOpSpace(node_tags[slice.ast.end]) else false;824 if (slice.ast.end != 0) nodeCausesSliceOpSpace(node_tags[slice.ast.end]) else false;
824 const after_start_space = if (after_start_space_bool) Space.space else Space.none;825 const after_start_space = if (after_start_space_bool) Space.space else Space.none;
825 const after_dots_space = if (slice.ast.end != 0) after_start_space else Space.none;826 const after_dots_space = if (slice.ast.end != 0)
827 after_start_space
828 else if (slice.ast.sentinel != 0) Space.space else Space.none;
826829
827 try renderExpression(gpa, ais, tree, slice.ast.sliced, .none);830 try renderExpression(gpa, ais, tree, slice.ast.sliced, .none);
828 try renderToken(ais, tree, slice.ast.lbracket, .none); // lbracket831 try renderToken(ais, tree, slice.ast.lbracket, .none); // lbracket
...@@ -830,20 +833,18 @@ fn renderSlice(...@@ -830,20 +833,18 @@ fn renderSlice(
830 const start_last = tree.lastToken(slice.ast.start);833 const start_last = tree.lastToken(slice.ast.start);
831 try renderExpression(gpa, ais, tree, slice.ast.start, after_start_space);834 try renderExpression(gpa, ais, tree, slice.ast.start, after_start_space);
832 try renderToken(ais, tree, start_last + 1, after_dots_space); // ellipsis2 ("..")835 try renderToken(ais, tree, start_last + 1, after_dots_space); // ellipsis2 ("..")
833 if (slice.ast.end == 0) {836
834 return renderToken(ais, tree, start_last + 2, space); // rbracket837 if (slice.ast.end != 0) {
838 const after_end_space = if (slice.ast.sentinel != 0) Space.space else Space.none;
839 try renderExpression(gpa, ais, tree, slice.ast.end, after_end_space);
835 }840 }
836841
837 const end_last = tree.lastToken(slice.ast.end);842 if (slice.ast.sentinel != 0) {
838 const after_end_space = if (slice.ast.sentinel != 0) Space.space else Space.none;843 try renderToken(ais, tree, tree.firstToken(slice.ast.sentinel) - 1, .none); // colon
839 try renderExpression(gpa, ais, tree, slice.ast.end, after_end_space);844 try renderExpression(gpa, ais, tree, slice.ast.sentinel, .none);
840 if (slice.ast.sentinel == 0) {
841 return renderToken(ais, tree, end_last + 1, space); // rbracket
842 }845 }
843846
844 try renderToken(ais, tree, end_last + 1, .none); // colon847 try renderToken(ais, tree, tree.lastToken(slice_node), space); // rbracket
845 try renderExpression(gpa, ais, tree, slice.ast.sentinel, .none);
846 try renderToken(ais, tree, tree.lastToken(slice.ast.sentinel) + 1, space); // rbracket
847}848}
848849
849fn renderAsmOutput(850fn renderAsmOutput(
src/astgen.zig+3-1
...@@ -848,7 +848,9 @@ pub fn blockExpr(...@@ -848,7 +848,9 @@ pub fn blockExpr(
848 const token_tags = tree.tokens.items(.tag);848 const token_tags = tree.tokens.items(.tag);
849849
850 const lbrace = main_tokens[block_node];850 const lbrace = main_tokens[block_node];
851 if (token_tags[lbrace - 1] == .colon) {851 if (token_tags[lbrace - 1] == .colon and
852 token_tags[lbrace - 2] == .identifier)
853 {
852 return labeledBlockExpr(mod, scope, rl, block_node, statements, .block);854 return labeledBlockExpr(mod, scope, rl, block_node, statements, .block);
853 }855 }
854856