Commit 57cec38e6144754fcd15266100974a7cf0059570 | zig

author	Isaac Freund <mail@isaacfreund.com> 2021-02-07 23:14:33+01:00
committer	Andrew Kelley <andrew@ziglang.org> 2021-02-07 14:51:37-08:00
log	57cec38e6144754fcd15266100974a7cf0059570
tree	0e84bb83b0802b7643236cf64585a7c46d7e6543
parent	0e38362d244f118565f447f8ee2c6b8a700d05bf

std/zig/ast: fix Tree.lastToken() for blocks

The fact that blocks may end in a semicolon but this semicolon is not
counted by recursive lastToken() evaluation on the sub expression causes
off-by-one errors for lastToken() on blocks currently.

To fix this, introduce BlockSemicolon and BlockTwoSemicolon following
the pattern used for trailing commas in e.g. builtin function arguments.

4 files changed, 42 insertions(+), 11 deletions(-)

lib
- std
  - zig
    ast.zig
    parse.zig
    parser_test.zig
    render.zig

lib/std/zig/ast.zig+12-6

 @@ -357,7 +357,9 @@ pub const Tree = struct {
             },
             .Block,
+            .BlockSemicolon,
             .BlockTwo,
+            .BlockTwoSemicolon,
             => {
                 // Look for a label.
                 const lbrace = main_tokens[n];
 @@ -552,18 +554,17 @@ pub const Tree = struct {
             .TaggedUnion,
             .BuiltinCall,
             => {
+                assert(datas[n].rhs - datas[n].lhs > 0);
                 end_offset += 1; // for the rbrace
-                if (datas[n].rhs - datas[n].lhs == 0) {
-                    return main_tokens[n] + end_offset;
-                }
                 n = tree.extra_data[datas[n].rhs - 1]; // last statement
             },
+            .BlockSemicolon,
             .ContainerDeclComma,
             .TaggedUnionComma,
             .BuiltinCallComma,
             => {
                 assert(datas[n].rhs - datas[n].lhs > 0);
-                end_offset += 2; // for the comma + rbrace/rparen
+                end_offset += 2; // for the comma/semicolon + rbrace/rparen
                 n = tree.extra_data[datas[n].rhs - 1]; // last member
             },
             .CallOne,
 @@ -594,11 +595,12 @@ pub const Tree = struct {
             },
             .ArrayInitDotTwoComma,
             .BuiltinCallTwoComma,
+            .BlockTwoSemicolon,
             .StructInitDotTwoComma,
             .ContainerDeclTwoComma,
             .TaggedUnionTwoComma,
             => {
-                end_offset += 2; // for the comma + rbrace/rparen
+                end_offset += 2; // for the comma/semicolon + rbrace/rparen
                 if (datas[n].rhs != 0) {
                     n = datas[n].rhs;
                 } else if (datas[n].lhs != 0) {
 @@ -2137,12 +2139,16 @@ pub const Node = struct {
         Comptime,
         /// `nosuspend lhs`. rhs unused.
         Nosuspend,
-        /// `{lhs; rhs;}`. rhs or lhs can be omitted.
+        /// `{lhs rhs}`. rhs or lhs can be omitted.
         /// main_token points at the lbrace.
         BlockTwo,
+        /// Same as BlockTwo but there is known to be a semicolon before the rbrace.
+        BlockTwoSemicolon,
         /// `{}`. `sub_list[lhs..rhs]`.
         /// main_token points at the lbrace.
         Block,
+        /// Same as BlockTwo but there is known to be a semicolon before the rbrace.
+        BlockSemicolon,
         /// `asm(lhs)`. rhs unused.
         AsmSimple,
         /// `asm(lhs, a)`. `sub_range_list[rhs]`.

lib/std/zig/parse.zig+6-3

 @@ -1984,8 +1984,9 @@ const Parser = struct {
         const stmt_one = try p.expectStatementRecoverable();
         if (p.eatToken(.RBrace)) |_| {
+            const semicolon = p.token_tags[p.tok_i - 2] == .Semicolon;
             return p.addNode(.{
-                .tag = .BlockTwo,
+                .tag = if (semicolon) .BlockTwoSemicolon else .BlockTwo,
                 .main_token = lbrace,
                 .data = .{
                     .lhs = stmt_one,
 @@ -1995,8 +1996,9 @@ const Parser = struct {
         }
         const stmt_two = try p.expectStatementRecoverable();
         if (p.eatToken(.RBrace)) |_| {
+            const semicolon = p.token_tags[p.tok_i - 2] == .Semicolon;
             return p.addNode(.{
-                .tag = .BlockTwo,
+                .tag = if (semicolon) .BlockTwoSemicolon else .BlockTwo,
                 .main_token = lbrace,
                 .data = .{
                     .lhs = stmt_one,
 @@ -2017,9 +2019,10 @@ const Parser = struct {
             if (p.token_tags[p.tok_i] == .RBrace) break;
         }
         _ = try p.expectToken(.RBrace);
+        const semicolon = p.token_tags[p.tok_i - 2] == .Semicolon;
         const statements_span = try p.listToSpan(statements.items);
         return p.addNode(.{
-            .tag = .Block,
+            .tag = if (semicolon) .BlockSemicolon else .Block,
             .main_token = lbrace,
             .data = .{
                 .lhs = statements_span.start,

lib/std/zig/parser_test.zig+18

...	@@ -655,6 +655,24 @@ test "zig fmt: slices with spaces in bounds" {	...	@@ -655,6 +655,24 @@ test "zig fmt: slices with spaces in bounds" {
655	);	655	);
656	}	656	}
657		657
		658	test "zig fmt: block in slice expression" {
		659	try testCanonical(
		660	\\const a = b[{
		661	\\ _ = x;
		662	\\}..];
		663	\\const c = d[0..{
		664	\\ _ = x;
		665	\\ _ = y;
		666	\\}];
		667	\\const e = f[0..1 :{
		668	\\ _ = x;
		669	\\ _ = y;
		670	\\ _ = z;
		671	\\}];
		672	\\
		673	);
		674	}
		675
658	//test "zig fmt: async function" {	676	//test "zig fmt: async function" {
659	// try testCanonical(	677	// try testCanonical(
660	// \\pub const Server = struct {	678	// \\pub const Server = struct {

lib/std/zig/render.zig+6-2

 @@ -202,7 +202,9 @@ fn renderExpression(ais: *Ais, tree: ast.Tree, node: ast.Node.Index, space: Spac
         //    }
         //    return renderToken(ais, tree, any_type.token, space);
         //},
-        .BlockTwo => {
+        .BlockTwo,
+        .BlockTwoSemicolon,
+        => {
             const statements = [2]ast.Node.Index{ datas[node].lhs, datas[node].rhs };
             if (datas[node].lhs == 0) {
                 return renderBlock(ais, tree, main_tokens[node], statements[0..0], space);
 @@ -212,7 +214,9 @@ fn renderExpression(ais: *Ais, tree: ast.Tree, node: ast.Node.Index, space: Spac
                 return renderBlock(ais, tree, main_tokens[node], statements[0..2], space);
             }
         },
-        .Block => {
+        .Block,
+        .BlockSemicolon,
+        => {
             const lbrace = main_tokens[node];
             const statements = tree.extra_data[datas[node].lhs..datas[node].rhs];
             return renderBlock(ais, tree, main_tokens[node], statements, space);