authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-02-14 23:39:20-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-02-15 12:30:29-05:00
log5f5880979e8c5a7cfa4efdabad6358ceb89cc0e7
treea35163af2182869eb56dd1fbc031caf39be4d4d9
parentcc26148ba776f713bb81b5ac06fc646eb323e6dc

zig fmt supports simple line comments

3 files changed, 95 insertions(+), 24 deletions(-)

std/zig/ast.zig+22
...@@ -6,6 +6,7 @@ const mem = std.mem;...@@ -6,6 +6,7 @@ const mem = std.mem;
66
7pub const Node = struct {7pub const Node = struct {
8 id: Id,8 id: Id,
9 comment: ?&NodeLineComment,
910
10 pub const Id = enum {11 pub const Id = enum {
11 Root,12 Root,
...@@ -20,6 +21,7 @@ pub const Node = struct {...@@ -20,6 +21,7 @@ pub const Node = struct {
20 FloatLiteral,21 FloatLiteral,
21 StringLiteral,22 StringLiteral,
22 BuiltinCall,23 BuiltinCall,
24 LineComment,
23 };25 };
2426
25 pub fn iterate(base: &Node, index: usize) ?&Node {27 pub fn iterate(base: &Node, index: usize) ?&Node {
...@@ -36,6 +38,7 @@ pub const Node = struct {...@@ -36,6 +38,7 @@ pub const Node = struct {
36 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),38 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
37 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),39 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),
38 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),40 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
41 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index),
39 };42 };
40 }43 }
4144
...@@ -53,6 +56,7 @@ pub const Node = struct {...@@ -53,6 +56,7 @@ pub const Node = struct {
53 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),56 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
54 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),57 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),
55 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),58 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),
59 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(),
56 };60 };
57 }61 }
5862
...@@ -70,6 +74,7 @@ pub const Node = struct {...@@ -70,6 +74,7 @@ pub const Node = struct {
70 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),74 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
71 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),75 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),
72 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),76 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),
77 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(),
73 };78 };
74 }79 }
75};80};
...@@ -454,3 +459,20 @@ pub const NodeStringLiteral = struct {...@@ -454,3 +459,20 @@ pub const NodeStringLiteral = struct {
454 return self.token;459 return self.token;
455 }460 }
456};461};
462
463pub const NodeLineComment = struct {
464 base: Node,
465 lines: ArrayList(Token),
466
467 pub fn iterate(self: &NodeLineComment, index: usize) ?&Node {
468 return null;
469 }
470
471 pub fn firstToken(self: &NodeLineComment) Token {
472 return self.lines.at(0);
473 }
474
475 pub fn lastToken(self: &NodeLineComment) Token {
476 return self.lines.at(self.lines.len - 1);
477 }
478};
std/zig/parser.zig+70-13
...@@ -18,6 +18,7 @@ pub const Parser = struct {...@@ -18,6 +18,7 @@ pub const Parser = struct {
18 put_back_tokens: [2]Token,18 put_back_tokens: [2]Token,
19 put_back_count: usize,19 put_back_count: usize,
20 source_file_name: []const u8,20 source_file_name: []const u8,
21 pending_line_comment_node: ?&ast.NodeLineComment,
2122
22 pub const Tree = struct {23 pub const Tree = struct {
23 root_node: &ast.NodeRoot,24 root_node: &ast.NodeRoot,
...@@ -43,6 +44,7 @@ pub const Parser = struct {...@@ -43,6 +44,7 @@ pub const Parser = struct {
43 .put_back_count = 0,44 .put_back_count = 0,
44 .source_file_name = source_file_name,45 .source_file_name = source_file_name,
45 .utility_bytes = []align(utility_bytes_align) u8{},46 .utility_bytes = []align(utility_bytes_align) u8{},
47 .pending_line_comment_node = null,
46 };48 };
47 }49 }
4850
...@@ -131,6 +133,33 @@ pub const Parser = struct {...@@ -131,6 +133,33 @@ pub const Parser = struct {
131 // warn("\n");133 // warn("\n");
132 //}134 //}
133135
136 // look for line comments
137 while (true) {
138 const token = self.getNextToken();
139 if (token.id == Token.Id.LineComment) {
140 const node = blk: {
141 if (self.pending_line_comment_node) |comment_node| {
142 break :blk comment_node;
143 } else {
144 const comment_node = try arena.create(ast.NodeLineComment);
145 *comment_node = ast.NodeLineComment {
146 .base = ast.Node {
147 .id = ast.Node.Id.LineComment,
148 .comment = null,
149 },
150 .lines = ArrayList(Token).init(arena),
151 };
152 self.pending_line_comment_node = comment_node;
153 break :blk comment_node;
154 }
155 };
156 try node.lines.append(token);
157 continue;
158 }
159 self.putBackToken(token);
160 break;
161 }
162
134 // This gives us 1 free append that can't fail163 // This gives us 1 free append that can't fail
135 const state = stack.pop();164 const state = stack.pop();
136165
...@@ -329,7 +358,7 @@ pub const Parser = struct {...@@ -329,7 +358,7 @@ pub const Parser = struct {
329 Token.Id.Builtin => {358 Token.Id.Builtin => {
330 const node = try arena.create(ast.NodeBuiltinCall);359 const node = try arena.create(ast.NodeBuiltinCall);
331 *node = ast.NodeBuiltinCall {360 *node = ast.NodeBuiltinCall {
332 .base = ast.Node {.id = ast.Node.Id.BuiltinCall},361 .base = self.initNode(ast.Node.Id.BuiltinCall),
333 .builtin_token = token,362 .builtin_token = token,
334 .params = ArrayList(&ast.Node).init(arena),363 .params = ArrayList(&ast.Node).init(arena),
335 .rparen_token = undefined,364 .rparen_token = undefined,
...@@ -350,7 +379,7 @@ pub const Parser = struct {...@@ -350,7 +379,7 @@ pub const Parser = struct {
350 Token.Id.StringLiteral => {379 Token.Id.StringLiteral => {
351 const node = try arena.create(ast.NodeStringLiteral);380 const node = try arena.create(ast.NodeStringLiteral);
352 *node = ast.NodeStringLiteral {381 *node = ast.NodeStringLiteral {
353 .base = ast.Node {.id = ast.Node.Id.StringLiteral},382 .base = self.initNode(ast.Node.Id.StringLiteral),
354 .token = token,383 .token = token,
355 };384 };
356 try stack.append(State {385 try stack.append(State {
...@@ -359,6 +388,7 @@ pub const Parser = struct {...@@ -359,6 +388,7 @@ pub const Parser = struct {
359 try stack.append(State.AfterOperand);388 try stack.append(State.AfterOperand);
360 continue;389 continue;
361 },390 },
391
362 else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)),392 else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)),
363 }393 }
364 },394 },
...@@ -660,11 +690,19 @@ pub const Parser = struct {...@@ -660,11 +690,19 @@ pub const Parser = struct {
660 }690 }
661 }691 }
662692
693 fn initNode(self: &Parser, id: ast.Node.Id) ast.Node {
694 if (self.pending_line_comment_node) |comment_node| {
695 self.pending_line_comment_node = null;
696 return ast.Node {.id = id, .comment = comment_node};
697 }
698 return ast.Node {.id = id, .comment = null };
699 }
700
663 fn createRoot(self: &Parser, arena: &mem.Allocator) !&ast.NodeRoot {701 fn createRoot(self: &Parser, arena: &mem.Allocator) !&ast.NodeRoot {
664 const node = try arena.create(ast.NodeRoot);702 const node = try arena.create(ast.NodeRoot);
665703
666 *node = ast.NodeRoot {704 *node = ast.NodeRoot {
667 .base = ast.Node {.id = ast.Node.Id.Root},705 .base = self.initNode(ast.Node.Id.Root),
668 .decls = ArrayList(&ast.Node).init(arena),706 .decls = ArrayList(&ast.Node).init(arena),
669 // initialized when we get the eof token707 // initialized when we get the eof token
670 .eof_token = undefined,708 .eof_token = undefined,
...@@ -678,7 +716,7 @@ pub const Parser = struct {...@@ -678,7 +716,7 @@ pub const Parser = struct {
678 const node = try arena.create(ast.NodeVarDecl);716 const node = try arena.create(ast.NodeVarDecl);
679717
680 *node = ast.NodeVarDecl {718 *node = ast.NodeVarDecl {
681 .base = ast.Node {.id = ast.Node.Id.VarDecl},719 .base = self.initNode(ast.Node.Id.VarDecl),
682 .visib_token = *visib_token,720 .visib_token = *visib_token,
683 .mut_token = *mut_token,721 .mut_token = *mut_token,
684 .comptime_token = *comptime_token,722 .comptime_token = *comptime_token,
...@@ -701,7 +739,7 @@ pub const Parser = struct {...@@ -701,7 +739,7 @@ pub const Parser = struct {
701 const node = try arena.create(ast.NodeFnProto);739 const node = try arena.create(ast.NodeFnProto);
702740
703 *node = ast.NodeFnProto {741 *node = ast.NodeFnProto {
704 .base = ast.Node {.id = ast.Node.Id.FnProto},742 .base = self.initNode(ast.Node.Id.FnProto),
705 .visib_token = *visib_token,743 .visib_token = *visib_token,
706 .name_token = null,744 .name_token = null,
707 .fn_token = *fn_token,745 .fn_token = *fn_token,
...@@ -722,7 +760,7 @@ pub const Parser = struct {...@@ -722,7 +760,7 @@ pub const Parser = struct {
722 const node = try arena.create(ast.NodeParamDecl);760 const node = try arena.create(ast.NodeParamDecl);
723761
724 *node = ast.NodeParamDecl {762 *node = ast.NodeParamDecl {
725 .base = ast.Node {.id = ast.Node.Id.ParamDecl},763 .base = self.initNode(ast.Node.Id.ParamDecl),
726 .comptime_token = null,764 .comptime_token = null,
727 .noalias_token = null,765 .noalias_token = null,
728 .name_token = null,766 .name_token = null,
...@@ -736,7 +774,7 @@ pub const Parser = struct {...@@ -736,7 +774,7 @@ pub const Parser = struct {
736 const node = try arena.create(ast.NodeBlock);774 const node = try arena.create(ast.NodeBlock);
737775
738 *node = ast.NodeBlock {776 *node = ast.NodeBlock {
739 .base = ast.Node {.id = ast.Node.Id.Block},777 .base = self.initNode(ast.Node.Id.Block),
740 .begin_token = *begin_token,778 .begin_token = *begin_token,
741 .end_token = undefined,779 .end_token = undefined,
742 .statements = ArrayList(&ast.Node).init(arena),780 .statements = ArrayList(&ast.Node).init(arena),
...@@ -748,7 +786,7 @@ pub const Parser = struct {...@@ -748,7 +786,7 @@ pub const Parser = struct {
748 const node = try arena.create(ast.NodeInfixOp);786 const node = try arena.create(ast.NodeInfixOp);
749787
750 *node = ast.NodeInfixOp {788 *node = ast.NodeInfixOp {
751 .base = ast.Node {.id = ast.Node.Id.InfixOp},789 .base = self.initNode(ast.Node.Id.InfixOp),
752 .op_token = *op_token,790 .op_token = *op_token,
753 .lhs = undefined,791 .lhs = undefined,
754 .op = *op,792 .op = *op,
...@@ -761,7 +799,7 @@ pub const Parser = struct {...@@ -761,7 +799,7 @@ pub const Parser = struct {
761 const node = try arena.create(ast.NodePrefixOp);799 const node = try arena.create(ast.NodePrefixOp);
762800
763 *node = ast.NodePrefixOp {801 *node = ast.NodePrefixOp {
764 .base = ast.Node {.id = ast.Node.Id.PrefixOp},802 .base = self.initNode(ast.Node.Id.PrefixOp),
765 .op_token = *op_token,803 .op_token = *op_token,
766 .op = *op,804 .op = *op,
767 .rhs = undefined,805 .rhs = undefined,
...@@ -773,7 +811,7 @@ pub const Parser = struct {...@@ -773,7 +811,7 @@ pub const Parser = struct {
773 const node = try arena.create(ast.NodeIdentifier);811 const node = try arena.create(ast.NodeIdentifier);
774812
775 *node = ast.NodeIdentifier {813 *node = ast.NodeIdentifier {
776 .base = ast.Node {.id = ast.Node.Id.Identifier},814 .base = self.initNode(ast.Node.Id.Identifier),
777 .name_token = *name_token,815 .name_token = *name_token,
778 };816 };
779 return node;817 return node;
...@@ -783,7 +821,7 @@ pub const Parser = struct {...@@ -783,7 +821,7 @@ pub const Parser = struct {
783 const node = try arena.create(ast.NodeIntegerLiteral);821 const node = try arena.create(ast.NodeIntegerLiteral);
784822
785 *node = ast.NodeIntegerLiteral {823 *node = ast.NodeIntegerLiteral {
786 .base = ast.Node {.id = ast.Node.Id.IntegerLiteral},824 .base = self.initNode(ast.Node.Id.IntegerLiteral),
787 .token = *token,825 .token = *token,
788 };826 };
789 return node;827 return node;
...@@ -793,7 +831,7 @@ pub const Parser = struct {...@@ -793,7 +831,7 @@ pub const Parser = struct {
793 const node = try arena.create(ast.NodeFloatLiteral);831 const node = try arena.create(ast.NodeFloatLiteral);
794832
795 *node = ast.NodeFloatLiteral {833 *node = ast.NodeFloatLiteral {
796 .base = ast.Node {.id = ast.Node.Id.FloatLiteral},834 .base = self.initNode(ast.Node.Id.FloatLiteral),
797 .token = *token,835 .token = *token,
798 };836 };
799 return node;837 return node;
...@@ -1158,9 +1196,11 @@ pub const Parser = struct {...@@ -1158,9 +1196,11 @@ pub const Parser = struct {
1158 }1196 }
1159 }1197 }
1160 },1198 },
1199 ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"),
1200 ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"),
1201
1161 ast.Node.Id.Root,1202 ast.Node.Id.Root,
1162 ast.Node.Id.VarDecl,1203 ast.Node.Id.VarDecl,
1163 ast.Node.Id.FnProto,
1164 ast.Node.Id.ParamDecl => unreachable,1204 ast.Node.Id.ParamDecl => unreachable,
1165 },1205 },
1166 RenderState.FnProtoRParen => |fn_proto| {1206 RenderState.FnProtoRParen => |fn_proto| {
...@@ -1187,6 +1227,12 @@ pub const Parser = struct {...@@ -1187,6 +1227,12 @@ pub const Parser = struct {
1187 }1227 }
1188 },1228 },
1189 RenderState.Statement => |base| {1229 RenderState.Statement => |base| {
1230 if (base.comment) |comment| {
1231 for (comment.lines.toSliceConst()) |line_token| {
1232 try stream.print("{}\n", self.tokenizer.getTokenSlice(line_token));
1233 try stream.writeByteNTimes(' ', indent);
1234 }
1235 }
1190 switch (base.id) {1236 switch (base.id) {
1191 ast.Node.Id.VarDecl => {1237 ast.Node.Id.VarDecl => {
1192 const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base);1238 const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base);
...@@ -1279,6 +1325,17 @@ fn testCanonical(source: []const u8) !void {...@@ -1279,6 +1325,17 @@ fn testCanonical(source: []const u8) !void {
1279}1325}
12801326
1281test "zig fmt" {1327test "zig fmt" {
1328 try testCanonical(
1329 \\const std = @import("std");
1330 \\
1331 \\pub fn main() !void {
1332 \\ // If this program is run without stdout attached, exit with an error.
1333 \\ // another comment
1334 \\ var stdout_file = try std.io.getStdOut;
1335 \\}
1336 \\
1337 );
1338
1282 try testCanonical(1339 try testCanonical(
1283 \\const std = @import("std");1340 \\const std = @import("std");
1284 \\1341 \\
std/zig/tokenizer.zig+3-11
...@@ -99,6 +99,7 @@ pub const Token = struct {...@@ -99,6 +99,7 @@ pub const Token = struct {
99 AmpersandEqual,99 AmpersandEqual,
100 IntegerLiteral,100 IntegerLiteral,
101 FloatLiteral,101 FloatLiteral,
102 LineComment,
102 Keyword_align,103 Keyword_align,
103 Keyword_and,104 Keyword_and,
104 Keyword_asm,105 Keyword_asm,
...@@ -470,7 +471,7 @@ pub const Tokenizer = struct {...@@ -470,7 +471,7 @@ pub const Tokenizer = struct {
470471
471 State.Slash => switch (c) {472 State.Slash => switch (c) {
472 '/' => {473 '/' => {
473 result.id = undefined;474 result.id = Token.Id.LineComment;
474 state = State.LineComment;475 state = State.LineComment;
475 },476 },
476 else => {477 else => {
...@@ -479,16 +480,7 @@ pub const Tokenizer = struct {...@@ -479,16 +480,7 @@ pub const Tokenizer = struct {
479 },480 },
480 },481 },
481 State.LineComment => switch (c) {482 State.LineComment => switch (c) {
482 '\n' => {483 '\n' => break,
483 state = State.Start;
484 result = Token {
485 .id = Token.Id.Eof,
486 .start = self.index + 1,
487 .column = 0,
488 .line = self.line + 1,
489 .end = undefined,
490 };
491 },
492 else => self.checkLiteralCharacter(),484 else => self.checkLiteralCharacter(),
493 },485 },
494 State.Zero => switch (c) {486 State.Zero => switch (c) {