authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-04-30 00:19:55-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-04-30 00:19:55-04:00
logfd2cd38bdb831ef78a0d4ab0973020dfbd348c1f
tree58f228d2a8c7ab6d00e7f18e9e0116a699822657
parent39befc35a851f6cf206c7e3b19d66f7df00440b0

zig fmt: support line comments and doc comments

line comments can go anywhere a list of something is allowed

4 files changed, 168 insertions(+), 56 deletions(-)

std/zig/ast.zig+24-5
......@@ -6,7 +6,7 @@ const mem = std.mem;
66
77pub const Node = struct {
88 id: Id,
9 before_comments: ?&LineComment,
9 doc_comments: ?&DocComment,
1010 same_line_comment: ?&Token,
1111
1212 pub const Id = enum {
......@@ -59,6 +59,7 @@ pub const Node = struct {
5959
6060 // Misc
6161 LineComment,
62 DocComment,
6263 SwitchCase,
6364 SwitchElse,
6465 Else,
......@@ -718,7 +719,8 @@ pub const Node = struct {
718719 base: Node,
719720 switch_token: Token,
720721 expr: &Node,
721 cases: ArrayList(&SwitchCase),
722 /// these can be SwitchCase nodes or LineComment nodes
723 cases: ArrayList(&Node),
722724 rbrace: Token,
723725
724726 pub fn iterate(self: &Switch, index: usize) ?&Node {
......@@ -727,7 +729,7 @@ pub const Node = struct {
727729 if (i < 1) return self.expr;
728730 i -= 1;
729731
730 if (i < self.cases.len) return &self.cases.at(i).base;
732 if (i < self.cases.len) return self.cases.at(i);
731733 i -= self.cases.len;
732734
733735 return null;
......@@ -1715,17 +1717,34 @@ pub const Node = struct {
17151717
17161718 pub const LineComment = struct {
17171719 base: Node,
1718 lines: ArrayList(Token),
1720 token: Token,
17191721
17201722 pub fn iterate(self: &LineComment, index: usize) ?&Node {
17211723 return null;
17221724 }
17231725
17241726 pub fn firstToken(self: &LineComment) Token {
1725 return self.lines.at(0);
1727 return self.token;
17261728 }
17271729
17281730 pub fn lastToken(self: &LineComment) Token {
1731 return self.token;
1732 }
1733 };
1734
1735 pub const DocComment = struct {
1736 base: Node,
1737 lines: ArrayList(Token),
1738
1739 pub fn iterate(self: &DocComment, index: usize) ?&Node {
1740 return null;
1741 }
1742
1743 pub fn firstToken(self: &DocComment) Token {
1744 return self.lines.at(0);
1745 }
1746
1747 pub fn lastToken(self: &DocComment) Token {
17291748 return self.lines.at(self.lines.len - 1);
17301749 }
17311750 };
std/zig/parser.zig+96-46
......@@ -55,7 +55,7 @@ pub const Parser = struct {
5555 visib_token: ?Token,
5656 extern_export_inline_token: ?Token,
5757 lib_name: ?&ast.Node,
58 comments: ?&ast.Node.LineComment,
58 comments: ?&ast.Node.DocComment,
5959 };
6060
6161 const VarDeclCtx = struct {
......@@ -65,19 +65,19 @@ pub const Parser = struct {
6565 extern_export_token: ?Token,
6666 lib_name: ?&ast.Node,
6767 list: &ArrayList(&ast.Node),
68 comments: ?&ast.Node.LineComment,
68 comments: ?&ast.Node.DocComment,
6969 };
7070
7171 const TopLevelExternOrFieldCtx = struct {
7272 visib_token: Token,
7373 container_decl: &ast.Node.ContainerDecl,
74 comments: ?&ast.Node.LineComment,
74 comments: ?&ast.Node.DocComment,
7575 };
7676
7777 const ExternTypeCtx = struct {
7878 opt_ctx: OptionalCtx,
7979 extern_token: Token,
80 comments: ?&ast.Node.LineComment,
80 comments: ?&ast.Node.DocComment,
8181 };
8282
8383 const ContainerKindCtx = struct {
......@@ -186,7 +186,7 @@ pub const Parser = struct {
186186
187187 const AddCommentsCtx = struct {
188188 node_ptr: &&ast.Node,
189 comments: ?&ast.Node.LineComment,
189 comments: ?&ast.Node.DocComment,
190190 };
191191
192192 const State = union(enum) {
......@@ -244,8 +244,8 @@ pub const Parser = struct {
244244 FieldListCommaOrEnd: &ast.Node.ContainerDecl,
245245 IdentifierListItemOrEnd: ListSave(&ast.Node),
246246 IdentifierListCommaOrEnd: ListSave(&ast.Node),
247 SwitchCaseOrEnd: ListSave(&ast.Node.SwitchCase),
248 SwitchCaseCommaOrEnd: ListSave(&ast.Node.SwitchCase),
247 SwitchCaseOrEnd: ListSave(&ast.Node),
248 SwitchCaseCommaOrEnd: ListSave(&ast.Node),
249249 SwitchCaseFirstItem: &ArrayList(&ast.Node),
250250 SwitchCaseItem: &ArrayList(&ast.Node),
251251 SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node),
......@@ -349,6 +349,10 @@ pub const Parser = struct {
349349
350350 switch (state) {
351351 State.TopLevel => {
352 while (try self.eatLineComment(arena)) |line_comment| {
353 try root_node.decls.append(&line_comment.base);
354 }
355
352356 const comments = try self.eatComments(arena);
353357 const token = self.getNextToken();
354358 switch (token.id) {
......@@ -358,7 +362,7 @@ pub const Parser = struct {
358362 const block = try arena.construct(ast.Node.Block {
359363 .base = ast.Node {
360364 .id = ast.Node.Id.Block,
361 .before_comments = null,
365 .doc_comments = null,
362366 .same_line_comment = null,
363367 },
364368 .label = null,
......@@ -369,7 +373,7 @@ pub const Parser = struct {
369373 const test_node = try arena.construct(ast.Node.TestDecl {
370374 .base = ast.Node {
371375 .id = ast.Node.Id.TestDecl,
372 .before_comments = comments,
376 .doc_comments = comments,
373377 .same_line_comment = null,
374378 },
375379 .test_token = token,
......@@ -551,7 +555,7 @@ pub const Parser = struct {
551555 const fn_proto = try arena.construct(ast.Node.FnProto {
552556 .base = ast.Node {
553557 .id = ast.Node.Id.FnProto,
554 .before_comments = ctx.comments,
558 .doc_comments = ctx.comments,
555559 .same_line_comment = null,
556560 },
557561 .visib_token = ctx.visib_token,
......@@ -620,7 +624,7 @@ pub const Parser = struct {
620624 const node = try arena.construct(ast.Node.StructField {
621625 .base = ast.Node {
622626 .id = ast.Node.Id.StructField,
623 .before_comments = null,
627 .doc_comments = null,
624628 .same_line_comment = null,
625629 },
626630 .visib_token = ctx.visib_token,
......@@ -706,6 +710,10 @@ pub const Parser = struct {
706710 continue;
707711 },
708712 State.ContainerDecl => |container_decl| {
713 while (try self.eatLineComment(arena)) |line_comment| {
714 try container_decl.fields_and_decls.append(&line_comment.base);
715 }
716
709717 const comments = try self.eatComments(arena);
710718 const token = self.getNextToken();
711719 switch (token.id) {
......@@ -715,7 +723,7 @@ pub const Parser = struct {
715723 const node = try arena.construct(ast.Node.StructField {
716724 .base = ast.Node {
717725 .id = ast.Node.Id.StructField,
718 .before_comments = comments,
726 .doc_comments = comments,
719727 .same_line_comment = null,
720728 },
721729 .visib_token = null,
......@@ -826,7 +834,7 @@ pub const Parser = struct {
826834 const var_decl = try arena.construct(ast.Node.VarDecl {
827835 .base = ast.Node {
828836 .id = ast.Node.Id.VarDecl,
829 .before_comments = ctx.comments,
837 .doc_comments = ctx.comments,
830838 .same_line_comment = null,
831839 },
832840 .visib_token = ctx.visib_token,
......@@ -1222,6 +1230,14 @@ pub const Parser = struct {
12221230 else => {
12231231 self.putBackToken(token);
12241232 stack.append(State { .Block = block }) catch unreachable;
1233
1234 var any_comments = false;
1235 while (try self.eatLineComment(arena)) |line_comment| {
1236 try block.statements.append(&line_comment.base);
1237 any_comments = true;
1238 }
1239 if (any_comments) continue;
1240
12251241 try stack.append(State { .Statement = block });
12261242 continue;
12271243 },
......@@ -1258,7 +1274,7 @@ pub const Parser = struct {
12581274 const node = try arena.construct(ast.Node.Defer {
12591275 .base = ast.Node {
12601276 .id = ast.Node.Id.Defer,
1261 .before_comments = comments,
1277 .doc_comments = comments,
12621278 .same_line_comment = null,
12631279 },
12641280 .defer_token = token,
......@@ -1342,7 +1358,7 @@ pub const Parser = struct {
13421358
13431359 State.AddComments => |add_comments_ctx| {
13441360 const node = *add_comments_ctx.node_ptr;
1345 node.before_comments = add_comments_ctx.comments;
1361 node.doc_comments = add_comments_ctx.comments;
13461362 continue;
13471363 },
13481364
......@@ -1466,7 +1482,7 @@ pub const Parser = struct {
14661482 const node = try arena.construct(ast.Node.FieldInitializer {
14671483 .base = ast.Node {
14681484 .id = ast.Node.Id.FieldInitializer,
1469 .before_comments = null,
1485 .doc_comments = null,
14701486 .same_line_comment = null,
14711487 },
14721488 .period_token = undefined,
......@@ -1512,6 +1528,10 @@ pub const Parser = struct {
15121528 continue;
15131529 },
15141530 State.IdentifierListItemOrEnd => |list_state| {
1531 while (try self.eatLineComment(arena)) |line_comment| {
1532 try list_state.list.append(&line_comment.base);
1533 }
1534
15151535 if (self.eatToken(Token.Id.RBrace)) |rbrace| {
15161536 *list_state.ptr = rbrace;
15171537 continue;
......@@ -1538,6 +1558,10 @@ pub const Parser = struct {
15381558 }
15391559 },
15401560 State.SwitchCaseOrEnd => |list_state| {
1561 while (try self.eatLineComment(arena)) |line_comment| {
1562 try list_state.list.append(&line_comment.base);
1563 }
1564
15411565 if (self.eatToken(Token.Id.RBrace)) |rbrace| {
15421566 *list_state.ptr = rbrace;
15431567 continue;
......@@ -1547,14 +1571,14 @@ pub const Parser = struct {
15471571 const node = try arena.construct(ast.Node.SwitchCase {
15481572 .base = ast.Node {
15491573 .id = ast.Node.Id.SwitchCase,
1550 .before_comments = comments,
1574 .doc_comments = comments,
15511575 .same_line_comment = null,
15521576 },
15531577 .items = ArrayList(&ast.Node).init(arena),
15541578 .payload = null,
15551579 .expr = undefined,
15561580 });
1557 try list_state.list.append(node);
1581 try list_state.list.append(&node.base);
15581582 try stack.append(State { .SwitchCaseCommaOrEnd = list_state });
15591583 try stack.append(State { .AssignmentExpressionBegin = OptionalCtx { .Required = &node.expr } });
15601584 try stack.append(State { .PointerPayload = OptionalCtx { .Optional = &node.payload } });
......@@ -1569,8 +1593,8 @@ pub const Parser = struct {
15691593 continue;
15701594 }
15711595
1572 const switch_case = list_state.list.toSlice()[list_state.list.len - 1];
1573 try self.lookForSameLineComment(arena, &switch_case.base);
1596 const node = list_state.list.toSlice()[list_state.list.len - 1];
1597 try self.lookForSameLineComment(arena, node);
15741598 try stack.append(State { .SwitchCaseOrEnd = list_state });
15751599 continue;
15761600 },
......@@ -1660,7 +1684,7 @@ pub const Parser = struct {
16601684 const fn_proto = try arena.construct(ast.Node.FnProto {
16611685 .base = ast.Node {
16621686 .id = ast.Node.Id.FnProto,
1663 .before_comments = ctx.comments,
1687 .doc_comments = ctx.comments,
16641688 .same_line_comment = null,
16651689 },
16661690 .visib_token = null,
......@@ -2632,7 +2656,7 @@ pub const Parser = struct {
26322656 const fn_proto = try arena.construct(ast.Node.FnProto {
26332657 .base = ast.Node {
26342658 .id = ast.Node.Id.FnProto,
2635 .before_comments = null,
2659 .doc_comments = null,
26362660 .same_line_comment = null,
26372661 },
26382662 .visib_token = null,
......@@ -2656,7 +2680,7 @@ pub const Parser = struct {
26562680 const fn_proto = try arena.construct(ast.Node.FnProto {
26572681 .base = ast.Node {
26582682 .id = ast.Node.Id.FnProto,
2659 .before_comments = null,
2683 .doc_comments = null,
26602684 .same_line_comment = null,
26612685 },
26622686 .visib_token = null,
......@@ -2749,7 +2773,7 @@ pub const Parser = struct {
27492773 const node = try arena.construct(ast.Node.ErrorSetDecl {
27502774 .base = ast.Node {
27512775 .id = ast.Node.Id.ErrorSetDecl,
2752 .before_comments = null,
2776 .doc_comments = null,
27532777 .same_line_comment = null,
27542778 },
27552779 .error_token = ctx.error_token,
......@@ -2829,18 +2853,18 @@ pub const Parser = struct {
28292853 }
28302854 }
28312855
2832 fn eatComments(self: &Parser, arena: &mem.Allocator) !?&ast.Node.LineComment {
2833 var result: ?&ast.Node.LineComment = null;
2856 fn eatComments(self: &Parser, arena: &mem.Allocator) !?&ast.Node.DocComment {
2857 var result: ?&ast.Node.DocComment = null;
28342858 while (true) {
2835 if (self.eatToken(Token.Id.LineComment)) |line_comment| {
2859 if (self.eatToken(Token.Id.DocComment)) |line_comment| {
28362860 const node = blk: {
28372861 if (result) |comment_node| {
28382862 break :blk comment_node;
28392863 } else {
2840 const comment_node = try arena.construct(ast.Node.LineComment {
2864 const comment_node = try arena.construct(ast.Node.DocComment {
28412865 .base = ast.Node {
2842 .id = ast.Node.Id.LineComment,
2843 .before_comments = null,
2866 .id = ast.Node.Id.DocComment,
2867 .doc_comments = null,
28442868 .same_line_comment = null,
28452869 },
28462870 .lines = ArrayList(Token).init(arena),
......@@ -2857,6 +2881,18 @@ pub const Parser = struct {
28572881 return result;
28582882 }
28592883
2884 fn eatLineComment(self: &Parser, arena: &mem.Allocator) !?&ast.Node.LineComment {
2885 const token = self.eatToken(Token.Id.LineComment) ?? return null;
2886 return try arena.construct(ast.Node.LineComment {
2887 .base = ast.Node {
2888 .id = ast.Node.Id.LineComment,
2889 .doc_comments = null,
2890 .same_line_comment = null,
2891 },
2892 .token = token,
2893 });
2894 }
2895
28602896 fn requireSemiColon(node: &const ast.Node) bool {
28612897 var n = node;
28622898 while (true) {
......@@ -2874,6 +2910,7 @@ pub const Parser = struct {
28742910 ast.Node.Id.SwitchCase,
28752911 ast.Node.Id.SwitchElse,
28762912 ast.Node.Id.FieldInitializer,
2913 ast.Node.Id.DocComment,
28772914 ast.Node.Id.LineComment,
28782915 ast.Node.Id.TestDecl => return false,
28792916 ast.Node.Id.While => {
......@@ -2933,7 +2970,7 @@ pub const Parser = struct {
29332970 const node_last_token = node.lastToken();
29342971
29352972 const line_comment_token = self.getNextToken();
2936 if (line_comment_token.id != Token.Id.LineComment) {
2973 if (line_comment_token.id != Token.Id.DocComment and line_comment_token.id != Token.Id.LineComment) {
29372974 self.putBackToken(line_comment_token);
29382975 return;
29392976 }
......@@ -3038,18 +3075,21 @@ pub const Parser = struct {
30383075 return true;
30393076 },
30403077 Token.Id.Keyword_switch => {
3041 const node = try self.createToCtxNode(arena, ctx, ast.Node.Switch,
3042 ast.Node.Switch {
3043 .base = undefined,
3044 .switch_token = *token,
3045 .expr = undefined,
3046 .cases = ArrayList(&ast.Node.SwitchCase).init(arena),
3047 .rbrace = undefined,
3048 }
3049 );
3078 const node = try arena.construct(ast.Node.Switch {
3079 .base = ast.Node {
3080 .id = ast.Node.Id.Switch,
3081 .doc_comments = null,
3082 .same_line_comment = null,
3083 },
3084 .switch_token = *token,
3085 .expr = undefined,
3086 .cases = ArrayList(&ast.Node).init(arena),
3087 .rbrace = undefined,
3088 });
3089 ctx.store(&node.base);
30503090
30513091 stack.append(State {
3052 .SwitchCaseOrEnd = ListSave(&ast.Node.SwitchCase) {
3092 .SwitchCaseOrEnd = ListSave(&ast.Node) {
30533093 .list = &node.cases,
30543094 .ptr = &node.rbrace,
30553095 },
......@@ -3208,7 +3248,7 @@ pub const Parser = struct {
32083248 const id = ast.Node.typeToId(T);
32093249 break :blk ast.Node {
32103250 .id = id,
3211 .before_comments = null,
3251 .doc_comments = null,
32123252 .same_line_comment = null,
32133253 };
32143254 };
......@@ -3454,6 +3494,10 @@ pub const Parser = struct {
34543494 }
34553495 try stack.append(RenderState { .Expression = decl });
34563496 },
3497 ast.Node.Id.LineComment => {
3498 const line_comment_node = @fieldParentPtr(ast.Node.LineComment, "base", decl);
3499 try stream.write(self.tokenizer.getTokenSlice(line_comment_node.token));
3500 },
34573501 else => unreachable,
34583502 }
34593503 },
......@@ -3987,7 +4031,9 @@ pub const Parser = struct {
39874031 while (i != 0) {
39884032 i -= 1;
39894033 const node = decls[i];
3990 try stack.append(RenderState { .Text = "," });
4034 if (node.id != ast.Node.Id.LineComment) {
4035 try stack.append(RenderState { .Text = "," });
4036 }
39914037 try stack.append(RenderState { .Expression = node });
39924038 try stack.append(RenderState { .PrintComments = node });
39934039 try stack.append(RenderState.PrintIndent);
......@@ -4100,7 +4146,11 @@ pub const Parser = struct {
41004146 try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(visib_token) });
41014147 }
41024148 },
4103 ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"),
4149 ast.Node.Id.LineComment => {
4150 const line_comment_node = @fieldParentPtr(ast.Node.LineComment, "base", base);
4151 try stream.write(self.tokenizer.getTokenSlice(line_comment_node.token));
4152 },
4153 ast.Node.Id.DocComment => unreachable, // doc comments are attached to nodes
41044154 ast.Node.Id.Switch => {
41054155 const switch_node = @fieldParentPtr(ast.Node.Switch, "base", base);
41064156 try stream.print("{} (", self.tokenizer.getTokenSlice(switch_node.switch_token));
......@@ -4115,7 +4165,7 @@ pub const Parser = struct {
41154165 while (i != 0) {
41164166 i -= 1;
41174167 const node = cases[i];
4118 try stack.append(RenderState { .Expression = &node.base});
4168 try stack.append(RenderState { .Expression = node});
41194169 try stack.append(RenderState.PrintIndent);
41204170 try stack.append(RenderState {
41214171 .Text = blk: {
......@@ -4487,7 +4537,7 @@ pub const Parser = struct {
44874537 }
44884538
44894539 fn renderComments(self: &Parser, stream: var, node: &ast.Node, indent: usize) !void {
4490 const comment = node.before_comments ?? return;
4540 const comment = node.doc_comments ?? return;
44914541 for (comment.lines.toSliceConst()) |line_token| {
44924542 try stream.print("{}\n", self.tokenizer.getTokenSlice(line_token));
44934543 try stream.writeByteNTimes(' ', indent);
std/zig/parser_test.zig+33-3
......@@ -3,9 +3,12 @@ test "zig fmt: comments before error set decl" {
33 \\const UnexpectedError = error {
44 \\ /// The Operating System returned an undocumented error code.
55 \\ Unexpected,
6 \\
76 \\ // another
87 \\ Another,
8 \\
9 \\ // in between
10 \\
11 \\ // at end
912 \\};
1013 \\
1114 );
......@@ -18,8 +21,10 @@ test "zig fmt: comments before switch prong" {
1821 \\ error.PathAlreadyExists => continue,
1922 \\
2023 \\ // comment 1
24 \\
2125 \\ // comment 2
2226 \\ else => return err,
27 \\ // at end
2328 \\ }
2429 \\}
2530 \\
......@@ -47,6 +52,17 @@ test "zig fmt: comments before var decl in struct" {
4752 \\ permitted: u32,
4853 \\ inheritable: u32,
4954 \\ };
55 \\
56 \\ // in between
57 \\
58 \\ /// All of these are mandated as little endian
59 \\ /// when on disk.
60 \\ const Data = struct {
61 \\ permitted: u32,
62 \\ inheritable: u32,
63 \\ };
64 \\
65 \\ // at end
5066 \\};
5167 \\
5268 );
......@@ -106,6 +122,10 @@ test "zig fmt: comments before statements" {
106122 \\test "std" {
107123 \\ // statement comment
108124 \\ _ = @import("foo/bar.zig");
125 \\
126 \\ // middle
127 \\
128 \\ // end
109129 \\}
110130 \\
111131 );
......@@ -113,17 +133,27 @@ test "zig fmt: comments before statements" {
113133
114134test "zig fmt: comments before test decl" {
115135 try testCanonical(
116 \\// top level comment
136 \\/// top level doc comment
117137 \\test "hi" {}
118138 \\
139 \\// top level normal comment
140 \\test "hi" {}
141 \\
142 \\// middle
143 \\
144 \\// end
145 \\
119146 );
120147}
121148
122test "zig fmt: get stdout or fail" {
149test "zig fmt: comments before variable declarations" {
123150 try testCanonical(
124151 \\const std = @import("std");
125152 \\
126153 \\pub fn main() !void {
154 \\ /// If this program is run without stdout attached, exit with an error.
155 \\ /// another comment
156 \\ var stdout_file = try std.io.getStdOut;
127157 \\ // If this program is run without stdout attached, exit with an error.
128158 \\ // another comment
129159 \\ var stdout_file = try std.io.getStdOut;
std/zig/tokenizer.zig+15-2
......@@ -137,6 +137,7 @@ pub const Token = struct {
137137 IntegerLiteral,
138138 FloatLiteral,
139139 LineComment,
140 DocComment,
140141 Keyword_align,
141142 Keyword_and,
142143 Keyword_asm,
......@@ -257,6 +258,7 @@ pub const Tokenizer = struct {
257258 Asterisk,
258259 AsteriskPercent,
259260 Slash,
261 LineCommentStart,
260262 LineComment,
261263 Zero,
262264 IntegerLiteral,
......@@ -822,8 +824,7 @@ pub const Tokenizer = struct {
822824
823825 State.Slash => switch (c) {
824826 '/' => {
825 result.id = Token.Id.LineComment;
826 state = State.LineComment;
827 state = State.LineCommentStart;
827828 },
828829 '=' => {
829830 result.id = Token.Id.SlashEqual;
......@@ -835,6 +836,17 @@ pub const Tokenizer = struct {
835836 break;
836837 },
837838 },
839 State.LineCommentStart => switch (c) {
840 '/' => {
841 result.id = Token.Id.DocComment;
842 state = State.LineComment;
843 },
844 '\n' => {
845 result.id = Token.Id.LineComment;
846 break;
847 },
848 else => self.checkLiteralCharacter(),
849 },
838850 State.LineComment => switch (c) {
839851 '\n' => break,
840852 else => self.checkLiteralCharacter(),
......@@ -920,6 +932,7 @@ pub const Tokenizer = struct {
920932 result.id = id;
921933 }
922934 },
935 State.LineCommentStart,
923936 State.LineComment => {
924937 result.id = Token.Id.Eof;
925938 },