authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-04-30 00:19:55-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-04-30 00:19:55-04:00
logfd2cd38bdb831ef78a0d4ab0973020dfbd348c1f
tree58f228d2a8c7ab6d00e7f18e9e0116a699822657
parent39befc35a851f6cf206c7e3b19d66f7df00440b0

zig fmt: support line comments and doc comments

line comments can go anywhere a list of something is allowed

4 files changed, 168 insertions(+), 56 deletions(-)

std/zig/ast.zig+24-5
...@@ -6,7 +6,7 @@ const mem = std.mem;...@@ -6,7 +6,7 @@ const mem = std.mem;
66
7pub const Node = struct {7pub const Node = struct {
8 id: Id,8 id: Id,
9 before_comments: ?&LineComment,9 doc_comments: ?&DocComment,
10 same_line_comment: ?&Token,10 same_line_comment: ?&Token,
1111
12 pub const Id = enum {12 pub const Id = enum {
...@@ -59,6 +59,7 @@ pub const Node = struct {...@@ -59,6 +59,7 @@ pub const Node = struct {
5959
60 // Misc60 // Misc
61 LineComment,61 LineComment,
62 DocComment,
62 SwitchCase,63 SwitchCase,
63 SwitchElse,64 SwitchElse,
64 Else,65 Else,
...@@ -718,7 +719,8 @@ pub const Node = struct {...@@ -718,7 +719,8 @@ pub const Node = struct {
718 base: Node,719 base: Node,
719 switch_token: Token,720 switch_token: Token,
720 expr: &Node,721 expr: &Node,
721 cases: ArrayList(&SwitchCase),722 /// these can be SwitchCase nodes or LineComment nodes
723 cases: ArrayList(&Node),
722 rbrace: Token,724 rbrace: Token,
723725
724 pub fn iterate(self: &Switch, index: usize) ?&Node {726 pub fn iterate(self: &Switch, index: usize) ?&Node {
...@@ -727,7 +729,7 @@ pub const Node = struct {...@@ -727,7 +729,7 @@ pub const Node = struct {
727 if (i < 1) return self.expr;729 if (i < 1) return self.expr;
728 i -= 1;730 i -= 1;
729731
730 if (i < self.cases.len) return &self.cases.at(i).base;732 if (i < self.cases.len) return self.cases.at(i);
731 i -= self.cases.len;733 i -= self.cases.len;
732734
733 return null;735 return null;
...@@ -1715,17 +1717,34 @@ pub const Node = struct {...@@ -1715,17 +1717,34 @@ pub const Node = struct {
17151717
1716 pub const LineComment = struct {1718 pub const LineComment = struct {
1717 base: Node,1719 base: Node,
1718 lines: ArrayList(Token),1720 token: Token,
17191721
1720 pub fn iterate(self: &LineComment, index: usize) ?&Node {1722 pub fn iterate(self: &LineComment, index: usize) ?&Node {
1721 return null;1723 return null;
1722 }1724 }
17231725
1724 pub fn firstToken(self: &LineComment) Token {1726 pub fn firstToken(self: &LineComment) Token {
1725 return self.lines.at(0);1727 return self.token;
1726 }1728 }
17271729
1728 pub fn lastToken(self: &LineComment) Token {1730 pub fn lastToken(self: &LineComment) Token {
1731 return self.token;
1732 }
1733 };
1734
1735 pub const DocComment = struct {
1736 base: Node,
1737 lines: ArrayList(Token),
1738
1739 pub fn iterate(self: &DocComment, index: usize) ?&Node {
1740 return null;
1741 }
1742
1743 pub fn firstToken(self: &DocComment) Token {
1744 return self.lines.at(0);
1745 }
1746
1747 pub fn lastToken(self: &DocComment) Token {
1729 return self.lines.at(self.lines.len - 1);1748 return self.lines.at(self.lines.len - 1);
1730 }1749 }
1731 };1750 };
std/zig/parser.zig+96-46
...@@ -55,7 +55,7 @@ pub const Parser = struct {...@@ -55,7 +55,7 @@ pub const Parser = struct {
55 visib_token: ?Token,55 visib_token: ?Token,
56 extern_export_inline_token: ?Token,56 extern_export_inline_token: ?Token,
57 lib_name: ?&ast.Node,57 lib_name: ?&ast.Node,
58 comments: ?&ast.Node.LineComment,58 comments: ?&ast.Node.DocComment,
59 };59 };
6060
61 const VarDeclCtx = struct {61 const VarDeclCtx = struct {
...@@ -65,19 +65,19 @@ pub const Parser = struct {...@@ -65,19 +65,19 @@ pub const Parser = struct {
65 extern_export_token: ?Token,65 extern_export_token: ?Token,
66 lib_name: ?&ast.Node,66 lib_name: ?&ast.Node,
67 list: &ArrayList(&ast.Node),67 list: &ArrayList(&ast.Node),
68 comments: ?&ast.Node.LineComment,68 comments: ?&ast.Node.DocComment,
69 };69 };
7070
71 const TopLevelExternOrFieldCtx = struct {71 const TopLevelExternOrFieldCtx = struct {
72 visib_token: Token,72 visib_token: Token,
73 container_decl: &ast.Node.ContainerDecl,73 container_decl: &ast.Node.ContainerDecl,
74 comments: ?&ast.Node.LineComment,74 comments: ?&ast.Node.DocComment,
75 };75 };
7676
77 const ExternTypeCtx = struct {77 const ExternTypeCtx = struct {
78 opt_ctx: OptionalCtx,78 opt_ctx: OptionalCtx,
79 extern_token: Token,79 extern_token: Token,
80 comments: ?&ast.Node.LineComment,80 comments: ?&ast.Node.DocComment,
81 };81 };
8282
83 const ContainerKindCtx = struct {83 const ContainerKindCtx = struct {
...@@ -186,7 +186,7 @@ pub const Parser = struct {...@@ -186,7 +186,7 @@ pub const Parser = struct {
186186
187 const AddCommentsCtx = struct {187 const AddCommentsCtx = struct {
188 node_ptr: &&ast.Node,188 node_ptr: &&ast.Node,
189 comments: ?&ast.Node.LineComment,189 comments: ?&ast.Node.DocComment,
190 };190 };
191191
192 const State = union(enum) {192 const State = union(enum) {
...@@ -244,8 +244,8 @@ pub const Parser = struct {...@@ -244,8 +244,8 @@ pub const Parser = struct {
244 FieldListCommaOrEnd: &ast.Node.ContainerDecl,244 FieldListCommaOrEnd: &ast.Node.ContainerDecl,
245 IdentifierListItemOrEnd: ListSave(&ast.Node),245 IdentifierListItemOrEnd: ListSave(&ast.Node),
246 IdentifierListCommaOrEnd: ListSave(&ast.Node),246 IdentifierListCommaOrEnd: ListSave(&ast.Node),
247 SwitchCaseOrEnd: ListSave(&ast.Node.SwitchCase),247 SwitchCaseOrEnd: ListSave(&ast.Node),
248 SwitchCaseCommaOrEnd: ListSave(&ast.Node.SwitchCase),248 SwitchCaseCommaOrEnd: ListSave(&ast.Node),
249 SwitchCaseFirstItem: &ArrayList(&ast.Node),249 SwitchCaseFirstItem: &ArrayList(&ast.Node),
250 SwitchCaseItem: &ArrayList(&ast.Node),250 SwitchCaseItem: &ArrayList(&ast.Node),
251 SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node),251 SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node),
...@@ -349,6 +349,10 @@ pub const Parser = struct {...@@ -349,6 +349,10 @@ pub const Parser = struct {
349349
350 switch (state) {350 switch (state) {
351 State.TopLevel => {351 State.TopLevel => {
352 while (try self.eatLineComment(arena)) |line_comment| {
353 try root_node.decls.append(&line_comment.base);
354 }
355
352 const comments = try self.eatComments(arena);356 const comments = try self.eatComments(arena);
353 const token = self.getNextToken();357 const token = self.getNextToken();
354 switch (token.id) {358 switch (token.id) {
...@@ -358,7 +362,7 @@ pub const Parser = struct {...@@ -358,7 +362,7 @@ pub const Parser = struct {
358 const block = try arena.construct(ast.Node.Block {362 const block = try arena.construct(ast.Node.Block {
359 .base = ast.Node {363 .base = ast.Node {
360 .id = ast.Node.Id.Block,364 .id = ast.Node.Id.Block,
361 .before_comments = null,365 .doc_comments = null,
362 .same_line_comment = null,366 .same_line_comment = null,
363 },367 },
364 .label = null,368 .label = null,
...@@ -369,7 +373,7 @@ pub const Parser = struct {...@@ -369,7 +373,7 @@ pub const Parser = struct {
369 const test_node = try arena.construct(ast.Node.TestDecl {373 const test_node = try arena.construct(ast.Node.TestDecl {
370 .base = ast.Node {374 .base = ast.Node {
371 .id = ast.Node.Id.TestDecl,375 .id = ast.Node.Id.TestDecl,
372 .before_comments = comments,376 .doc_comments = comments,
373 .same_line_comment = null,377 .same_line_comment = null,
374 },378 },
375 .test_token = token,379 .test_token = token,
...@@ -551,7 +555,7 @@ pub const Parser = struct {...@@ -551,7 +555,7 @@ pub const Parser = struct {
551 const fn_proto = try arena.construct(ast.Node.FnProto {555 const fn_proto = try arena.construct(ast.Node.FnProto {
552 .base = ast.Node {556 .base = ast.Node {
553 .id = ast.Node.Id.FnProto,557 .id = ast.Node.Id.FnProto,
554 .before_comments = ctx.comments,558 .doc_comments = ctx.comments,
555 .same_line_comment = null,559 .same_line_comment = null,
556 },560 },
557 .visib_token = ctx.visib_token,561 .visib_token = ctx.visib_token,
...@@ -620,7 +624,7 @@ pub const Parser = struct {...@@ -620,7 +624,7 @@ pub const Parser = struct {
620 const node = try arena.construct(ast.Node.StructField {624 const node = try arena.construct(ast.Node.StructField {
621 .base = ast.Node {625 .base = ast.Node {
622 .id = ast.Node.Id.StructField,626 .id = ast.Node.Id.StructField,
623 .before_comments = null,627 .doc_comments = null,
624 .same_line_comment = null,628 .same_line_comment = null,
625 },629 },
626 .visib_token = ctx.visib_token,630 .visib_token = ctx.visib_token,
...@@ -706,6 +710,10 @@ pub const Parser = struct {...@@ -706,6 +710,10 @@ pub const Parser = struct {
706 continue;710 continue;
707 },711 },
708 State.ContainerDecl => |container_decl| {712 State.ContainerDecl => |container_decl| {
713 while (try self.eatLineComment(arena)) |line_comment| {
714 try container_decl.fields_and_decls.append(&line_comment.base);
715 }
716
709 const comments = try self.eatComments(arena);717 const comments = try self.eatComments(arena);
710 const token = self.getNextToken();718 const token = self.getNextToken();
711 switch (token.id) {719 switch (token.id) {
...@@ -715,7 +723,7 @@ pub const Parser = struct {...@@ -715,7 +723,7 @@ pub const Parser = struct {
715 const node = try arena.construct(ast.Node.StructField {723 const node = try arena.construct(ast.Node.StructField {
716 .base = ast.Node {724 .base = ast.Node {
717 .id = ast.Node.Id.StructField,725 .id = ast.Node.Id.StructField,
718 .before_comments = comments,726 .doc_comments = comments,
719 .same_line_comment = null,727 .same_line_comment = null,
720 },728 },
721 .visib_token = null,729 .visib_token = null,
...@@ -826,7 +834,7 @@ pub const Parser = struct {...@@ -826,7 +834,7 @@ pub const Parser = struct {
826 const var_decl = try arena.construct(ast.Node.VarDecl {834 const var_decl = try arena.construct(ast.Node.VarDecl {
827 .base = ast.Node {835 .base = ast.Node {
828 .id = ast.Node.Id.VarDecl,836 .id = ast.Node.Id.VarDecl,
829 .before_comments = ctx.comments,837 .doc_comments = ctx.comments,
830 .same_line_comment = null,838 .same_line_comment = null,
831 },839 },
832 .visib_token = ctx.visib_token,840 .visib_token = ctx.visib_token,
...@@ -1222,6 +1230,14 @@ pub const Parser = struct {...@@ -1222,6 +1230,14 @@ pub const Parser = struct {
1222 else => {1230 else => {
1223 self.putBackToken(token);1231 self.putBackToken(token);
1224 stack.append(State { .Block = block }) catch unreachable;1232 stack.append(State { .Block = block }) catch unreachable;
1233
1234 var any_comments = false;
1235 while (try self.eatLineComment(arena)) |line_comment| {
1236 try block.statements.append(&line_comment.base);
1237 any_comments = true;
1238 }
1239 if (any_comments) continue;
1240
1225 try stack.append(State { .Statement = block });1241 try stack.append(State { .Statement = block });
1226 continue;1242 continue;
1227 },1243 },
...@@ -1258,7 +1274,7 @@ pub const Parser = struct {...@@ -1258,7 +1274,7 @@ pub const Parser = struct {
1258 const node = try arena.construct(ast.Node.Defer {1274 const node = try arena.construct(ast.Node.Defer {
1259 .base = ast.Node {1275 .base = ast.Node {
1260 .id = ast.Node.Id.Defer,1276 .id = ast.Node.Id.Defer,
1261 .before_comments = comments,1277 .doc_comments = comments,
1262 .same_line_comment = null,1278 .same_line_comment = null,
1263 },1279 },
1264 .defer_token = token,1280 .defer_token = token,
...@@ -1342,7 +1358,7 @@ pub const Parser = struct {...@@ -1342,7 +1358,7 @@ pub const Parser = struct {
13421358
1343 State.AddComments => |add_comments_ctx| {1359 State.AddComments => |add_comments_ctx| {
1344 const node = *add_comments_ctx.node_ptr;1360 const node = *add_comments_ctx.node_ptr;
1345 node.before_comments = add_comments_ctx.comments;1361 node.doc_comments = add_comments_ctx.comments;
1346 continue;1362 continue;
1347 },1363 },
13481364
...@@ -1466,7 +1482,7 @@ pub const Parser = struct {...@@ -1466,7 +1482,7 @@ pub const Parser = struct {
1466 const node = try arena.construct(ast.Node.FieldInitializer {1482 const node = try arena.construct(ast.Node.FieldInitializer {
1467 .base = ast.Node {1483 .base = ast.Node {
1468 .id = ast.Node.Id.FieldInitializer,1484 .id = ast.Node.Id.FieldInitializer,
1469 .before_comments = null,1485 .doc_comments = null,
1470 .same_line_comment = null,1486 .same_line_comment = null,
1471 },1487 },
1472 .period_token = undefined,1488 .period_token = undefined,
...@@ -1512,6 +1528,10 @@ pub const Parser = struct {...@@ -1512,6 +1528,10 @@ pub const Parser = struct {
1512 continue;1528 continue;
1513 },1529 },
1514 State.IdentifierListItemOrEnd => |list_state| {1530 State.IdentifierListItemOrEnd => |list_state| {
1531 while (try self.eatLineComment(arena)) |line_comment| {
1532 try list_state.list.append(&line_comment.base);
1533 }
1534
1515 if (self.eatToken(Token.Id.RBrace)) |rbrace| {1535 if (self.eatToken(Token.Id.RBrace)) |rbrace| {
1516 *list_state.ptr = rbrace;1536 *list_state.ptr = rbrace;
1517 continue;1537 continue;
...@@ -1538,6 +1558,10 @@ pub const Parser = struct {...@@ -1538,6 +1558,10 @@ pub const Parser = struct {
1538 }1558 }
1539 },1559 },
1540 State.SwitchCaseOrEnd => |list_state| {1560 State.SwitchCaseOrEnd => |list_state| {
1561 while (try self.eatLineComment(arena)) |line_comment| {
1562 try list_state.list.append(&line_comment.base);
1563 }
1564
1541 if (self.eatToken(Token.Id.RBrace)) |rbrace| {1565 if (self.eatToken(Token.Id.RBrace)) |rbrace| {
1542 *list_state.ptr = rbrace;1566 *list_state.ptr = rbrace;
1543 continue;1567 continue;
...@@ -1547,14 +1571,14 @@ pub const Parser = struct {...@@ -1547,14 +1571,14 @@ pub const Parser = struct {
1547 const node = try arena.construct(ast.Node.SwitchCase {1571 const node = try arena.construct(ast.Node.SwitchCase {
1548 .base = ast.Node {1572 .base = ast.Node {
1549 .id = ast.Node.Id.SwitchCase,1573 .id = ast.Node.Id.SwitchCase,
1550 .before_comments = comments,1574 .doc_comments = comments,
1551 .same_line_comment = null,1575 .same_line_comment = null,
1552 },1576 },
1553 .items = ArrayList(&ast.Node).init(arena),1577 .items = ArrayList(&ast.Node).init(arena),
1554 .payload = null,1578 .payload = null,
1555 .expr = undefined,1579 .expr = undefined,
1556 });1580 });
1557 try list_state.list.append(node);1581 try list_state.list.append(&node.base);
1558 try stack.append(State { .SwitchCaseCommaOrEnd = list_state });1582 try stack.append(State { .SwitchCaseCommaOrEnd = list_state });
1559 try stack.append(State { .AssignmentExpressionBegin = OptionalCtx { .Required = &node.expr } });1583 try stack.append(State { .AssignmentExpressionBegin = OptionalCtx { .Required = &node.expr } });
1560 try stack.append(State { .PointerPayload = OptionalCtx { .Optional = &node.payload } });1584 try stack.append(State { .PointerPayload = OptionalCtx { .Optional = &node.payload } });
...@@ -1569,8 +1593,8 @@ pub const Parser = struct {...@@ -1569,8 +1593,8 @@ pub const Parser = struct {
1569 continue;1593 continue;
1570 }1594 }
15711595
1572 const switch_case = list_state.list.toSlice()[list_state.list.len - 1];1596 const node = list_state.list.toSlice()[list_state.list.len - 1];
1573 try self.lookForSameLineComment(arena, &switch_case.base);1597 try self.lookForSameLineComment(arena, node);
1574 try stack.append(State { .SwitchCaseOrEnd = list_state });1598 try stack.append(State { .SwitchCaseOrEnd = list_state });
1575 continue;1599 continue;
1576 },1600 },
...@@ -1660,7 +1684,7 @@ pub const Parser = struct {...@@ -1660,7 +1684,7 @@ pub const Parser = struct {
1660 const fn_proto = try arena.construct(ast.Node.FnProto {1684 const fn_proto = try arena.construct(ast.Node.FnProto {
1661 .base = ast.Node {1685 .base = ast.Node {
1662 .id = ast.Node.Id.FnProto,1686 .id = ast.Node.Id.FnProto,
1663 .before_comments = ctx.comments,1687 .doc_comments = ctx.comments,
1664 .same_line_comment = null,1688 .same_line_comment = null,
1665 },1689 },
1666 .visib_token = null,1690 .visib_token = null,
...@@ -2632,7 +2656,7 @@ pub const Parser = struct {...@@ -2632,7 +2656,7 @@ pub const Parser = struct {
2632 const fn_proto = try arena.construct(ast.Node.FnProto {2656 const fn_proto = try arena.construct(ast.Node.FnProto {
2633 .base = ast.Node {2657 .base = ast.Node {
2634 .id = ast.Node.Id.FnProto,2658 .id = ast.Node.Id.FnProto,
2635 .before_comments = null,2659 .doc_comments = null,
2636 .same_line_comment = null,2660 .same_line_comment = null,
2637 },2661 },
2638 .visib_token = null,2662 .visib_token = null,
...@@ -2656,7 +2680,7 @@ pub const Parser = struct {...@@ -2656,7 +2680,7 @@ pub const Parser = struct {
2656 const fn_proto = try arena.construct(ast.Node.FnProto {2680 const fn_proto = try arena.construct(ast.Node.FnProto {
2657 .base = ast.Node {2681 .base = ast.Node {
2658 .id = ast.Node.Id.FnProto,2682 .id = ast.Node.Id.FnProto,
2659 .before_comments = null,2683 .doc_comments = null,
2660 .same_line_comment = null,2684 .same_line_comment = null,
2661 },2685 },
2662 .visib_token = null,2686 .visib_token = null,
...@@ -2749,7 +2773,7 @@ pub const Parser = struct {...@@ -2749,7 +2773,7 @@ pub const Parser = struct {
2749 const node = try arena.construct(ast.Node.ErrorSetDecl {2773 const node = try arena.construct(ast.Node.ErrorSetDecl {
2750 .base = ast.Node {2774 .base = ast.Node {
2751 .id = ast.Node.Id.ErrorSetDecl,2775 .id = ast.Node.Id.ErrorSetDecl,
2752 .before_comments = null,2776 .doc_comments = null,
2753 .same_line_comment = null,2777 .same_line_comment = null,
2754 },2778 },
2755 .error_token = ctx.error_token,2779 .error_token = ctx.error_token,
...@@ -2829,18 +2853,18 @@ pub const Parser = struct {...@@ -2829,18 +2853,18 @@ pub const Parser = struct {
2829 }2853 }
2830 }2854 }
28312855
2832 fn eatComments(self: &Parser, arena: &mem.Allocator) !?&ast.Node.LineComment {2856 fn eatComments(self: &Parser, arena: &mem.Allocator) !?&ast.Node.DocComment {
2833 var result: ?&ast.Node.LineComment = null;2857 var result: ?&ast.Node.DocComment = null;
2834 while (true) {2858 while (true) {
2835 if (self.eatToken(Token.Id.LineComment)) |line_comment| {2859 if (self.eatToken(Token.Id.DocComment)) |line_comment| {
2836 const node = blk: {2860 const node = blk: {
2837 if (result) |comment_node| {2861 if (result) |comment_node| {
2838 break :blk comment_node;2862 break :blk comment_node;
2839 } else {2863 } else {
2840 const comment_node = try arena.construct(ast.Node.LineComment {2864 const comment_node = try arena.construct(ast.Node.DocComment {
2841 .base = ast.Node {2865 .base = ast.Node {
2842 .id = ast.Node.Id.LineComment,2866 .id = ast.Node.Id.DocComment,
2843 .before_comments = null,2867 .doc_comments = null,
2844 .same_line_comment = null,2868 .same_line_comment = null,
2845 },2869 },
2846 .lines = ArrayList(Token).init(arena),2870 .lines = ArrayList(Token).init(arena),
...@@ -2857,6 +2881,18 @@ pub const Parser = struct {...@@ -2857,6 +2881,18 @@ pub const Parser = struct {
2857 return result;2881 return result;
2858 }2882 }
28592883
2884 fn eatLineComment(self: &Parser, arena: &mem.Allocator) !?&ast.Node.LineComment {
2885 const token = self.eatToken(Token.Id.LineComment) ?? return null;
2886 return try arena.construct(ast.Node.LineComment {
2887 .base = ast.Node {
2888 .id = ast.Node.Id.LineComment,
2889 .doc_comments = null,
2890 .same_line_comment = null,
2891 },
2892 .token = token,
2893 });
2894 }
2895
2860 fn requireSemiColon(node: &const ast.Node) bool {2896 fn requireSemiColon(node: &const ast.Node) bool {
2861 var n = node;2897 var n = node;
2862 while (true) {2898 while (true) {
...@@ -2874,6 +2910,7 @@ pub const Parser = struct {...@@ -2874,6 +2910,7 @@ pub const Parser = struct {
2874 ast.Node.Id.SwitchCase,2910 ast.Node.Id.SwitchCase,
2875 ast.Node.Id.SwitchElse,2911 ast.Node.Id.SwitchElse,
2876 ast.Node.Id.FieldInitializer,2912 ast.Node.Id.FieldInitializer,
2913 ast.Node.Id.DocComment,
2877 ast.Node.Id.LineComment,2914 ast.Node.Id.LineComment,
2878 ast.Node.Id.TestDecl => return false,2915 ast.Node.Id.TestDecl => return false,
2879 ast.Node.Id.While => {2916 ast.Node.Id.While => {
...@@ -2933,7 +2970,7 @@ pub const Parser = struct {...@@ -2933,7 +2970,7 @@ pub const Parser = struct {
2933 const node_last_token = node.lastToken();2970 const node_last_token = node.lastToken();
29342971
2935 const line_comment_token = self.getNextToken();2972 const line_comment_token = self.getNextToken();
2936 if (line_comment_token.id != Token.Id.LineComment) {2973 if (line_comment_token.id != Token.Id.DocComment and line_comment_token.id != Token.Id.LineComment) {
2937 self.putBackToken(line_comment_token);2974 self.putBackToken(line_comment_token);
2938 return;2975 return;
2939 }2976 }
...@@ -3038,18 +3075,21 @@ pub const Parser = struct {...@@ -3038,18 +3075,21 @@ pub const Parser = struct {
3038 return true;3075 return true;
3039 },3076 },
3040 Token.Id.Keyword_switch => {3077 Token.Id.Keyword_switch => {
3041 const node = try self.createToCtxNode(arena, ctx, ast.Node.Switch,3078 const node = try arena.construct(ast.Node.Switch {
3042 ast.Node.Switch {3079 .base = ast.Node {
3043 .base = undefined,3080 .id = ast.Node.Id.Switch,
3044 .switch_token = *token,3081 .doc_comments = null,
3045 .expr = undefined,3082 .same_line_comment = null,
3046 .cases = ArrayList(&ast.Node.SwitchCase).init(arena),3083 },
3047 .rbrace = undefined,3084 .switch_token = *token,
3048 }3085 .expr = undefined,
3049 );3086 .cases = ArrayList(&ast.Node).init(arena),
3087 .rbrace = undefined,
3088 });
3089 ctx.store(&node.base);
30503090
3051 stack.append(State {3091 stack.append(State {
3052 .SwitchCaseOrEnd = ListSave(&ast.Node.SwitchCase) {3092 .SwitchCaseOrEnd = ListSave(&ast.Node) {
3053 .list = &node.cases,3093 .list = &node.cases,
3054 .ptr = &node.rbrace,3094 .ptr = &node.rbrace,
3055 },3095 },
...@@ -3208,7 +3248,7 @@ pub const Parser = struct {...@@ -3208,7 +3248,7 @@ pub const Parser = struct {
3208 const id = ast.Node.typeToId(T);3248 const id = ast.Node.typeToId(T);
3209 break :blk ast.Node {3249 break :blk ast.Node {
3210 .id = id,3250 .id = id,
3211 .before_comments = null,3251 .doc_comments = null,
3212 .same_line_comment = null,3252 .same_line_comment = null,
3213 };3253 };
3214 };3254 };
...@@ -3454,6 +3494,10 @@ pub const Parser = struct {...@@ -3454,6 +3494,10 @@ pub const Parser = struct {
3454 }3494 }
3455 try stack.append(RenderState { .Expression = decl });3495 try stack.append(RenderState { .Expression = decl });
3456 },3496 },
3497 ast.Node.Id.LineComment => {
3498 const line_comment_node = @fieldParentPtr(ast.Node.LineComment, "base", decl);
3499 try stream.write(self.tokenizer.getTokenSlice(line_comment_node.token));
3500 },
3457 else => unreachable,3501 else => unreachable,
3458 }3502 }
3459 },3503 },
...@@ -3987,7 +4031,9 @@ pub const Parser = struct {...@@ -3987,7 +4031,9 @@ pub const Parser = struct {
3987 while (i != 0) {4031 while (i != 0) {
3988 i -= 1;4032 i -= 1;
3989 const node = decls[i];4033 const node = decls[i];
3990 try stack.append(RenderState { .Text = "," });4034 if (node.id != ast.Node.Id.LineComment) {
4035 try stack.append(RenderState { .Text = "," });
4036 }
3991 try stack.append(RenderState { .Expression = node });4037 try stack.append(RenderState { .Expression = node });
3992 try stack.append(RenderState { .PrintComments = node });4038 try stack.append(RenderState { .PrintComments = node });
3993 try stack.append(RenderState.PrintIndent);4039 try stack.append(RenderState.PrintIndent);
...@@ -4100,7 +4146,11 @@ pub const Parser = struct {...@@ -4100,7 +4146,11 @@ pub const Parser = struct {
4100 try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(visib_token) });4146 try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(visib_token) });
4101 }4147 }
4102 },4148 },
4103 ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"),4149 ast.Node.Id.LineComment => {
4150 const line_comment_node = @fieldParentPtr(ast.Node.LineComment, "base", base);
4151 try stream.write(self.tokenizer.getTokenSlice(line_comment_node.token));
4152 },
4153 ast.Node.Id.DocComment => unreachable, // doc comments are attached to nodes
4104 ast.Node.Id.Switch => {4154 ast.Node.Id.Switch => {
4105 const switch_node = @fieldParentPtr(ast.Node.Switch, "base", base);4155 const switch_node = @fieldParentPtr(ast.Node.Switch, "base", base);
4106 try stream.print("{} (", self.tokenizer.getTokenSlice(switch_node.switch_token));4156 try stream.print("{} (", self.tokenizer.getTokenSlice(switch_node.switch_token));
...@@ -4115,7 +4165,7 @@ pub const Parser = struct {...@@ -4115,7 +4165,7 @@ pub const Parser = struct {
4115 while (i != 0) {4165 while (i != 0) {
4116 i -= 1;4166 i -= 1;
4117 const node = cases[i];4167 const node = cases[i];
4118 try stack.append(RenderState { .Expression = &node.base});4168 try stack.append(RenderState { .Expression = node});
4119 try stack.append(RenderState.PrintIndent);4169 try stack.append(RenderState.PrintIndent);
4120 try stack.append(RenderState {4170 try stack.append(RenderState {
4121 .Text = blk: {4171 .Text = blk: {
...@@ -4487,7 +4537,7 @@ pub const Parser = struct {...@@ -4487,7 +4537,7 @@ pub const Parser = struct {
4487 }4537 }
44884538
4489 fn renderComments(self: &Parser, stream: var, node: &ast.Node, indent: usize) !void {4539 fn renderComments(self: &Parser, stream: var, node: &ast.Node, indent: usize) !void {
4490 const comment = node.before_comments ?? return;4540 const comment = node.doc_comments ?? return;
4491 for (comment.lines.toSliceConst()) |line_token| {4541 for (comment.lines.toSliceConst()) |line_token| {
4492 try stream.print("{}\n", self.tokenizer.getTokenSlice(line_token));4542 try stream.print("{}\n", self.tokenizer.getTokenSlice(line_token));
4493 try stream.writeByteNTimes(' ', indent);4543 try stream.writeByteNTimes(' ', indent);
std/zig/parser_test.zig+33-3
...@@ -3,9 +3,12 @@ test "zig fmt: comments before error set decl" {...@@ -3,9 +3,12 @@ test "zig fmt: comments before error set decl" {
3 \\const UnexpectedError = error {3 \\const UnexpectedError = error {
4 \\ /// The Operating System returned an undocumented error code.4 \\ /// The Operating System returned an undocumented error code.
5 \\ Unexpected,5 \\ Unexpected,
6 \\
7 \\ // another6 \\ // another
8 \\ Another,7 \\ Another,
8 \\
9 \\ // in between
10 \\
11 \\ // at end
9 \\};12 \\};
10 \\13 \\
11 );14 );
...@@ -18,8 +21,10 @@ test "zig fmt: comments before switch prong" {...@@ -18,8 +21,10 @@ test "zig fmt: comments before switch prong" {
18 \\ error.PathAlreadyExists => continue,21 \\ error.PathAlreadyExists => continue,
19 \\22 \\
20 \\ // comment 123 \\ // comment 1
24 \\
21 \\ // comment 225 \\ // comment 2
22 \\ else => return err,26 \\ else => return err,
27 \\ // at end
23 \\ }28 \\ }
24 \\}29 \\}
25 \\30 \\
...@@ -47,6 +52,17 @@ test "zig fmt: comments before var decl in struct" {...@@ -47,6 +52,17 @@ test "zig fmt: comments before var decl in struct" {
47 \\ permitted: u32,52 \\ permitted: u32,
48 \\ inheritable: u32,53 \\ inheritable: u32,
49 \\ };54 \\ };
55 \\
56 \\ // in between
57 \\
58 \\ /// All of these are mandated as little endian
59 \\ /// when on disk.
60 \\ const Data = struct {
61 \\ permitted: u32,
62 \\ inheritable: u32,
63 \\ };
64 \\
65 \\ // at end
50 \\};66 \\};
51 \\67 \\
52 );68 );
...@@ -106,6 +122,10 @@ test "zig fmt: comments before statements" {...@@ -106,6 +122,10 @@ test "zig fmt: comments before statements" {
106 \\test "std" {122 \\test "std" {
107 \\ // statement comment123 \\ // statement comment
108 \\ _ = @import("foo/bar.zig");124 \\ _ = @import("foo/bar.zig");
125 \\
126 \\ // middle
127 \\
128 \\ // end
109 \\}129 \\}
110 \\130 \\
111 );131 );
...@@ -113,17 +133,27 @@ test "zig fmt: comments before statements" {...@@ -113,17 +133,27 @@ test "zig fmt: comments before statements" {
113133
114test "zig fmt: comments before test decl" {134test "zig fmt: comments before test decl" {
115 try testCanonical(135 try testCanonical(
116 \\// top level comment136 \\/// top level doc comment
117 \\test "hi" {}137 \\test "hi" {}
118 \\138 \\
139 \\// top level normal comment
140 \\test "hi" {}
141 \\
142 \\// middle
143 \\
144 \\// end
145 \\
119 );146 );
120}147}
121148
122test "zig fmt: get stdout or fail" {149test "zig fmt: comments before variable declarations" {
123 try testCanonical(150 try testCanonical(
124 \\const std = @import("std");151 \\const std = @import("std");
125 \\152 \\
126 \\pub fn main() !void {153 \\pub fn main() !void {
154 \\ /// If this program is run without stdout attached, exit with an error.
155 \\ /// another comment
156 \\ var stdout_file = try std.io.getStdOut;
127 \\ // If this program is run without stdout attached, exit with an error.157 \\ // If this program is run without stdout attached, exit with an error.
128 \\ // another comment158 \\ // another comment
129 \\ var stdout_file = try std.io.getStdOut;159 \\ var stdout_file = try std.io.getStdOut;
std/zig/tokenizer.zig+15-2
...@@ -137,6 +137,7 @@ pub const Token = struct {...@@ -137,6 +137,7 @@ pub const Token = struct {
137 IntegerLiteral,137 IntegerLiteral,
138 FloatLiteral,138 FloatLiteral,
139 LineComment,139 LineComment,
140 DocComment,
140 Keyword_align,141 Keyword_align,
141 Keyword_and,142 Keyword_and,
142 Keyword_asm,143 Keyword_asm,
...@@ -257,6 +258,7 @@ pub const Tokenizer = struct {...@@ -257,6 +258,7 @@ pub const Tokenizer = struct {
257 Asterisk,258 Asterisk,
258 AsteriskPercent,259 AsteriskPercent,
259 Slash,260 Slash,
261 LineCommentStart,
260 LineComment,262 LineComment,
261 Zero,263 Zero,
262 IntegerLiteral,264 IntegerLiteral,
...@@ -822,8 +824,7 @@ pub const Tokenizer = struct {...@@ -822,8 +824,7 @@ pub const Tokenizer = struct {
822824
823 State.Slash => switch (c) {825 State.Slash => switch (c) {
824 '/' => {826 '/' => {
825 result.id = Token.Id.LineComment;827 state = State.LineCommentStart;
826 state = State.LineComment;
827 },828 },
828 '=' => {829 '=' => {
829 result.id = Token.Id.SlashEqual;830 result.id = Token.Id.SlashEqual;
...@@ -835,6 +836,17 @@ pub const Tokenizer = struct {...@@ -835,6 +836,17 @@ pub const Tokenizer = struct {
835 break;836 break;
836 },837 },
837 },838 },
839 State.LineCommentStart => switch (c) {
840 '/' => {
841 result.id = Token.Id.DocComment;
842 state = State.LineComment;
843 },
844 '\n' => {
845 result.id = Token.Id.LineComment;
846 break;
847 },
848 else => self.checkLiteralCharacter(),
849 },
838 State.LineComment => switch (c) {850 State.LineComment => switch (c) {
839 '\n' => break,851 '\n' => break,
840 else => self.checkLiteralCharacter(),852 else => self.checkLiteralCharacter(),
...@@ -920,6 +932,7 @@ pub const Tokenizer = struct {...@@ -920,6 +932,7 @@ pub const Tokenizer = struct {
920 result.id = id;932 result.id = id;
921 }933 }
922 },934 },
935 State.LineCommentStart,
923 State.LineComment => {936 State.LineComment => {
924 result.id = Token.Id.Eof;937 result.id = Token.Id.Eof;
925 },938 },