authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-10-12 12:38:09+03:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-11-14 16:25:06+02:00
logb92f42d1f4b91bb343558f72af84ea052da4ac98
treeec297a65dcd93c06557f0e0a76353f4682004b46
parentd89f39d71949c85b26f2ccd4071c9445aa8b6d7c
signaturelock-open Commit is signed but in an unrecognized format.

implemented container doc comments in stage 2

6 files changed, 108 insertions(+), 10 deletions(-)

lib/std/zig/ast.zig-1
...@@ -576,7 +576,6 @@ pub const Node = struct {...@@ -576,7 +576,6 @@ pub const Node = struct {
576576
577 pub const Root = struct {577 pub const Root = struct {
578 base: Node,578 base: Node,
579 doc_comments: ?*DocComment,
580 decls: DeclList,579 decls: DeclList,
581 eof_token: TokenIndex,580 eof_token: TokenIndex,
582581
lib/std/zig/parse.zig+28-7
...@@ -58,13 +58,6 @@ fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error...@@ -58,13 +58,6 @@ fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error
58 node.* = Node.Root{58 node.* = Node.Root{
59 .base = Node{ .id = .Root },59 .base = Node{ .id = .Root },
60 .decls = undefined,60 .decls = undefined,
61 // TODO: Because zig fmt collapses consecutive comments separated by blank lines into
62 // a single multi-line comment, it is currently impossible to have a container-level
63 // doc comment and NO doc comment on the first decl. For now, simply
64 // ignore the problem and assume that there will be no container-level
65 // doc comments.
66 // See: https://github.com/ziglang/zig/issues/2288
67 .doc_comments = null,
68 .eof_token = undefined,61 .eof_token = undefined,
69 };62 };
70 node.decls = parseContainerMembers(arena, it, tree) catch |err| {63 node.decls = parseContainerMembers(arena, it, tree) catch |err| {
...@@ -94,6 +87,11 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No...@@ -94,6 +87,11 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
94 var list = Node.Root.DeclList.init(arena);87 var list = Node.Root.DeclList.init(arena);
9588
96 while (true) {89 while (true) {
90 if (try parseContainerDocComments(arena, it, tree)) |node| {
91 try list.push(node);
92 continue;
93 }
94
97 const doc_comments = try parseDocComment(arena, it, tree);95 const doc_comments = try parseDocComment(arena, it, tree);
9896
99 if (try parseTestDecl(arena, it, tree)) |node| {97 if (try parseTestDecl(arena, it, tree)) |node| {
...@@ -155,12 +153,35 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No...@@ -155,12 +153,35 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
155 continue;153 continue;
156 }154 }
157155
156 // Dangling doc comment
157 if (doc_comments != null) {
158 try tree.errors.push(AstError{
159 .UnattachedDocComment = AstError.UnattachedDocComment{ .token = doc_comments.?.firstToken() },
160 });
161 }
158 break;162 break;
159 }163 }
160164
161 return list;165 return list;
162}166}
163167
168/// Eat a multiline container doc comment
169fn parseContainerDocComments(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
170 var lines = Node.DocComment.LineList.init(arena);
171 while (eatToken(it, .ContainerDocComment)) |line| {
172 try lines.push(line);
173 }
174
175 if (lines.len == 0) return null;
176
177 const node = try arena.create(Node.DocComment);
178 node.* = Node.DocComment{
179 .base = Node{ .id = .DocComment },
180 .lines = lines,
181 };
182 return &node.base;
183}
184
164/// TestDecl <- KEYWORD_test STRINGLITERAL Block185/// TestDecl <- KEYWORD_test STRINGLITERAL Block
165fn parseTestDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {186fn parseTestDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
166 const test_token = eatToken(it, .Keyword_test) orelse return null;187 const test_token = eatToken(it, .Keyword_test) orelse return null;
lib/std/zig/parser_test.zig+56
...@@ -2556,6 +2556,62 @@ test "zig fmt: comments at several places in struct init" {...@@ -2556,6 +2556,62 @@ test "zig fmt: comments at several places in struct init" {
2556 );2556 );
2557}2557}
25582558
2559test "zig fmt: top level doc comments" {
2560 try testCanonical(
2561 \\//! tld 1
2562 \\//! tld 2
2563 \\//! tld 3
2564 \\
2565 \\// comment
2566 \\
2567 \\/// A doc
2568 \\const A = struct {
2569 \\ //! A tld 1
2570 \\ //! A tld 2
2571 \\ //! A tld 3
2572 \\};
2573 \\
2574 \\/// B doc
2575 \\const B = struct {
2576 \\ //! B tld 1
2577 \\ //! B tld 2
2578 \\ //! B tld 3
2579 \\
2580 \\ /// b doc
2581 \\ b: u32,
2582 \\};
2583 \\
2584 \\/// C doc
2585 \\const C = struct {
2586 \\ //! C tld 1
2587 \\ //! C tld 2
2588 \\ //! C tld 3
2589 \\
2590 \\ /// c1 doc
2591 \\ c1: u32,
2592 \\
2593 \\ //! C tld 4
2594 \\ //! C tld 5
2595 \\ //! C tld 6
2596 \\
2597 \\ /// c2 doc
2598 \\ c2: u32,
2599 \\};
2600 \\
2601 );
2602 try testCanonical(
2603 \\//! Top-level documentation.
2604 \\
2605 \\/// This is A
2606 \\pub const A = usize;
2607 \\
2608 );
2609 try testCanonical(
2610 \\//! Nothing here
2611 \\
2612 );
2613}
2614
2559const std = @import("std");2615const std = @import("std");
2560const mem = std.mem;2616const mem = std.mem;
2561const warn = std.debug.warn;2617const warn = std.debug.warn;
lib/std/zig/render.zig+11
...@@ -299,6 +299,17 @@ fn renderTopLevelDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, i...@@ -299,6 +299,17 @@ fn renderTopLevelDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, i
299 assert(!decl.requireSemiColon());299 assert(!decl.requireSemiColon());
300 try renderExpression(allocator, stream, tree, indent, start_col, decl, Space.Newline);300 try renderExpression(allocator, stream, tree, indent, start_col, decl, Space.Newline);
301 },301 },
302
303 ast.Node.Id.DocComment => {
304 const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl);
305 var it = comment.lines.iterator(0);
306 while (it.next()) |line_token_index| {
307 try renderToken(tree, stream, line_token_index.*, indent, start_col, Space.Newline);
308 if (it.peek()) |_| {
309 try stream.writeByteNTimes(' ', indent);
310 }
311 }
312 },
302 else => unreachable,313 else => unreachable,
303 }314 }
304}315}
lib/std/zig/tokenizer.zig+13-1
...@@ -142,6 +142,7 @@ pub const Token = struct {...@@ -142,6 +142,7 @@ pub const Token = struct {
142 FloatLiteral,142 FloatLiteral,
143 LineComment,143 LineComment,
144 DocComment,144 DocComment,
145 ContainerDocComment,
145 BracketStarBracket,146 BracketStarBracket,
146 BracketStarCBracket,147 BracketStarCBracket,
147 ShebangLine,148 ShebangLine,
...@@ -211,6 +212,7 @@ pub const Token = struct {...@@ -211,6 +212,7 @@ pub const Token = struct {
211 .FloatLiteral => "FloatLiteral",212 .FloatLiteral => "FloatLiteral",
212 .LineComment => "LineComment",213 .LineComment => "LineComment",
213 .DocComment => "DocComment",214 .DocComment => "DocComment",
215 .ContainerDocComment => "ContainerDocComment",
214 .ShebangLine => "ShebangLine",216 .ShebangLine => "ShebangLine",
215217
216 .Bang => "!",218 .Bang => "!",
...@@ -387,6 +389,7 @@ pub const Tokenizer = struct {...@@ -387,6 +389,7 @@ pub const Tokenizer = struct {
387 LineComment,389 LineComment,
388 DocCommentStart,390 DocCommentStart,
389 DocComment,391 DocComment,
392 ContainerDocComment,
390 Zero,393 Zero,
391 IntegerLiteral,394 IntegerLiteral,
392 IntegerLiteralWithRadix,395 IntegerLiteralWithRadix,
...@@ -1076,6 +1079,10 @@ pub const Tokenizer = struct {...@@ -1076,6 +1079,10 @@ pub const Tokenizer = struct {
1076 '/' => {1079 '/' => {
1077 state = State.DocCommentStart;1080 state = State.DocCommentStart;
1078 },1081 },
1082 '!' => {
1083 result.id = Token.Id.ContainerDocComment;
1084 state = State.ContainerDocComment;
1085 },
1079 '\n' => break,1086 '\n' => break,
1080 else => {1087 else => {
1081 state = State.LineComment;1088 state = State.LineComment;
...@@ -1096,7 +1103,7 @@ pub const Tokenizer = struct {...@@ -1096,7 +1103,7 @@ pub const Tokenizer = struct {
1096 self.checkLiteralCharacter();1103 self.checkLiteralCharacter();
1097 },1104 },
1098 },1105 },
1099 State.LineComment, State.DocComment => switch (c) {1106 State.LineComment, State.DocComment, State.ContainerDocComment => switch (c) {
1100 '\n' => break,1107 '\n' => break,
1101 else => self.checkLiteralCharacter(),1108 else => self.checkLiteralCharacter(),
1102 },1109 },
...@@ -1234,6 +1241,9 @@ pub const Tokenizer = struct {...@@ -1234,6 +1241,9 @@ pub const Tokenizer = struct {
1234 State.DocComment, State.DocCommentStart => {1241 State.DocComment, State.DocCommentStart => {
1235 result.id = Token.Id.DocComment;1242 result.id = Token.Id.DocComment;
1236 },1243 },
1244 State.ContainerDocComment => {
1245 result.id = Token.Id.ContainerDocComment;
1246 },
12371247
1238 State.NumberDot,1248 State.NumberDot,
1239 State.NumberDotHex,1249 State.NumberDotHex,
...@@ -1601,6 +1611,8 @@ test "tokenizer - line comment and doc comment" {...@@ -1601,6 +1611,8 @@ test "tokenizer - line comment and doc comment" {
1601 testTokenize("/// a", [_]Token.Id{Token.Id.DocComment});1611 testTokenize("/// a", [_]Token.Id{Token.Id.DocComment});
1602 testTokenize("///", [_]Token.Id{Token.Id.DocComment});1612 testTokenize("///", [_]Token.Id{Token.Id.DocComment});
1603 testTokenize("////", [_]Token.Id{Token.Id.LineComment});1613 testTokenize("////", [_]Token.Id{Token.Id.LineComment});
1614 testTokenize("//!", [_]Token.Id{Token.Id.ContainerDocComment});
1615 testTokenize("//!!", [_]Token.Id{Token.Id.ContainerDocComment});
1604}1616}
16051617
1606test "tokenizer - line comment followed by identifier" {1618test "tokenizer - line comment followed by identifier" {
src-self-hosted/translate_c.zig-1
...@@ -174,7 +174,6 @@ pub fn translate(...@@ -174,7 +174,6 @@ pub fn translate(
174 tree.root_node.* = ast.Node.Root{174 tree.root_node.* = ast.Node.Root{
175 .base = ast.Node{ .id = ast.Node.Id.Root },175 .base = ast.Node{ .id = ast.Node.Id.Root },
176 .decls = ast.Node.Root.DeclList.init(arena),176 .decls = ast.Node.Root.DeclList.init(arena),
177 .doc_comments = null,
178 // initialized with the eof token at the end177 // initialized with the eof token at the end
179 .eof_token = undefined,178 .eof_token = undefined,
180 };179 };