authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-10-12 12:38:09+03:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-11-14 16:25:06+02:00
logb92f42d1f4b91bb343558f72af84ea052da4ac98
treeec297a65dcd93c06557f0e0a76353f4682004b46
parentd89f39d71949c85b26f2ccd4071c9445aa8b6d7c
signaturelock-open Commit is signed but in an unrecognized format.

implemented container doc comments in stage 2

6 files changed, 108 insertions(+), 10 deletions(-)

lib/std/zig/ast.zig-1
......@@ -576,7 +576,6 @@ pub const Node = struct {
576576
577577 pub const Root = struct {
578578 base: Node,
579 doc_comments: ?*DocComment,
580579 decls: DeclList,
581580 eof_token: TokenIndex,
582581
lib/std/zig/parse.zig+28-7
......@@ -58,13 +58,6 @@ fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error
5858 node.* = Node.Root{
5959 .base = Node{ .id = .Root },
6060 .decls = undefined,
61 // TODO: Because zig fmt collapses consecutive comments separated by blank lines into
62 // a single multi-line comment, it is currently impossible to have a container-level
63 // doc comment and NO doc comment on the first decl. For now, simply
64 // ignore the problem and assume that there will be no container-level
65 // doc comments.
66 // See: https://github.com/ziglang/zig/issues/2288
67 .doc_comments = null,
6861 .eof_token = undefined,
6962 };
7063 node.decls = parseContainerMembers(arena, it, tree) catch |err| {
......@@ -94,6 +87,11 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
9487 var list = Node.Root.DeclList.init(arena);
9588
9689 while (true) {
90 if (try parseContainerDocComments(arena, it, tree)) |node| {
91 try list.push(node);
92 continue;
93 }
94
9795 const doc_comments = try parseDocComment(arena, it, tree);
9896
9997 if (try parseTestDecl(arena, it, tree)) |node| {
......@@ -155,12 +153,35 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
155153 continue;
156154 }
157155
156 // Dangling doc comment
157 if (doc_comments != null) {
158 try tree.errors.push(AstError{
159 .UnattachedDocComment = AstError.UnattachedDocComment{ .token = doc_comments.?.firstToken() },
160 });
161 }
158162 break;
159163 }
160164
161165 return list;
162166}
163167
168/// Eat a multiline container doc comment
169fn parseContainerDocComments(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
170 var lines = Node.DocComment.LineList.init(arena);
171 while (eatToken(it, .ContainerDocComment)) |line| {
172 try lines.push(line);
173 }
174
175 if (lines.len == 0) return null;
176
177 const node = try arena.create(Node.DocComment);
178 node.* = Node.DocComment{
179 .base = Node{ .id = .DocComment },
180 .lines = lines,
181 };
182 return &node.base;
183}
184
164185/// TestDecl <- KEYWORD_test STRINGLITERAL Block
165186fn parseTestDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
166187 const test_token = eatToken(it, .Keyword_test) orelse return null;
lib/std/zig/parser_test.zig+56
......@@ -2556,6 +2556,62 @@ test "zig fmt: comments at several places in struct init" {
25562556 );
25572557}
25582558
2559test "zig fmt: top level doc comments" {
2560 try testCanonical(
2561 \\//! tld 1
2562 \\//! tld 2
2563 \\//! tld 3
2564 \\
2565 \\// comment
2566 \\
2567 \\/// A doc
2568 \\const A = struct {
2569 \\ //! A tld 1
2570 \\ //! A tld 2
2571 \\ //! A tld 3
2572 \\};
2573 \\
2574 \\/// B doc
2575 \\const B = struct {
2576 \\ //! B tld 1
2577 \\ //! B tld 2
2578 \\ //! B tld 3
2579 \\
2580 \\ /// b doc
2581 \\ b: u32,
2582 \\};
2583 \\
2584 \\/// C doc
2585 \\const C = struct {
2586 \\ //! C tld 1
2587 \\ //! C tld 2
2588 \\ //! C tld 3
2589 \\
2590 \\ /// c1 doc
2591 \\ c1: u32,
2592 \\
2593 \\ //! C tld 4
2594 \\ //! C tld 5
2595 \\ //! C tld 6
2596 \\
2597 \\ /// c2 doc
2598 \\ c2: u32,
2599 \\};
2600 \\
2601 );
2602 try testCanonical(
2603 \\//! Top-level documentation.
2604 \\
2605 \\/// This is A
2606 \\pub const A = usize;
2607 \\
2608 );
2609 try testCanonical(
2610 \\//! Nothing here
2611 \\
2612 );
2613}
2614
25592615const std = @import("std");
25602616const mem = std.mem;
25612617const warn = std.debug.warn;
lib/std/zig/render.zig+11
......@@ -299,6 +299,17 @@ fn renderTopLevelDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, i
299299 assert(!decl.requireSemiColon());
300300 try renderExpression(allocator, stream, tree, indent, start_col, decl, Space.Newline);
301301 },
302
303 ast.Node.Id.DocComment => {
304 const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl);
305 var it = comment.lines.iterator(0);
306 while (it.next()) |line_token_index| {
307 try renderToken(tree, stream, line_token_index.*, indent, start_col, Space.Newline);
308 if (it.peek()) |_| {
309 try stream.writeByteNTimes(' ', indent);
310 }
311 }
312 },
302313 else => unreachable,
303314 }
304315}
lib/std/zig/tokenizer.zig+13-1
......@@ -142,6 +142,7 @@ pub const Token = struct {
142142 FloatLiteral,
143143 LineComment,
144144 DocComment,
145 ContainerDocComment,
145146 BracketStarBracket,
146147 BracketStarCBracket,
147148 ShebangLine,
......@@ -211,6 +212,7 @@ pub const Token = struct {
211212 .FloatLiteral => "FloatLiteral",
212213 .LineComment => "LineComment",
213214 .DocComment => "DocComment",
215 .ContainerDocComment => "ContainerDocComment",
214216 .ShebangLine => "ShebangLine",
215217
216218 .Bang => "!",
......@@ -387,6 +389,7 @@ pub const Tokenizer = struct {
387389 LineComment,
388390 DocCommentStart,
389391 DocComment,
392 ContainerDocComment,
390393 Zero,
391394 IntegerLiteral,
392395 IntegerLiteralWithRadix,
......@@ -1076,6 +1079,10 @@ pub const Tokenizer = struct {
10761079 '/' => {
10771080 state = State.DocCommentStart;
10781081 },
1082 '!' => {
1083 result.id = Token.Id.ContainerDocComment;
1084 state = State.ContainerDocComment;
1085 },
10791086 '\n' => break,
10801087 else => {
10811088 state = State.LineComment;
......@@ -1096,7 +1103,7 @@ pub const Tokenizer = struct {
10961103 self.checkLiteralCharacter();
10971104 },
10981105 },
1099 State.LineComment, State.DocComment => switch (c) {
1106 State.LineComment, State.DocComment, State.ContainerDocComment => switch (c) {
11001107 '\n' => break,
11011108 else => self.checkLiteralCharacter(),
11021109 },
......@@ -1234,6 +1241,9 @@ pub const Tokenizer = struct {
12341241 State.DocComment, State.DocCommentStart => {
12351242 result.id = Token.Id.DocComment;
12361243 },
1244 State.ContainerDocComment => {
1245 result.id = Token.Id.ContainerDocComment;
1246 },
12371247
12381248 State.NumberDot,
12391249 State.NumberDotHex,
......@@ -1601,6 +1611,8 @@ test "tokenizer - line comment and doc comment" {
16011611 testTokenize("/// a", [_]Token.Id{Token.Id.DocComment});
16021612 testTokenize("///", [_]Token.Id{Token.Id.DocComment});
16031613 testTokenize("////", [_]Token.Id{Token.Id.LineComment});
1614 testTokenize("//!", [_]Token.Id{Token.Id.ContainerDocComment});
1615 testTokenize("//!!", [_]Token.Id{Token.Id.ContainerDocComment});
16041616}
16051617
16061618test "tokenizer - line comment followed by identifier" {
src-self-hosted/translate_c.zig-1
......@@ -174,7 +174,6 @@ pub fn translate(
174174 tree.root_node.* = ast.Node.Root{
175175 .base = ast.Node{ .id = ast.Node.Id.Root },
176176 .decls = ast.Node.Root.DeclList.init(arena),
177 .doc_comments = null,
178177 // initialized with the eof token at the end
179178 .eof_token = undefined,
180179 };