authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-11-17 22:31:12+00:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-11-17 22:31:12+00:00
log57b8614a5a287d0a312b1cade463ec5485f0518f
treefda14f96b79a6878208ab483027f9fad97cf2460
parent4e28d7a5f7d6346acc42a7524dd77fa5f9322029
parent314cb707fce553e51d2ffd5c1ea506fbd1acdf76
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #3697 from Vexu/container-docs

Implement container level doc comments

14 files changed, 190 insertions(+), 25 deletions(-)

doc/docgen.zig+1
......@@ -856,6 +856,7 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok
856856
857857 .LineComment,
858858 .DocComment,
859 .ContainerDocComment,
859860 .ShebangLine,
860861 => {
861862 try out.write("<span class=\"tok-comment\">");
lib/std/special/docs/index.html+1-1
......@@ -484,7 +484,7 @@
484484 doc comments.
485485 </p>
486486 </div>
487 <div id="fnDocs" class="hidden"></div>
487 <div id="tldDocs" class="hidden"></div>
488488 <div id="sectFnErrors" class="hidden">
489489 <h2>Errors</h2>
490490 <div id="fnErrorsAnyError">
lib/std/special/docs/main.js+19-12
......@@ -20,7 +20,7 @@
2020 var domListValues = document.getElementById("listValues");
2121 var domFnProto = document.getElementById("fnProto");
2222 var domFnProtoCode = document.getElementById("fnProtoCode");
23 var domFnDocs = document.getElementById("fnDocs");
23 var domTldDocs = document.getElementById("tldDocs");
2424 var domSectFnErrors = document.getElementById("sectFnErrors");
2525 var domListFnErrors = document.getElementById("listFnErrors");
2626 var domTableFnErrors = document.getElementById("tableFnErrors");
......@@ -34,7 +34,6 @@
3434 var domListSearchResults = document.getElementById("listSearchResults");
3535 var domSectSearchNoResults = document.getElementById("sectSearchNoResults");
3636 var domSectInfo = document.getElementById("sectInfo");
37 var domListInfo = document.getElementById("listInfo");
3837 var domTdTarget = document.getElementById("tdTarget");
3938 var domTdZigVer = document.getElementById("tdZigVer");
4039 var domHdrName = document.getElementById("hdrName");
......@@ -102,7 +101,7 @@
102101 function render() {
103102 domStatus.classList.add("hidden");
104103 domFnProto.classList.add("hidden");
105 domFnDocs.classList.add("hidden");
104 domTldDocs.classList.add("hidden");
106105 domSectPkgs.classList.add("hidden");
107106 domSectTypes.classList.add("hidden");
108107 domSectNamespaces.classList.add("hidden");
......@@ -190,11 +189,11 @@
190189
191190 var docs = zigAnalysis.astNodes[decl.src].docs;
192191 if (docs != null) {
193 domFnDocs.innerHTML = markdown(docs);
192 domTldDocs.innerHTML = markdown(docs);
194193 } else {
195 domFnDocs.innerHTML = '<p>There are no doc comments for this declaration.</p>';
194 domTldDocs.innerHTML = '<p>There are no doc comments for this declaration.</p>';
196195 }
197 domFnDocs.classList.remove("hidden");
196 domTldDocs.classList.remove("hidden");
198197 }
199198
200199 function typeIsErrSet(typeIndex) {
......@@ -274,8 +273,8 @@
274273 docsSource = protoSrcNode.docs;
275274 }
276275 if (docsSource != null) {
277 domFnDocs.innerHTML = markdown(docsSource);
278 domFnDocs.classList.remove("hidden");
276 domTldDocs.innerHTML = markdown(docsSource);
277 domTldDocs.classList.remove("hidden");
279278 }
280279 domFnProto.classList.remove("hidden");
281280 }
......@@ -893,8 +892,8 @@
893892
894893 var docs = zigAnalysis.astNodes[decl.src].docs;
895894 if (docs != null) {
896 domFnDocs.innerHTML = markdown(docs);
897 domFnDocs.classList.remove("hidden");
895 domTldDocs.innerHTML = markdown(docs);
896 domTldDocs.classList.remove("hidden");
898897 }
899898
900899 domFnProto.classList.remove("hidden");
......@@ -906,8 +905,8 @@
906905
907906 var docs = zigAnalysis.astNodes[decl.src].docs;
908907 if (docs != null) {
909 domFnDocs.innerHTML = markdown(docs);
910 domFnDocs.classList.remove("hidden");
908 domTldDocs.innerHTML = markdown(docs);
909 domTldDocs.classList.remove("hidden");
911910 }
912911
913912 domFnProto.classList.remove("hidden");
......@@ -957,6 +956,14 @@
957956 varsList.sort(byNameProperty);
958957 valsList.sort(byNameProperty);
959958
959 if (container.src != null) {
960 var docs = zigAnalysis.astNodes[container.src].docs;
961 if (docs != null) {
962 domTldDocs.innerHTML = markdown(docs);
963 domTldDocs.classList.remove("hidden");
964 }
965 }
966
960967 if (typesList.length !== 0) {
961968 resizeDomList(domListTypes, typesList.length, '<li><a href="#"></a></li>');
962969 for (var i = 0; i < typesList.length; i += 1) {
lib/std/zig/ast.zig-2
......@@ -576,7 +576,6 @@ pub const Node = struct {
576576
577577 pub const Root = struct {
578578 base: Node,
579 doc_comments: ?*DocComment,
580579 decls: DeclList,
581580 eof_token: TokenIndex,
582581
......@@ -2254,7 +2253,6 @@ pub const Node = struct {
22542253test "iterate" {
22552254 var root = Node.Root{
22562255 .base = Node{ .id = Node.Id.Root },
2257 .doc_comments = null,
22582256 .decls = Node.Root.DeclList.init(std.debug.global_allocator),
22592257 .eof_token = 0,
22602258 };
lib/std/zig/parse.zig+28-7
......@@ -58,13 +58,6 @@ fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error
5858 node.* = Node.Root{
5959 .base = Node{ .id = .Root },
6060 .decls = undefined,
61 // TODO: Because zig fmt collapses consecutive comments separated by blank lines into
62 // a single multi-line comment, it is currently impossible to have a container-level
63 // doc comment and NO doc comment on the first decl. For now, simply
64 // ignore the problem and assume that there will be no container-level
65 // doc comments.
66 // See: https://github.com/ziglang/zig/issues/2288
67 .doc_comments = null,
6861 .eof_token = undefined,
6962 };
7063 node.decls = parseContainerMembers(arena, it, tree) catch |err| {
......@@ -94,6 +87,11 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
9487 var list = Node.Root.DeclList.init(arena);
9588
9689 while (true) {
90 if (try parseContainerDocComments(arena, it, tree)) |node| {
91 try list.push(node);
92 continue;
93 }
94
9795 const doc_comments = try parseDocComment(arena, it, tree);
9896
9997 if (try parseTestDecl(arena, it, tree)) |node| {
......@@ -155,12 +153,35 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
155153 continue;
156154 }
157155
156 // Dangling doc comment
157 if (doc_comments != null) {
158 try tree.errors.push(AstError{
159 .UnattachedDocComment = AstError.UnattachedDocComment{ .token = doc_comments.?.firstToken() },
160 });
161 }
158162 break;
159163 }
160164
161165 return list;
162166}
163167
168/// Eat a multiline container doc comment
169fn parseContainerDocComments(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
170 var lines = Node.DocComment.LineList.init(arena);
171 while (eatToken(it, .ContainerDocComment)) |line| {
172 try lines.push(line);
173 }
174
175 if (lines.len == 0) return null;
176
177 const node = try arena.create(Node.DocComment);
178 node.* = Node.DocComment{
179 .base = Node{ .id = .DocComment },
180 .lines = lines,
181 };
182 return &node.base;
183}
184
164185/// TestDecl <- KEYWORD_test STRINGLITERAL Block
165186fn parseTestDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
166187 const test_token = eatToken(it, .Keyword_test) orelse return null;
lib/std/zig/parser_test.zig+56
......@@ -2566,6 +2566,62 @@ test "zig fmt: comments at several places in struct init" {
25662566 );
25672567}
25682568
2569test "zig fmt: top level doc comments" {
2570 try testCanonical(
2571 \\//! tld 1
2572 \\//! tld 2
2573 \\//! tld 3
2574 \\
2575 \\// comment
2576 \\
2577 \\/// A doc
2578 \\const A = struct {
2579 \\ //! A tld 1
2580 \\ //! A tld 2
2581 \\ //! A tld 3
2582 \\};
2583 \\
2584 \\/// B doc
2585 \\const B = struct {
2586 \\ //! B tld 1
2587 \\ //! B tld 2
2588 \\ //! B tld 3
2589 \\
2590 \\ /// b doc
2591 \\ b: u32,
2592 \\};
2593 \\
2594 \\/// C doc
2595 \\const C = struct {
2596 \\ //! C tld 1
2597 \\ //! C tld 2
2598 \\ //! C tld 3
2599 \\
2600 \\ /// c1 doc
2601 \\ c1: u32,
2602 \\
2603 \\ //! C tld 4
2604 \\ //! C tld 5
2605 \\ //! C tld 6
2606 \\
2607 \\ /// c2 doc
2608 \\ c2: u32,
2609 \\};
2610 \\
2611 );
2612 try testCanonical(
2613 \\//! Top-level documentation.
2614 \\
2615 \\/// This is A
2616 \\pub const A = usize;
2617 \\
2618 );
2619 try testCanonical(
2620 \\//! Nothing here
2621 \\
2622 );
2623}
2624
25692625const std = @import("std");
25702626const mem = std.mem;
25712627const warn = std.debug.warn;
lib/std/zig/render.zig+11
......@@ -299,6 +299,17 @@ fn renderTopLevelDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, i
299299 assert(!decl.requireSemiColon());
300300 try renderExpression(allocator, stream, tree, indent, start_col, decl, Space.Newline);
301301 },
302
303 ast.Node.Id.DocComment => {
304 const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl);
305 var it = comment.lines.iterator(0);
306 while (it.next()) |line_token_index| {
307 try renderToken(tree, stream, line_token_index.*, indent, start_col, Space.Newline);
308 if (it.peek()) |_| {
309 try stream.writeByteNTimes(' ', indent);
310 }
311 }
312 },
302313 else => unreachable,
303314 }
304315}
lib/std/zig/tokenizer.zig+13-1
......@@ -142,6 +142,7 @@ pub const Token = struct {
142142 FloatLiteral,
143143 LineComment,
144144 DocComment,
145 ContainerDocComment,
145146 BracketStarBracket,
146147 BracketStarCBracket,
147148 ShebangLine,
......@@ -211,6 +212,7 @@ pub const Token = struct {
211212 .FloatLiteral => "FloatLiteral",
212213 .LineComment => "LineComment",
213214 .DocComment => "DocComment",
215 .ContainerDocComment => "ContainerDocComment",
214216 .ShebangLine => "ShebangLine",
215217
216218 .Bang => "!",
......@@ -387,6 +389,7 @@ pub const Tokenizer = struct {
387389 LineComment,
388390 DocCommentStart,
389391 DocComment,
392 ContainerDocComment,
390393 Zero,
391394 IntegerLiteral,
392395 IntegerLiteralWithRadix,
......@@ -1076,6 +1079,10 @@ pub const Tokenizer = struct {
10761079 '/' => {
10771080 state = State.DocCommentStart;
10781081 },
1082 '!' => {
1083 result.id = Token.Id.ContainerDocComment;
1084 state = State.ContainerDocComment;
1085 },
10791086 '\n' => break,
10801087 else => {
10811088 state = State.LineComment;
......@@ -1096,7 +1103,7 @@ pub const Tokenizer = struct {
10961103 self.checkLiteralCharacter();
10971104 },
10981105 },
1099 State.LineComment, State.DocComment => switch (c) {
1106 State.LineComment, State.DocComment, State.ContainerDocComment => switch (c) {
11001107 '\n' => break,
11011108 else => self.checkLiteralCharacter(),
11021109 },
......@@ -1234,6 +1241,9 @@ pub const Tokenizer = struct {
12341241 State.DocComment, State.DocCommentStart => {
12351242 result.id = Token.Id.DocComment;
12361243 },
1244 State.ContainerDocComment => {
1245 result.id = Token.Id.ContainerDocComment;
1246 },
12371247
12381248 State.NumberDot,
12391249 State.NumberDotHex,
......@@ -1601,6 +1611,8 @@ test "tokenizer - line comment and doc comment" {
16011611 testTokenize("/// a", [_]Token.Id{Token.Id.DocComment});
16021612 testTokenize("///", [_]Token.Id{Token.Id.DocComment});
16031613 testTokenize("////", [_]Token.Id{Token.Id.LineComment});
1614 testTokenize("//!", [_]Token.Id{Token.Id.ContainerDocComment});
1615 testTokenize("//!!", [_]Token.Id{Token.Id.ContainerDocComment});
16041616}
16051617
16061618test "tokenizer - line comment followed by identifier" {
src-self-hosted/translate_c.zig-1
......@@ -174,7 +174,6 @@ pub fn translate(
174174 tree.root_node.* = ast.Node.Root{
175175 .base = ast.Node{ .id = ast.Node.Id.Root },
176176 .decls = ast.Node.Root.DeclList.init(arena),
177 .doc_comments = null,
178177 // initialized with the eof token at the end
179178 .eof_token = undefined,
180179 };
src/all_types.hpp+1
......@@ -968,6 +968,7 @@ struct AstNodeContainerDecl {
968968 AstNode *init_arg_expr; // enum(T), struct(endianness), or union(T), or union(enum(T))
969969 ZigList<AstNode *> fields;
970970 ZigList<AstNode *> decls;
971 Buf doc_comments;
971972
972973 ContainerKind kind;
973974 ContainerLayout layout;
src/dump_analysis.cpp+1
......@@ -1088,6 +1088,7 @@ static void anal_dump_node(AnalDumpCtx *ctx, const AstNode *node) {
10881088 break;
10891089 case NodeTypeContainerDecl:
10901090 field_nodes = &node->data.container_decl.fields;
1091 doc_comments_buf = &node->data.container_decl.doc_comments;
10911092 break;
10921093 default:
10931094 break;
src/parser.cpp+26-1
......@@ -493,6 +493,9 @@ static AstNode *ast_parse_root(ParseContext *pc) {
493493 node->data.container_decl.layout = ContainerLayoutAuto;
494494 node->data.container_decl.kind = ContainerKindStruct;
495495 node->data.container_decl.is_root = true;
496 if (buf_len(&members.doc_comments) != 0) {
497 node->data.container_decl.doc_comments = members.doc_comments;
498 }
496499
497500 return node;
498501}
......@@ -514,6 +517,21 @@ static Token *ast_parse_doc_comments(ParseContext *pc, Buf *buf) {
514517 return first_doc_token;
515518}
516519
520static void ast_parse_container_doc_comments(ParseContext *pc, Buf *buf) {
521 if (buf_len(buf) != 0 && peek_token(pc)->id == TokenIdContainerDocComment) {
522 buf_append_char(buf, '\n');
523 }
524 Token *doc_token = nullptr;
525 while ((doc_token = eat_token_if(pc, TokenIdContainerDocComment))) {
526 if (buf->list.length == 0) {
527 buf_resize(buf, 0);
528 }
529 // chops off '//!' but leaves '\n'
530 buf_append_mem(buf, buf_ptr(pc->buf) + doc_token->start_pos + 3,
531 doc_token->end_pos - doc_token->start_pos - 3);
532 }
533}
534
517535// ContainerMembers
518536// <- TestDecl ContainerMembers
519537// / TopLevelComptime ContainerMembers
......@@ -523,7 +541,11 @@ static Token *ast_parse_doc_comments(ParseContext *pc, Buf *buf) {
523541// /
524542static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {
525543 AstNodeContainerDecl res = {};
544 Buf tld_doc_comment_buf = BUF_INIT;
545 buf_resize(&tld_doc_comment_buf, 0);
526546 for (;;) {
547 ast_parse_container_doc_comments(pc, &tld_doc_comment_buf);
548
527549 AstNode *test_decl = ast_parse_test_decl(pc);
528550 if (test_decl != nullptr) {
529551 res.decls.append(test_decl);
......@@ -566,7 +588,7 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {
566588
567589 break;
568590 }
569
591 res.doc_comments = tld_doc_comment_buf;
570592 return res;
571593}
572594
......@@ -2802,6 +2824,9 @@ static AstNode *ast_parse_container_decl_auto(ParseContext *pc) {
28022824
28032825 res->data.container_decl.fields = members.fields;
28042826 res->data.container_decl.decls = members.decls;
2827 if (buf_len(&members.doc_comments) != 0) {
2828 res->data.container_decl.doc_comments = members.doc_comments;
2829 }
28052830 return res;
28062831}
28072832
src/tokenizer.cpp+32
......@@ -198,6 +198,7 @@ enum TokenizeState {
198198 TokenizeStateSawSlash,
199199 TokenizeStateSawSlash2,
200200 TokenizeStateSawSlash3,
201 TokenizeStateSawSlashBang,
201202 TokenizeStateSawBackslash,
202203 TokenizeStateSawPercent,
203204 TokenizeStateSawPlus,
......@@ -209,6 +210,7 @@ enum TokenizeState {
209210 TokenizeStateSawBar,
210211 TokenizeStateSawBarBar,
211212 TokenizeStateDocComment,
213 TokenizeStateContainerDocComment,
212214 TokenizeStateLineComment,
213215 TokenizeStateLineString,
214216 TokenizeStateLineStringEnd,
......@@ -938,6 +940,9 @@ void tokenize(Buf *buf, Tokenization *out) {
938940 case '/':
939941 t.state = TokenizeStateSawSlash3;
940942 break;
943 case '!':
944 t.state = TokenizeStateSawSlashBang;
945 break;
941946 case '\n':
942947 cancel_token(&t);
943948 t.state = TokenizeStateStart;
......@@ -965,6 +970,19 @@ void tokenize(Buf *buf, Tokenization *out) {
965970 break;
966971 }
967972 break;
973 case TokenizeStateSawSlashBang:
974 switch (c) {
975 case '\n':
976 set_token_id(&t, t.cur_tok, TokenIdContainerDocComment);
977 end_token(&t);
978 t.state = TokenizeStateStart;
979 break;
980 default:
981 set_token_id(&t, t.cur_tok, TokenIdContainerDocComment);
982 t.state = TokenizeStateContainerDocComment;
983 break;
984 }
985 break;
968986 case TokenizeStateSawBackslash:
969987 switch (c) {
970988 case '\\':
......@@ -1055,6 +1073,17 @@ void tokenize(Buf *buf, Tokenization *out) {
10551073 break;
10561074 }
10571075 break;
1076 case TokenizeStateContainerDocComment:
1077 switch (c) {
1078 case '\n':
1079 end_token(&t);
1080 t.state = TokenizeStateStart;
1081 break;
1082 default:
1083 // do nothing
1084 break;
1085 }
1086 break;
10581087 case TokenizeStateSymbolFirstC:
10591088 switch (c) {
10601089 case '"':
......@@ -1545,6 +1574,7 @@ void tokenize(Buf *buf, Tokenization *out) {
15451574 case TokenizeStateSawBarBar:
15461575 case TokenizeStateLBracket:
15471576 case TokenizeStateDocComment:
1577 case TokenizeStateContainerDocComment:
15481578 end_token(&t);
15491579 break;
15501580 case TokenizeStateSawDotDot:
......@@ -1559,6 +1589,7 @@ void tokenize(Buf *buf, Tokenization *out) {
15591589 case TokenizeStateLineComment:
15601590 case TokenizeStateSawSlash2:
15611591 case TokenizeStateSawSlash3:
1592 case TokenizeStateSawSlashBang:
15621593 break;
15631594 }
15641595 if (t.state != TokenizeStateError) {
......@@ -1606,6 +1637,7 @@ const char * token_name(TokenId id) {
16061637 case TokenIdDash: return "-";
16071638 case TokenIdDivEq: return "/=";
16081639 case TokenIdDocComment: return "DocComment";
1640 case TokenIdContainerDocComment: return "ContainerDocComment";
16091641 case TokenIdDot: return ".";
16101642 case TokenIdDotStar: return ".*";
16111643 case TokenIdEllipsis2: return "..";
src/tokenizer.hpp+1
......@@ -43,6 +43,7 @@ enum TokenId {
4343 TokenIdDash,
4444 TokenIdDivEq,
4545 TokenIdDocComment,
46 TokenIdContainerDocComment,
4647 TokenIdDot,
4748 TokenIdDotStar,
4849 TokenIdEllipsis2,