authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-01-23 16:21:26-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-01-23 16:24:46-07:00
log8a697262099d64b2ff222dc9271493fe3b5f2b76
tree78ea36adce4a5fa5fa8739e38f408cb4f305c6db
parent8c23b81e8640df490e2168b9a623bbb57a7d353e

AstGen: doc comment fixups

* AstGen: use Ast.zig helper methods to avoid copy pasting token counting logic - take advantage of the `first_doc_comment` field we already have for param AST nodes * Add missing ZIR docs

3 files changed, 69 insertions(+), 41 deletions(-)

lib/std/zig/Ast.zig+18
...@@ -2120,6 +2120,14 @@ pub const full = struct {...@@ -2120,6 +2120,14 @@ pub const full = struct {
2120 section_node: Node.Index,2120 section_node: Node.Index,
2121 init_node: Node.Index,2121 init_node: Node.Index,
2122 };2122 };
2123
2124 pub fn firstToken(var_decl: VarDecl) TokenIndex {
2125 return var_decl.visib_token orelse
2126 var_decl.extern_export_token orelse
2127 var_decl.threadlocal_token orelse
2128 var_decl.comptime_token orelse
2129 var_decl.ast.mut_token;
2130 }
2123 };2131 };
21242132
2125 pub const If = struct {2133 pub const If = struct {
...@@ -2168,6 +2176,10 @@ pub const full = struct {...@@ -2168,6 +2176,10 @@ pub const full = struct {
2168 value_expr: Node.Index,2176 value_expr: Node.Index,
2169 align_expr: Node.Index,2177 align_expr: Node.Index,
2170 };2178 };
2179
2180 pub fn firstToken(cf: ContainerField) TokenIndex {
2181 return cf.comptime_token orelse cf.ast.name_token;
2182 }
2171 };2183 };
21722184
2173 pub const FnProto = struct {2185 pub const FnProto = struct {
...@@ -2197,6 +2209,12 @@ pub const full = struct {...@@ -2197,6 +2209,12 @@ pub const full = struct {
2197 type_expr: Node.Index,2209 type_expr: Node.Index,
2198 };2210 };
21992211
2212 pub fn firstToken(fn_proto: FnProto) TokenIndex {
2213 return fn_proto.visib_token orelse
2214 fn_proto.extern_export_inline_token orelse
2215 fn_proto.ast.fn_token;
2216 }
2217
2200 /// Abstracts over the fact that anytype and ... are not included2218 /// Abstracts over the fact that anytype and ... are not included
2201 /// in the params slice, since they are simple identifiers and2219 /// in the params slice, since they are simple identifiers and
2202 /// not sub-expressions.2220 /// not sub-expressions.
src/AstGen.zig+45-39
...@@ -1171,7 +1171,7 @@ fn fnProtoExpr(...@@ -1171,7 +1171,7 @@ fn fnProtoExpr(
1171 const main_tokens = tree.nodes.items(.main_token);1171 const main_tokens = tree.nodes.items(.main_token);
1172 const name_token = param.name_token orelse main_tokens[param_type_node];1172 const name_token = param.name_token orelse main_tokens[param_type_node];
1173 const tag: Zir.Inst.Tag = if (is_comptime) .param_comptime else .param;1173 const tag: Zir.Inst.Tag = if (is_comptime) .param_comptime else .param;
1174 const param_inst = try block_scope.addParam(&param_gz, tag, name_token, param_name);1174 const param_inst = try block_scope.addParam(&param_gz, tag, name_token, param_name, param.first_doc_comment);
1175 assert(param_inst_expected == param_inst);1175 assert(param_inst_expected == param_inst);
1176 }1176 }
1177 }1177 }
...@@ -3237,11 +3237,7 @@ fn fnDecl(...@@ -3237,11 +3237,7 @@ fn fnDecl(
3237 break :blk token_tags[maybe_inline_token] == .keyword_inline;3237 break :blk token_tags[maybe_inline_token] == .keyword_inline;
3238 };3238 };
32393239
3240 const doc_comment_index = try astgen.docCommentAsString(fn_name_token - 1 -3240 const doc_comment_index = try astgen.docCommentAsString(fn_proto.firstToken());
3241 @boolToInt(is_pub) -
3242 @boolToInt(is_export) -
3243 @boolToInt(is_extern) -
3244 @boolToInt(has_inline_keyword)); // TODO subtract noinline too
32453241
3246 const has_section_or_addrspace = fn_proto.ast.section_expr != 0 or fn_proto.ast.addrspace_expr != 0;3242 const has_section_or_addrspace = fn_proto.ast.section_expr != 0 or fn_proto.ast.addrspace_expr != 0;
3247 wip_members.nextDecl(is_pub, is_export, fn_proto.ast.align_expr != 0, has_section_or_addrspace);3243 wip_members.nextDecl(is_pub, is_export, fn_proto.ast.align_expr != 0, has_section_or_addrspace);
...@@ -3301,7 +3297,7 @@ fn fnDecl(...@@ -3301,7 +3297,7 @@ fn fnDecl(
3301 const main_tokens = tree.nodes.items(.main_token);3297 const main_tokens = tree.nodes.items(.main_token);
3302 const name_token = param.name_token orelse main_tokens[param_type_node];3298 const name_token = param.name_token orelse main_tokens[param_type_node];
3303 const tag: Zir.Inst.Tag = if (is_comptime) .param_comptime else .param;3299 const tag: Zir.Inst.Tag = if (is_comptime) .param_comptime else .param;
3304 const param_inst = try decl_gz.addParam(&param_gz, tag, name_token, param_name);3300 const param_inst = try decl_gz.addParam(&param_gz, tag, name_token, param_name, param.first_doc_comment);
3305 assert(param_inst_expected == param_inst);3301 assert(param_inst_expected == param_inst);
3306 break :param indexToRef(param_inst);3302 break :param indexToRef(param_inst);
3307 };3303 };
...@@ -3527,11 +3523,7 @@ fn globalVarDecl(...@@ -3527,11 +3523,7 @@ fn globalVarDecl(
3527 break :blk lib_name_str.index;3523 break :blk lib_name_str.index;
3528 } else 0;3524 } else 0;
35293525
3530 const doc_comment_index = try astgen.docCommentAsString(var_decl.ast.mut_token -3526 const doc_comment_index = try astgen.docCommentAsString(var_decl.firstToken());
3531 @boolToInt(is_pub) -
3532 @boolToInt(is_export) -
3533 @boolToInt(lib_name != 0) -
3534 @boolToInt(is_threadlocal));
35353527
3536 assert(var_decl.comptime_token == null); // handled by parser3528 assert(var_decl.comptime_token == null); // handled by parser
35373529
...@@ -3899,7 +3891,7 @@ fn structDeclInner(...@@ -3899,7 +3891,7 @@ fn structDeclInner(
3899 try typeExpr(&block_scope, &namespace.base, member.ast.type_expr);3891 try typeExpr(&block_scope, &namespace.base, member.ast.type_expr);
3900 wip_members.appendToField(@enumToInt(field_type));3892 wip_members.appendToField(@enumToInt(field_type));
39013893
3902 const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token);3894 const doc_comment_index = try astgen.docCommentAsString(member.firstToken());
3903 wip_members.appendToField(doc_comment_index);3895 wip_members.appendToField(doc_comment_index);
39043896
3905 known_has_bits = known_has_bits or nodeImpliesRuntimeBits(tree, member.ast.type_expr);3897 known_has_bits = known_has_bits or nodeImpliesRuntimeBits(tree, member.ast.type_expr);
...@@ -4016,7 +4008,7 @@ fn unionDeclInner(...@@ -4016,7 +4008,7 @@ fn unionDeclInner(
4016 const field_name = try astgen.identAsString(member.ast.name_token);4008 const field_name = try astgen.identAsString(member.ast.name_token);
4017 wip_members.appendToField(field_name);4009 wip_members.appendToField(field_name);
40184010
4019 const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token);4011 const doc_comment_index = try astgen.docCommentAsString(member.firstToken());
4020 wip_members.appendToField(doc_comment_index);4012 wip_members.appendToField(doc_comment_index);
40214013
4022 const have_type = member.ast.type_expr != 0;4014 const have_type = member.ast.type_expr != 0;
...@@ -4300,7 +4292,7 @@ fn containerDecl(...@@ -4300,7 +4292,7 @@ fn containerDecl(
4300 const field_name = try astgen.identAsString(member.ast.name_token);4292 const field_name = try astgen.identAsString(member.ast.name_token);
4301 wip_members.appendToField(field_name);4293 wip_members.appendToField(field_name);
43024294
4303 const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token);4295 const doc_comment_index = try astgen.docCommentAsString(member.firstToken());
4304 wip_members.appendToField(doc_comment_index);4296 wip_members.appendToField(doc_comment_index);
43054297
4306 const have_value = member.ast.value_expr != 0;4298 const have_value = member.ast.value_expr != 0;
...@@ -4533,9 +4525,6 @@ fn errorSetDecl(gz: *GenZir, rl: ResultLoc, node: Ast.Node.Index) InnerError!Zir...@@ -4533,9 +4525,6 @@ fn errorSetDecl(gz: *GenZir, rl: ResultLoc, node: Ast.Node.Index) InnerError!Zir
4533 switch (token_tags[tok_i]) {4525 switch (token_tags[tok_i]) {
4534 .doc_comment, .comma => {},4526 .doc_comment, .comma => {},
4535 .identifier => {4527 .identifier => {
4536 // TODO: maybe consider not using `docCommentAsString`
4537 // since we're already visiting the first doc comment
4538 // token.
4539 try astgen.extra.ensureUnusedCapacity(gpa, 2);4528 try astgen.extra.ensureUnusedCapacity(gpa, 2);
4540 const str_index = try astgen.identAsString(tok_i);4529 const str_index = try astgen.identAsString(tok_i);
4541 astgen.extra.appendAssumeCapacity(str_index);4530 astgen.extra.appendAssumeCapacity(str_index);
...@@ -8817,38 +8806,52 @@ fn identAsString(astgen: *AstGen, ident_token: Ast.TokenIndex) !u32 {...@@ -8817,38 +8806,52 @@ fn identAsString(astgen: *AstGen, ident_token: Ast.TokenIndex) !u32 {
8817 }8806 }
8818}8807}
88198808
8820/// Adds a doc comment block to `string_bytes` by walking backwards from `end_token`. 8809/// Adds a doc comment block to `string_bytes` by walking backwards from `end_token`.
8821/// `end_token` must point at the first token after the last doc coment line.8810/// `end_token` must point at the first token after the last doc coment line.
8822/// Returns 0 if no doc comment is present.8811/// Returns 0 if no doc comment is present.
8823fn docCommentAsString(astgen: *AstGen, end_token: Ast.TokenIndex) !u32 {8812fn docCommentAsString(astgen: *AstGen, end_token: Ast.TokenIndex) !u32 {
8813 if (end_token == 0) return @as(u32, 0);
8814
8815 const token_tags = astgen.tree.tokens.items(.tag);
8816
8817 var tok = end_token - 1;
8818 while (token_tags[tok] == .doc_comment) {
8819 if (tok == 0) break;
8820 tok -= 1;
8821 } else {
8822 tok += 1;
8823 }
8824 return docCommentAsStringFromFirst(astgen, end_token, tok);
8825}
8826
8827/// end_token must be > the index of the last doc comment.
8828fn docCommentAsStringFromFirst(
8829 astgen: *AstGen,
8830 end_token: Ast.TokenIndex,
8831 start_token: Ast.TokenIndex,
8832) !u32 {
8833 if (start_token == end_token) return 0;
8834
8824 const gpa = astgen.gpa;8835 const gpa = astgen.gpa;
8825 const string_bytes = &astgen.string_bytes;8836 const string_bytes = &astgen.string_bytes;
8826 const str_index = @intCast(u32, string_bytes.items.len);8837 const str_index = @intCast(u32, string_bytes.items.len);
8827 const token_tags = astgen.tree.tokens.items(.tag);
8828 const token_starts = astgen.tree.tokens.items(.start);8838 const token_starts = astgen.tree.tokens.items(.start);
88298839 const token_tags = astgen.tree.tokens.items(.tag);
8830 if (end_token == 0) return 0;
8831 const start_token: u32 = blk: {
8832 var tok = end_token - 1;
8833 while (token_tags[tok] == .doc_comment) {
8834 if (tok == 0) break;
8835 tok -= 1;
8836 } else {
8837 tok += 1;
8838 }
8839 break :blk tok;
8840 };
8841 if (start_token == end_token) return 0;
88428840
8843 const total_bytes = token_starts[end_token] - token_starts[start_token];8841 const total_bytes = token_starts[end_token] - token_starts[start_token];
8844 try string_bytes.ensureUnusedCapacity(gpa, total_bytes);8842 try string_bytes.ensureUnusedCapacity(gpa, total_bytes);
88458843
8846 var current_token = start_token;8844 var current_token = start_token;
8847 while (current_token < end_token) : (current_token += 1) {8845 while (current_token < end_token) : (current_token += 1) {
8848 const tok_bytes = astgen.tree.tokenSlice(current_token)[3..];8846 switch (token_tags[current_token]) {
8849 string_bytes.appendSliceAssumeCapacity(tok_bytes);8847 .doc_comment => {
8850 if (current_token != end_token - 1) {8848 const tok_bytes = astgen.tree.tokenSlice(current_token)[3..];
8851 string_bytes.appendAssumeCapacity('\n');8849 string_bytes.appendSliceAssumeCapacity(tok_bytes);
8850 if (current_token != end_token - 1) {
8851 string_bytes.appendAssumeCapacity('\n');
8852 }
8853 },
8854 else => break,
8852 }8855 }
8853 }8856 }
88548857
...@@ -9714,6 +9717,7 @@ const GenZir = struct {...@@ -9714,6 +9717,7 @@ const GenZir = struct {
9714 /// Absolute token index. This function does the conversion to Decl offset.9717 /// Absolute token index. This function does the conversion to Decl offset.
9715 abs_tok_index: Ast.TokenIndex,9718 abs_tok_index: Ast.TokenIndex,
9716 name: u32,9719 name: u32,
9720 first_doc_comment: ?Ast.TokenIndex,
9717 ) !Zir.Inst.Index {9721 ) !Zir.Inst.Index {
9718 const gpa = gz.astgen.gpa;9722 const gpa = gz.astgen.gpa;
9719 const param_body = param_gz.instructionsSlice();9723 const param_body = param_gz.instructionsSlice();
...@@ -9721,8 +9725,10 @@ const GenZir = struct {...@@ -9721,8 +9725,10 @@ const GenZir = struct {
9721 try gz.astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.Param).Struct.fields.len +9725 try gz.astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.Param).Struct.fields.len +
9722 param_body.len);9726 param_body.len);
97239727
9724 const doc_comment_index = try gz.astgen.docCommentAsString(abs_tok_index -9728 const doc_comment_index = if (first_doc_comment) |first|
9725 @boolToInt(tag == .param_comptime));9729 try gz.astgen.docCommentAsStringFromFirst(abs_tok_index, first)
9730 else
9731 0;
97269732
9727 const payload_index = gz.astgen.addExtraAssumeCapacity(Zir.Inst.Param{9733 const payload_index = gz.astgen.addExtraAssumeCapacity(Zir.Inst.Param{
9728 .name = name,9734 .name = name,
src/Zir.zig+6-2
...@@ -2767,7 +2767,11 @@ pub const Inst = struct {...@@ -2767,7 +2767,11 @@ pub const Inst = struct {
2767 };2767 };
2768 };2768 };
27692769
2770 /// Trailing: field_name: u32 // for every field: null terminated string index2770 /// Trailing:
2771 /// { // for every fields_len
2772 /// field_name: u32 // null terminated string index
2773 /// doc_comment: u32 // null terminated string index
2774 /// }
2771 pub const ErrorSetDecl = struct {2775 pub const ErrorSetDecl = struct {
2772 fields_len: u32,2776 fields_len: u32,
2773 };2777 };
...@@ -2906,7 +2910,7 @@ pub const Inst = struct {...@@ -2906,7 +2910,7 @@ pub const Inst = struct {
2906 pub const Param = struct {2910 pub const Param = struct {
2907 /// Null-terminated string index.2911 /// Null-terminated string index.
2908 name: u32,2912 name: u32,
2909 /// 0 if no doc comment 2913 /// 0 if no doc comment
2910 doc_comment: u32,2914 doc_comment: u32,
2911 /// The body contains the type of the parameter.2915 /// The body contains the type of the parameter.
2912 body_len: u32,2916 body_len: u32,