authorgravatar for kappaloris@gmail.comLoris Cro <kappaloris@gmail.com> 2022-01-20 16:08:50+01:00
committergravatar for kappaloris@gmail.comLoris Cro <kappaloris@gmail.com> 2022-01-21 22:00:56+01:00
log3010ccfca5f65a8ee0d70559a38d39c2c56f7367
tree1d268df600e14cb37fdd2b5f601bf33390b3ef49
parentc9ae24503dc8da2e59f46619695bf4eb863fb3ac

astgen saves decl doc comments in zir

The field is saved in `extra` unconditionally for each decl.

4 files changed, 77 insertions(+), 5 deletions(-)

src/AstGen.zig+62-3
......@@ -3080,9 +3080,9 @@ const WipMembers = struct {
30803080 /// struct, union, enum, and opaque decls all use same 4 bits per decl
30813081 const bits_per_decl = 4;
30823082 const decls_per_u32 = 32 / bits_per_decl;
3083 /// struct, union, enum, and opaque decls all have maximum size of 10 u32 slots
3084 /// (4 for src_hash + line + name + value + align + link_section + address_space)
3085 const max_decl_size = 10;
3083 /// struct, union, enum, and opaque decls all have maximum size of 11 u32 slots
3084 /// (4 for src_hash + line + name + value + doc_comment + align + link_section + address_space )
3085 const max_decl_size = 11;
30863086
30873087 pub fn init(gpa: Allocator, payload: *ArrayListUnmanaged(u32), decl_count: u32, field_count: u32, comptime bits_per_field: u32, comptime max_field_size: u32) Allocator.Error!Self {
30883088 const payload_top = @intCast(u32, payload.items.len);
......@@ -3193,6 +3193,7 @@ fn fnDecl(
31933193 // missing function name already happened in scanDecls()
31943194 const fn_name_token = fn_proto.name_token orelse return error.AnalysisFail;
31953195 const fn_name_str_index = try astgen.identAsString(fn_name_token);
3196 const doc_comment_index = try docCommentAsString(astgen, fn_name_token - 1);
31963197
31973198 // We insert this at the beginning so that its instruction index marks the
31983199 // start of the top level declaration.
......@@ -3445,6 +3446,7 @@ fn fnDecl(
34453446 }
34463447 wip_members.appendToDecl(fn_name_str_index);
34473448 wip_members.appendToDecl(block_inst);
3449 wip_members.appendToDecl(doc_comment_index);
34483450 if (align_inst != .none) {
34493451 wip_members.appendToDecl(@enumToInt(align_inst));
34503452 }
......@@ -3472,6 +3474,7 @@ fn globalVarDecl(
34723474
34733475 const name_token = var_decl.ast.mut_token + 1;
34743476 const name_str_index = try astgen.identAsString(name_token);
3477 const doc_comment_index = try docCommentAsString(astgen, var_decl.ast.mut_token);
34753478
34763479 var block_scope: GenZir = .{
34773480 .parent = scope,
......@@ -3594,6 +3597,7 @@ fn globalVarDecl(
35943597 }
35953598 wip_members.appendToDecl(name_str_index);
35963599 wip_members.appendToDecl(block_inst);
3600 wip_members.appendToDecl(doc_comment_index); // doc_comment wip
35973601 if (align_inst != .none) {
35983602 wip_members.appendToDecl(@enumToInt(align_inst));
35993603 }
......@@ -3648,6 +3652,7 @@ fn comptimeDecl(
36483652 }
36493653 wip_members.appendToDecl(0);
36503654 wip_members.appendToDecl(block_inst);
3655 wip_members.appendToDecl(0); // no doc comments on comptime decls
36513656}
36523657
36533658fn usingnamespaceDecl(
......@@ -3699,6 +3704,7 @@ fn usingnamespaceDecl(
36993704 }
37003705 wip_members.appendToDecl(0);
37013706 wip_members.appendToDecl(block_inst);
3707 wip_members.appendToDecl(0); // no doc comments on usingnamespace decls
37023708}
37033709
37043710fn testDecl(
......@@ -3802,6 +3808,7 @@ fn testDecl(
38023808 }
38033809 wip_members.appendToDecl(test_name);
38043810 wip_members.appendToDecl(block_inst);
3811 wip_members.appendToDecl(0); // no doc comments on test decls
38053812}
38063813
38073814fn structDeclInner(
......@@ -8784,6 +8791,58 @@ fn identAsString(astgen: *AstGen, ident_token: Ast.TokenIndex) !u32 {
87848791 }
87858792}
87868793
8794/// Adds a doc comment block to `string_bytes` by walking backwards from `end_token`.
8795/// `end_token` must point at the first token after the last doc coment line.
8796/// Returns 0 if no doc comment is present.
8797fn docCommentAsString(astgen: *AstGen, end_token: Ast.TokenIndex) !u32 {
8798 const gpa = astgen.gpa;
8799 const string_bytes = &astgen.string_bytes;
8800 const str_index = @intCast(u32, string_bytes.items.len);
8801 const token_tags = astgen.tree.tokens.items(.tag);
8802 const token_starts = astgen.tree.tokens.items(.start);
8803
8804 if (end_token == 0) return 0;
8805 const start_token: u32 = blk: {
8806 var tok = end_token - 1;
8807 while (token_tags[tok] == .doc_comment) {
8808 if (tok == 0) break;
8809 tok -= 1;
8810 } else {
8811 tok += 1;
8812 }
8813 break :blk tok;
8814 };
8815 if (start_token == end_token) return 0;
8816
8817 const total_bytes = token_starts[end_token] - token_starts[start_token];
8818 try string_bytes.ensureUnusedCapacity(gpa, total_bytes);
8819
8820 var current_token = start_token;
8821 while (current_token < end_token) : (current_token += 1) {
8822 const tok_bytes = astgen.tree.tokenSlice(current_token)[3..];
8823 string_bytes.appendSliceAssumeCapacity(tok_bytes);
8824 if (current_token != end_token - 1) {
8825 string_bytes.appendAssumeCapacity('\n');
8826 }
8827 }
8828
8829 const key = string_bytes.items[str_index..];
8830 const gop = try astgen.string_table.getOrPutContextAdapted(gpa, @as([]const u8, key), StringIndexAdapter{
8831 .bytes = string_bytes,
8832 }, StringIndexContext{
8833 .bytes = string_bytes,
8834 });
8835
8836 if (gop.found_existing) {
8837 string_bytes.shrinkRetainingCapacity(str_index);
8838 return gop.key_ptr.*;
8839 } else {
8840 gop.key_ptr.* = str_index;
8841 try string_bytes.append(gpa, 0);
8842 return str_index;
8843 }
8844}
8845
87878846const IndexSlice = struct { index: u32, len: u32 };
87888847
87898848fn strLitAsString(astgen: *AstGen, str_lit_token: Ast.TokenIndex) !IndexSlice {
src/Module.zig+1-1
......@@ -4011,7 +4011,7 @@ pub fn scanNamespace(
40114011 cur_bit_bag >>= 4;
40124012
40134013 const decl_sub_index = extra_index;
4014 extra_index += 7; // src_hash(4) + line(1) + name(1) + value(1)
4014 extra_index += 8; // src_hash(4) + line(1) + name(1) + value(1) + doc_comment(1)
40154015 extra_index += @truncate(u1, flags >> 2); // Align
40164016 extra_index += @as(u2, @truncate(u1, flags >> 3)) * 2; // Link section or address space, consists of 2 Refs
40174017
src/Zir.zig+1-1
......@@ -3001,7 +3001,7 @@ pub const DeclIterator = struct {
30013001 const sub_index = @intCast(u32, it.extra_index);
30023002 it.extra_index += 5; // src_hash(4) + line(1)
30033003 const name = it.zir.nullTerminatedString(it.zir.extra[it.extra_index]);
3004 it.extra_index += 2; // name(1) + value(1)
3004 it.extra_index += 3; // name(1) + value(1) + doc_comment(1)
30053005 it.extra_index += @truncate(u1, flags >> 2);
30063006 it.extra_index += @truncate(u1, flags >> 3);
30073007
src/print_zir.zig+13
......@@ -1398,6 +1398,9 @@ const Writer = struct {
13981398 extra_index += 1;
13991399 const decl_index = self.code.extra[extra_index];
14001400 extra_index += 1;
1401 const doc_comment_index = self.code.extra[extra_index];
1402 extra_index += 1;
1403
14011404 const align_inst: Zir.Inst.Ref = if (!has_align) .none else inst: {
14021405 const inst = @intToEnum(Zir.Inst.Ref, self.code.extra[extra_index]);
14031406 extra_index += 1;
......@@ -1431,6 +1434,16 @@ const Writer = struct {
14311434 raw_decl_name;
14321435 const test_str = if (raw_decl_name.len == 0) "test " else "";
14331436 const export_str = if (is_exported) "export " else "";
1437
1438 if (doc_comment_index != 0) {
1439 const doc_comment = self.code.nullTerminatedString(doc_comment_index);
1440 var it = std.mem.tokenize(u8, doc_comment, "\n");
1441 while (it.next()) |doc_line| {
1442 try stream.print("///{s}\n", .{doc_line});
1443 try stream.writeByteNTimes(' ', self.indent);
1444 }
1445 }
1446
14341447 try stream.print("[{d}] {s}{s}{s}{}", .{
14351448 sub_index, pub_str, test_str, export_str, std.zig.fmtId(decl_name),
14361449 });