authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-08-02 19:21:01-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-08-02 19:25:26-07:00
log1a3f250f195d4ed5455795d4fa6e4b0cf97cb6ce
treec478f030e5c8d7d1829a90b545a8306a9ff9f47c
parentba6e5cbfd279583dcc724f6a74a5593fa26ad5df

.debug_line incremental compilation initial support

Supports writing the first function. Still TODO is: * handling the .debug_line header growing too large * adding a new file to an existing compilation * adding an additional function to an existing file * handling incremental updates * adding the main IR debug ops for IR instructions There are also issues to work out: * readelf --debug-dump=rawline is saying there is no .debug_str section even though there is * readelf --debug-dump=decodedline is saying the file index 0 is bad and reporting some other kind of corruption.

2 files changed, 223 insertions(+), 72 deletions(-)

lib/std/hash_map.zig+13-5
......@@ -196,6 +196,10 @@ pub fn HashMap(
196196 return self.unmanaged.getEntry(key);
197197 }
198198
199 pub fn getIndex(self: Self, key: K) ?usize {
200 return self.unmanaged.getIndex(key);
201 }
202
199203 pub fn get(self: Self, key: K) ?V {
200204 return self.unmanaged.get(key);
201205 }
......@@ -479,17 +483,21 @@ pub fn HashMapUnmanaged(
479483 }
480484
481485 pub fn getEntry(self: Self, key: K) ?*Entry {
486 const index = self.getIndex(key) orelse return null;
487 return &self.entries.items[index];
488 }
489
490 pub fn getIndex(self: Self, key: K) ?usize {
482491 const header = self.index_header orelse {
483492 // Linear scan.
484493 const h = if (store_hash) hash(key) else {};
485 for (self.entries.items) |*item| {
494 for (self.entries.items) |*item, i| {
486495 if (item.hash == h and eql(key, item.key)) {
487 return item;
496 return i;
488497 }
489498 }
490499 return null;
491500 };
492
493501 switch (header.capacityIndexType()) {
494502 .u8 => return self.getInternal(key, header, u8),
495503 .u16 => return self.getInternal(key, header, u16),
......@@ -711,7 +719,7 @@ pub fn HashMapUnmanaged(
711719 unreachable;
712720 }
713721
714 fn getInternal(self: Self, key: K, header: *IndexHeader, comptime I: type) ?*Entry {
722 fn getInternal(self: Self, key: K, header: *IndexHeader, comptime I: type) ?usize {
715723 const indexes = header.indexes(I);
716724 const h = hash(key);
717725 const start_index = header.constrainIndex(h);
......@@ -725,7 +733,7 @@ pub fn HashMapUnmanaged(
725733 const entry = &self.entries.items[index.entry_index];
726734 const hash_match = if (store_hash) h == entry.hash else true;
727735 if (hash_match and eql(key, entry.key))
728 return entry;
736 return index.entry_index;
729737 }
730738 return null;
731739 }
src-self-hosted/link.zig+210-67
......@@ -412,14 +412,14 @@ pub const File = struct {
412412
413413 pub const SrcFn = struct {
414414 /// Offset from the `SrcFile` that contains this function.
415 dbg_line_off: u32,
415 off: u32,
416416 /// Size of the line number program component belonging to this function, not
417417 /// including padding.
418 dbg_line_len: u32,
418 len: u32,
419419
420420 pub const empty: SrcFn = .{
421 .dbg_line_off = 0,
422 .dbg_line_len = 0,
421 .off = 0,
422 .len = 0,
423423 };
424424 };
425425
......@@ -430,10 +430,17 @@ pub const File = struct {
430430 /// Line Number Program that contains it.
431431 len: u32,
432432
433 /// A list of `SrcFn` that have surplus capacity.
434 /// This is the same concept as `text_block_free_list` (see the doc comments there)
435 /// but it's for the function's component of the Line Number program.
436 free_list: std.ArrayListUnmanaged(*SrcFn),
433 /// An ordered list of all the `SrcFn` in this file. This list is not redundant with
434 /// the source Decl list, for two reasons:
435 /// * Lazy decl analysis: some source functions do not correspond to any compiled functions.
436 /// * Generic functions: some source functions correspond to many compiled functions.
437 /// This list corresponds to the file data in the Line Number Program. When a new `SrcFn`
438 /// is inserted, the list must be shifted to accomodate it, and likewise the Line
439 /// Number Program data must be shifted within the ELF file to accomodate (if there is
440 /// not enough padding).
441 /// It is a hash map so that we can look up the index based on the `*SrcFn` and therefore
442 /// find the next and previous functions.
443 fns: std.AutoHashMapUnmanaged(*SrcFn, void),
437444
438445 /// Points to the previous and next neighbors, based on the offset from .debug_line.
439446 /// This can be used to find, for example, the capacity of this `SrcFile`.
......@@ -443,7 +450,7 @@ pub const File = struct {
443450 pub const empty: SrcFile = .{
444451 .off = 0,
445452 .len = 0,
446 .free_list = .{},
453 .fns = .{},
447454 .prev = null,
448455 .next = null,
449456 };
......@@ -589,7 +596,7 @@ pub const File = struct {
589596 return self.first_dbg_line_file.?.off;
590597 }
591598
592 fn getDebugLineProgramLen(self: Elf) u32 {
599 fn getDebugLineProgramEnd(self: Elf) u32 {
593600 return self.last_dbg_line_file.?.off + self.last_dbg_line_file.?.len;
594601 }
595602
......@@ -767,27 +774,6 @@ pub const File = struct {
767774 self.shstrtab_dirty = true;
768775 self.shdr_table_dirty = true;
769776 }
770 if (self.debug_str_section_index == null) {
771 self.debug_str_section_index = @intCast(u16, self.sections.items.len);
772 assert(self.debug_strtab.items.len == 0);
773 try self.debug_strtab.append(self.allocator, 0); // need a 0 at position 0
774 const off = self.findFreeSpace(self.debug_strtab.items.len, 1);
775 log.debug(.link, "found debug_strtab free space 0x{x} to 0x{x}\n", .{ off, off + self.debug_strtab.items.len });
776 try self.sections.append(self.allocator, .{
777 .sh_name = try self.makeString(".debug_str"),
778 .sh_type = elf.SHT_PROGBITS,
779 .sh_flags = elf.SHF_MERGE | elf.SHF_STRINGS,
780 .sh_addr = 0,
781 .sh_offset = off,
782 .sh_size = self.debug_strtab.items.len,
783 .sh_link = 0,
784 .sh_info = 0,
785 .sh_addralign = 1,
786 .sh_entsize = 1,
787 });
788 self.debug_strtab_dirty = true;
789 self.shdr_table_dirty = true;
790 }
791777 if (self.text_section_index == null) {
792778 self.text_section_index = @intCast(u16, self.sections.items.len);
793779 const phdr = &self.program_headers.items[self.phdr_load_re_index.?];
......@@ -848,6 +834,24 @@ pub const File = struct {
848834 self.shdr_table_dirty = true;
849835 try self.writeSymbol(0);
850836 }
837 if (self.debug_str_section_index == null) {
838 self.debug_str_section_index = @intCast(u16, self.sections.items.len);
839 assert(self.debug_strtab.items.len == 0);
840 try self.sections.append(self.allocator, .{
841 .sh_name = try self.makeString(".debug_str"),
842 .sh_type = elf.SHT_PROGBITS,
843 .sh_flags = elf.SHF_MERGE | elf.SHF_STRINGS,
844 .sh_addr = 0,
845 .sh_offset = 0,
846 .sh_size = self.debug_strtab.items.len,
847 .sh_link = 0,
848 .sh_info = 0,
849 .sh_addralign = 1,
850 .sh_entsize = 1,
851 });
852 self.debug_strtab_dirty = true;
853 self.shdr_table_dirty = true;
854 }
851855 if (self.debug_info_section_index == null) {
852856 self.debug_info_section_index = @intCast(u16, self.sections.items.len);
853857
......@@ -1002,7 +1006,7 @@ pub const File = struct {
10021006 // we can simply append these bytes.
10031007 const abbrev_buf = [_]u8{
10041008 1, DW.TAG_compile_unit, DW.CHILDREN_no, // header
1005 DW.AT_stmt_list, DW.FORM_data1,
1009 DW.AT_stmt_list, DW.FORM_sec_offset,
10061010 DW.AT_low_pc , DW.FORM_addr,
10071011 DW.AT_high_pc , DW.FORM_addr,
10081012 DW.AT_name , DW.FORM_strp,
......@@ -1074,10 +1078,8 @@ pub const File = struct {
10741078 const low_pc = text_phdr.p_vaddr;
10751079 const high_pc = text_phdr.p_vaddr + text_phdr.p_memsz;
10761080
1077 di_buf.appendSliceAssumeCapacity(&[_]u8{
1078 1, // abbrev tag, matching the value from the abbrev table header
1079 0, // DW.AT_stmt_list, DW.FORM_data1: offset to corresponding .debug_line header
1080 });
1081 di_buf.appendAssumeCapacity(1); // abbrev tag, matching the value from the abbrev table header
1082 self.writeDwarfAddrAssumeCapacity(&di_buf, 0); // DW.AT_stmt_list, DW.FORM_sec_offset
10811083 self.writeDwarfAddrAssumeCapacity(&di_buf, low_pc);
10821084 self.writeDwarfAddrAssumeCapacity(&di_buf, high_pc);
10831085 self.writeDwarfAddrAssumeCapacity(&di_buf, name_strp);
......@@ -1194,22 +1196,23 @@ pub const File = struct {
11941196 }
11951197 if (self.debug_line_header_dirty) {
11961198 const dbg_line_prg_off = self.getDebugLineProgramOff();
1197 const dbg_line_prg_len = self.getDebugLineProgramLen();
1198 assert(dbg_line_prg_len != 0);
1199 const dbg_line_prg_end = self.getDebugLineProgramEnd();
1200 assert(dbg_line_prg_end != 0);
11991201
12001202 const debug_line_sect = &self.sections.items[self.debug_line_section_index.?];
12011203
12021204 var di_buf = std.ArrayList(u8).init(self.allocator);
12031205 defer di_buf.deinit();
12041206
1205 // This is a heuristic. The size of this header is variable, depending on
1206 // the number of directories, files, and padding.
1207 try di_buf.ensureCapacity(100);
1207 // The size of this header is variable, depending on the number of directories,
1208 // files, and padding. We have a function to compute the upper bound size, however,
1209 // because it's needed for determining where to put the offset of the first `SrcFile`.
1210 try di_buf.ensureCapacity(self.dbgLineNeededHeaderBytes());
12081211
12091212 // initial length - length of the .debug_line contribution for this compilation unit,
12101213 // not including the initial length itself.
12111214 const after_init_len = di_buf.items.len + init_len_size;
1212 const init_len = (dbg_line_prg_off + dbg_line_prg_len) - after_init_len;
1215 const init_len = dbg_line_prg_end - after_init_len;
12131216 switch (self.ptr_width) {
12141217 .p32 => {
12151218 mem.writeInt(u32, di_buf.addManyAsArrayAssumeCapacity(4), @intCast(u32, init_len), target_endian);
......@@ -1277,10 +1280,16 @@ pub const File = struct {
12771280 self.writeDwarfAddrAssumeCapacity(&di_buf, root_src_file_strp); // DW.LNCT_path, DW.FORM_strp
12781281 di_buf.appendAssumeCapacity(0); // LNCT_directory_index, FORM_data1
12791282
1280 if (di_buf.items.len > dbg_line_prg_off) {
1283 // Add a redundant NOP in case the consumer ignores header_length.
1284 const after_jmp = di_buf.items.len + 6;
1285 if (after_jmp > dbg_line_prg_off) {
12811286 // Move the first N files to the end to make more padding for the header.
12821287 @panic("TODO: handle .debug_line header exceeding its padding");
12831288 }
1289 const jmp_amt = dbg_line_prg_off - after_jmp + 1;
1290 di_buf.appendAssumeCapacity(DW.LNS_extended_op);
1291 leb128.writeUnsignedFixed(4, di_buf.addManyAsArrayAssumeCapacity(4), @intCast(u28, jmp_amt));
1292 di_buf.appendAssumeCapacity(DW.LNE_hi_user);
12841293
12851294 try self.file.?.pwriteAll(di_buf.items, debug_line_sect.sh_offset);
12861295 self.debug_line_header_dirty = false;
......@@ -1813,37 +1822,27 @@ pub const File = struct {
18131822 .Fn => true,
18141823 else => false,
18151824 };
1816 const dbg_line_vaddr_reloc_index = 1;
18171825 if (is_fn) {
1818 const scope_file = decl.scope.cast(Module.Scope.File).?;
1819 const line_off: u28 = blk: {
1820 const file_ast_decls = scope_file.contents.tree.root_node.decls();
1821 if (decl.src_index == 0) {
1822 // Then it's the line number of the open curly.
1823 const block = file_ast_decls[decl.src_index].castTag(.Block).?;
1824 @panic("TODO implement this");
1825 } else {
1826 const prev_decl = file_ast_decls[decl.src_index - 1];
1827 // Find the difference between prev decl end curly and this decl begin curly.
1828 @panic("TODO implement this");
1829 }
1830 };
1831
18321826 // For functions we need to add a prologue to the debug line program.
1833 try dbg_line_buffer.ensureCapacity(24);
1827 try dbg_line_buffer.ensureCapacity(26);
18341828
1835 dbg_line_buffer.appendAssumeCapacity(DW.LNE_set_address);
1829 const ptr_width_bytes = self.ptrWidthBytes();
1830 dbg_line_buffer.appendSliceAssumeCapacity(&[_]u8{
1831 DW.LNS_extended_op,
1832 ptr_width_bytes + 1,
1833 DW.LNE_set_address,
1834 });
18361835 // This is the "relocatable" vaddr, corresponding to `code_buffer` index `0`.
18371836 assert(dbg_line_vaddr_reloc_index == dbg_line_buffer.items.len);
1838 dbg_line_buffer.items.len += self.ptrWidthBytes();
1837 dbg_line_buffer.items.len += ptr_width_bytes;
18391838
18401839 dbg_line_buffer.appendAssumeCapacity(DW.LNS_advance_line);
18411840 // This is the "relocatable" relative line offset from the previous function's end curly
18421841 // to this function's begin curly.
18431842 assert(self.getRelocDbgLineOff() == dbg_line_buffer.items.len);
1844 // Here we use a ULEB128 but we write 4 bytes regardless (possibly wasting space)
1845 // so that we can patch this later as a fixed width field.
1846 leb128.writeUnsignedFixed(4, dbg_line_buffer.addManyAsArrayAssumeCapacity(4), line_off);
1843 // Here we allocate 4 bytes for the relocation. This field is a ULEB128, however,
1844 // it is possible to encode small values as still taking up 4 bytes.
1845 dbg_line_buffer.items.len += 4;
18471846
18481847 // Emit a line for the begin curly with prologue_end=false. The codegen will
18491848 // do the work of setting prologue_end=true and epilogue_begin=true.
......@@ -1917,6 +1916,14 @@ pub const File = struct {
19171916
19181917 // If the Decl is a function, we need to update the .debug_line program.
19191918 if (is_fn) {
1919 // For padding between functions, we terminate with `LNS_extended_op` with sub-op
1920 // `LNE_hi_user`, using a fixed 4-byte ULEB128 for the opcode size. This is always
1921 // found at the very end of the SrcFile's Line Number Program component.
1922 try dbg_line_buffer.ensureCapacity(dbg_line_buffer.items.len + 6);
1923 dbg_line_buffer.appendAssumeCapacity(DW.LNS_extended_op);
1924 leb128.writeUnsignedFixed(4, dbg_line_buffer.addManyAsArrayAssumeCapacity(4), 1);
1925 dbg_line_buffer.appendAssumeCapacity(DW.LNE_hi_user);
1926
19201927 // Perform the relocation based on vaddr.
19211928 const target_endian = self.base.options.target.cpu.arch.endian();
19221929 switch (self.ptr_width) {
......@@ -1930,13 +1937,91 @@ pub const File = struct {
19301937 },
19311938 }
19321939
1933 const src_file = &decl.scope.cast(Module.Scope.File).?.link;
1940 // Now we want to write the line offset relocation, however, first we must
1941 // "plug in" the SrcFn into its parent SrcFile, so that we know what function the line
1942 // number is offset from. It must go in the same order as the functions are found
1943 // in the Zig source. When we insert a function before another one, the latter one
1944 // must have its line offset relocation updated.
1945
1946 const debug_line_sect = &self.sections.items[self.debug_line_section_index.?];
1947 const scope_file = decl.scope.cast(Module.Scope.File).?;
1948 const src_file = &scope_file.link;
19341949 const src_fn = &typed_value.val.cast(Value.Payload.Function).?.func.link;
1935 if (src_file.next == null and src_file.prev == null) {
1936 @panic("TODO updateDecl for .debug_line: add new SrcFile");
1950 var src_fn_index: usize = undefined;
1951 if (src_file.len == 0) {
1952 // This is the first function of the SrcFile.
1953 assert(src_file.fns.entries.items.len == 0);
1954 src_fn_index = 0;
1955 try src_file.fns.put(self.allocator, src_fn, {});
1956
1957 if (self.last_dbg_line_file) |last| {
1958 src_file.prev = last;
1959 self.last_dbg_line_file = src_file;
1960
1961 // Update the previous last SrcFile's terminating NOP to skip to the start
1962 // of the new last SrcFile's start.
1963 @panic("TODO updateDecl for .debug_line: add new SrcFile: append");
1964 } else {
1965 // This is the first file (and function) of the Line Number Program.
1966 self.first_dbg_line_file = src_file;
1967 self.last_dbg_line_file = src_file;
1968
1969 src_fn.off = dbg_line_file_header_len;
1970 src_fn.len = @intCast(u32, dbg_line_buffer.items.len);
1971
1972 src_file.off = self.dbgLineNeededHeaderBytes() * alloc_num / alloc_den;
1973 src_file.len = src_fn.off + src_fn.len + dbg_line_file_trailer_len;
1974
1975 const needed_size = src_file.off + src_file.len;
1976 if (needed_size > debug_line_sect.sh_size) {
1977 debug_line_sect.sh_offset = self.findFreeSpace(needed_size, 1);
1978 }
1979 debug_line_sect.sh_size = needed_size;
1980 self.shdr_table_dirty = true; // TODO look into making only the one section dirty
1981 self.debug_line_header_dirty = true;
1982
1983 try self.updateDbgLineFile(src_file);
1984 }
19371985 } else {
19381986 @panic("TODO updateDecl for .debug_line: update existing SrcFile");
1987 //src_fn_index = @panic("TODO");
19391988 }
1989 const line_off: u28 = blk: {
1990 const tree = scope_file.contents.tree;
1991 const file_ast_decls = tree.root_node.decls();
1992 // TODO Look into improving the performance here by adding a token-index-to-line
1993 // lookup table. Currently this involves scanning over the source code for newlines
1994 // (but only from the previous decl to the current one).
1995 if (src_fn_index == 0) {
1996 // Since it's the first function in the file, the line number delta is just the
1997 // line number of the open curly from the beginning of the file.
1998 const fn_proto = file_ast_decls[decl.src_index].castTag(.FnProto).?;
1999 const block = fn_proto.body().?.castTag(.Block).?;
2000 const loc = tree.tokenLocation(0, block.lbrace);
2001 // No need to add one; this is a delta from DWARF's starting line number (1).
2002 break :blk @intCast(u28, loc.line);
2003 } else {
2004 const prev_src_fn = src_file.fns.entries.items[src_fn_index - 1].key;
2005 const mod_fn = @fieldParentPtr(Module.Fn, "link", prev_src_fn);
2006 const prev_fn_proto = file_ast_decls[mod_fn.owner_decl.src_index].castTag(.FnProto).?;
2007 const this_fn_proto = file_ast_decls[decl.src_index].castTag(.FnProto).?;
2008 const prev_block = prev_fn_proto.body().?.castTag(.Block).?;
2009 const this_block = this_fn_proto.body().?.castTag(.Block).?;
2010 // Find the difference between prev decl end curly and this decl begin curly.
2011 const loc = tree.tokenLocation(tree.token_locs[prev_block.rbrace].start, this_block.lbrace);
2012 // No need to add one; this is a delta from the previous line number.
2013 break :blk @intCast(u28, loc.line);
2014 }
2015 };
2016
2017 // Here we use a ULEB128 but we write 4 bytes regardless (possibly wasting space) because
2018 // that is the amount of space we allocated for this field.
2019 leb128.writeUnsignedFixed(4, dbg_line_buffer.items[self.getRelocDbgLineOff()..][0..4], line_off);
2020
2021 // We only have support for one compilation unit so far, so the offsets are directly
2022 // from the .debug_line section.
2023 const file_pos = debug_line_sect.sh_offset + src_file.off + src_fn.off;
2024 try self.file.?.pwriteAll(dbg_line_buffer.items, file_pos);
19402025 }
19412026
19422027 // Since we updated the vaddr and the size, each corresponding export symbol also needs to be updated.
......@@ -2028,6 +2113,46 @@ pub const File = struct {
20282113 self.global_symbols.items[sym_index].st_info = 0;
20292114 }
20302115
2116 const dbg_line_file_header_len = 5; // DW.LNS_set_file + ULEB128-fixed-4 file_index
2117 const dbg_line_file_trailer_len = 9; // DW.LNE_end_sequence + 6-byte terminating NOP
2118
2119 fn updateDbgLineFile(self: *Elf, src_file: *SrcFile) !void {
2120 const target_endian = self.base.options.target.cpu.arch.endian();
2121 const shdr = &self.sections.items[self.debug_line_section_index.?];
2122 const header_off = shdr.sh_offset + src_file.off;
2123 {
2124 var header: [dbg_line_file_header_len]u8 = undefined;
2125 header[0] = DW.LNS_set_file;
2126 // Once we support more than one source file, this will have the ability to be non-zero.
2127 const file_index = 0;
2128 leb128.writeUnsignedFixed(4, header[1..5], file_index);
2129 try self.file.?.pwriteAll(&header, header_off);
2130 }
2131 {
2132 const last_src_fn = src_file.fns.entries.items[src_file.fns.entries.items.len - 1].key;
2133 const trailer_off = header_off + last_src_fn.off + last_src_fn.len;
2134 const padding_to_next = blk: {
2135 if (src_file.next) |next| {
2136 break :blk next.off - (src_file.off + src_file.len);
2137 } else {
2138 // No need for padding after this one; we will add padding to it when a SrcFile
2139 // is added after it.
2140 break :blk 0;
2141 }
2142 };
2143 var trailer: [dbg_line_file_trailer_len]u8 = undefined;
2144
2145 trailer[0] = DW.LNS_extended_op;
2146 trailer[1] = 1;
2147 trailer[2] = DW.LNE_end_sequence;
2148
2149 trailer[3] = DW.LNS_extended_op;
2150 leb128.writeUnsignedFixed(4, trailer[4..8], @intCast(u28, padding_to_next + 1));
2151 trailer[8] = DW.LNE_hi_user;
2152 try self.file.?.pwriteAll(&trailer, trailer_off);
2153 }
2154 }
2155
20312156 fn writeProgHeader(self: *Elf, index: usize) !void {
20322157 const foreign_endian = self.base.options.target.cpu.arch.endian() != std.Target.current.cpu.arch.endian();
20332158 const offset = self.program_headers.items[index].p_offset;
......@@ -2227,6 +2352,24 @@ pub const File = struct {
22272352 };
22282353 }
22292354
2355 /// The reloc offset for the virtual address of a function in its Line Number Program.
2356 /// Size is a virtual address integer.
2357 const dbg_line_vaddr_reloc_index = 3;
2358
2359 /// The reloc offset for the line offset of a function from the previous function's line.
2360 /// It's a fixed-size 4-byte ULEB128.
2361 fn getRelocDbgLineOff(self: Elf) usize {
2362 return dbg_line_vaddr_reloc_index + self.ptrWidthBytes() + 1;
2363 }
2364
2365 fn dbgLineNeededHeaderBytes(self: Elf) u32 {
2366 const directory_entry_format_count = 1;
2367 const file_name_entry_format_count = 1;
2368 const directory_count = 1;
2369 const file_name_count = 1;
2370 return 53 + directory_entry_format_count * 2 + file_name_entry_format_count * 2 +
2371 directory_count * 8 + file_name_count * 8;
2372 }
22302373 };
22312374};
22322375