authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-06-19 13:15:40-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-06-19 13:15:40-07:00
log7d6fcf083122bfe049f5314b3ec8ddcbfb27569d
treefe6ea4c053b07f993a3d9f8a2d65db7690cef582
parent8fcc28d30275bf76313765616a2d2bdcaeb4faf2
parentc39191a0869cb6ca614c51fd6e3930908dcfd8df
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #15012 from xxxbxxx/objcpy-elf-compress

objcopy: add support for --compress-debug-sections

2 files changed, 98 insertions(+), 33 deletions(-)

lib/std/elf.zig+14-3
......@@ -706,13 +706,13 @@ pub const Elf64_Shdr = extern struct {
706706 sh_entsize: Elf64_Xword,
707707};
708708pub const Elf32_Chdr = extern struct {
709 ch_type: Elf32_Word,
709 ch_type: COMPRESS,
710710 ch_size: Elf32_Word,
711711 ch_addralign: Elf32_Word,
712712};
713713pub const Elf64_Chdr = extern struct {
714 ch_type: Elf64_Word,
715 ch_reserved: Elf64_Word,
714 ch_type: COMPRESS,
715 ch_reserved: Elf64_Word = 0,
716716 ch_size: Elf64_Xword,
717717 ch_addralign: Elf64_Xword,
718718};
......@@ -1730,6 +1730,17 @@ pub const SHN_COMMON = 0xfff2;
17301730/// End of reserved indices
17311731pub const SHN_HIRESERVE = 0xffff;
17321732
1733// Legal values for ch_type (compression algorithm).
1734pub const COMPRESS = enum(u32) {
1735 ZLIB = 1,
1736 ZSTD = 2,
1737 LOOS = 0x60000000,
1738 HIOS = 0x6fffffff,
1739 LOPROC = 0x70000000,
1740 HIPROC = 0x7fffffff,
1741 _,
1742};
1743
17331744/// AMD x86-64 relocations.
17341745/// No reloc
17351746pub const R_X86_64_NONE = 0;
src/objcopy.zig+84-30
......@@ -27,6 +27,7 @@ pub fn cmdObjCopy(
2727 var strip_all: bool = false;
2828 var strip_debug: bool = false;
2929 var only_keep_debug: bool = false;
30 var compress_debug_sections: bool = false;
3031 var listen = false;
3132 while (i < args.len) : (i += 1) {
3233 const arg = args[i];
......@@ -78,6 +79,8 @@ pub fn cmdObjCopy(
7879 strip_all = true;
7980 } else if (mem.eql(u8, arg, "--only-keep-debug")) {
8081 only_keep_debug = true;
82 } else if (mem.eql(u8, arg, "--compress-debug-sections")) {
83 compress_debug_sections = true;
8184 } else if (mem.startsWith(u8, arg, "--add-gnu-debuglink=")) {
8285 opt_add_debuglink = arg["--add-gnu-debuglink=".len..];
8386 } else if (mem.eql(u8, arg, "--add-gnu-debuglink")) {
......@@ -152,6 +155,7 @@ pub fn cmdObjCopy(
152155 .only_keep_debug = only_keep_debug,
153156 .add_debuglink = opt_add_debuglink,
154157 .extract_to = opt_extract,
158 .compress_debug = compress_debug_sections,
155159 });
156160 return std.process.cleanExit();
157161 },
......@@ -210,6 +214,7 @@ const usage =
210214 \\ --only-keep-debug Strip a file, removing contents of any sections that would not be stripped by --strip-debug and leaving the debugging sections intact.
211215 \\ --add-gnu-debuglink=<file> Creates a .gnu_debuglink section which contains a reference to <file> and adds it to the output file.
212216 \\ --extract-to <file> Extract the removed sections into <file>, and add a .gnu-debuglink section.
217 \\ --compress-debug-sections Compress DWARF debug sections with zlib
213218 \\
214219;
215220
......@@ -660,6 +665,7 @@ const StripElfOptions = struct {
660665 strip_all: bool = false,
661666 strip_debug: bool = false,
662667 only_keep_debug: bool = false,
668 compress_debug: bool = false,
663669};
664670
665671fn stripElf(
......@@ -711,11 +717,11 @@ fn stripElf(
711717 };
712718 defer dbg_file.close();
713719
714 try elf_file.emit(allocator, dbg_file, in_file, .{ .section_filter = flt });
720 try elf_file.emit(allocator, dbg_file, in_file, .{ .section_filter = flt, .compress_debug = options.compress_debug });
715721 }
716722
717723 const debuglink: ?DebugLink = if (debuglink_path) |path| ElfFileHelper.createDebugLink(path) else null;
718 try elf_file.emit(allocator, out_file, in_file, .{ .section_filter = filter, .debuglink = debuglink });
724 try elf_file.emit(allocator, out_file, in_file, .{ .section_filter = filter, .debuglink = debuglink, .compress_debug = options.compress_debug });
719725 },
720726 }
721727}
......@@ -730,6 +736,7 @@ fn ElfFile(comptime is_64: bool) type {
730736 const Elf_Ehdr = if (is_64) elf.Elf64_Ehdr else elf.Elf32_Ehdr;
731737 const Elf_Phdr = if (is_64) elf.Elf64_Phdr else elf.Elf32_Phdr;
732738 const Elf_Shdr = if (is_64) elf.Elf64_Shdr else elf.Elf32_Shdr;
739 const Elf_Chdr = if (is_64) elf.Elf64_Chdr else elf.Elf32_Chdr;
733740 const Elf_Sym = if (is_64) elf.Elf64_Sym else elf.Elf32_Sym;
734741 const Elf_Verdef = if (is_64) elf.Elf64_Verdef else elf.Elf32_Verdef;
735742 const Elf_OffSize = if (is_64) elf.Elf64_Off else elf.Elf32_Off;
......@@ -817,7 +824,7 @@ fn ElfFile(comptime is_64: bool) type {
817824 // fill-in sections info:
818825 // resolve the name
819826 // find if a program segment uses the section
820 // categorise sections usage (used by program segments, debug datadase, common metadata, symbol table)
827 // categorize sections usage (used by program segments, debug datadase, common metadata, symbol table)
821828 for (sections) |*section| {
822829 section.segment = for (program_segments) |*seg| {
823830 if (sectionWithinSegment(section.section, seg.*)) break seg;
......@@ -836,8 +843,8 @@ fn ElfFile(comptime is_64: bool) type {
836843 if (std.mem.eql(u8, section.name, ".gnu_debuglink")) break :cat .none;
837844 break :cat category_from_program;
838845 },
839 elf.SHT_LOPROC...elf.SHT_HIPROC => .common, // don't strip unkonwn sections
840 elf.SHT_LOUSER...elf.SHT_HIUSER => .common, // don't strip unkonwn sections
846 elf.SHT_LOPROC...elf.SHT_HIPROC => .common, // don't strip unknown sections
847 elf.SHT_LOUSER...elf.SHT_HIUSER => .common, // don't strip unknown sections
841848 else => category_from_program,
842849 };
843850 }
......@@ -846,7 +853,7 @@ fn ElfFile(comptime is_64: bool) type {
846853 if (header.shstrndx != elf.SHN_UNDEF)
847854 sections[header.shstrndx].category = .common; // string table for the headers
848855
849 // recursive dependencies
856 // recursively propagate section categories to their linked sections, so that they are kept together
850857 var dirty: u1 = 1;
851858 while (dirty != 0) {
852859 dirty = 0;
......@@ -856,29 +863,6 @@ fn ElfFile(comptime is_64: bool) type {
856863 dirty |= ElfFileHelper.propagateCategory(&sections[section.section.sh_link].category, section.category);
857864 if ((section.section.sh_flags & elf.SHF_INFO_LINK) != 0 and section.section.sh_info != elf.SHN_UNDEF)
858865 dirty |= ElfFileHelper.propagateCategory(&sections[section.section.sh_info].category, section.category);
859
860 if (section.payload) |data| {
861 switch (section.section.sh_type) {
862 elf.DT_VERSYM => {
863 assert(section.section.sh_entsize == @sizeOf(Elf_Verdef));
864 const defs = @ptrCast([*]const Elf_Verdef, data)[0 .. @intCast(usize, section.section.sh_size) / @sizeOf(Elf_Verdef)];
865 for (defs) |def| {
866 if (def.vd_ndx != elf.SHN_UNDEF)
867 dirty |= ElfFileHelper.propagateCategory(&sections[def.vd_ndx].category, section.category);
868 }
869 },
870 elf.SHT_SYMTAB, elf.SHT_DYNSYM => {
871 assert(section.section.sh_entsize == @sizeOf(Elf_Sym));
872 const syms = @ptrCast([*]const Elf_Sym, data)[0 .. @intCast(usize, section.section.sh_size) / @sizeOf(Elf_Sym)];
873
874 for (syms) |sym| {
875 if (sym.st_shndx != elf.SHN_UNDEF and sym.st_shndx < elf.SHN_LORESERVE)
876 dirty |= ElfFileHelper.propagateCategory(&sections[sym.st_shndx].category, section.category);
877 }
878 },
879 else => {},
880 }
881 }
882866 }
883867 }
884868
......@@ -899,6 +883,7 @@ fn ElfFile(comptime is_64: bool) type {
899883 const EmitElfOptions = struct {
900884 section_filter: Filter = .all,
901885 debuglink: ?DebugLink = null,
886 compress_debug: bool = false,
902887 };
903888 fn emit(self: *const Self, gpa: Allocator, out_file: File, in_file: File, options: EmitElfOptions) !void {
904889 var arena = std.heap.ArenaAllocator.init(gpa);
......@@ -962,6 +947,30 @@ fn ElfFile(comptime is_64: bool) type {
962947 break :blk new_offset;
963948 };
964949
950 // maybe compress .debug sections
951 if (options.compress_debug) {
952 for (self.sections[1..], sections_update[1..]) |section, *update| {
953 if (update.action != .keep) continue;
954 if (!std.mem.startsWith(u8, section.name, ".debug_")) continue;
955 if ((section.section.sh_flags & elf.SHF_COMPRESSED) != 0) continue; // already compressed
956
957 const chdr = Elf_Chdr{
958 .ch_type = elf.COMPRESS.ZLIB,
959 .ch_size = section.section.sh_size,
960 .ch_addralign = section.section.sh_addralign,
961 };
962
963 const compressed_payload = try ElfFileHelper.tryCompressSection(allocator, in_file, section.section.sh_offset, section.section.sh_size, std.mem.asBytes(&chdr));
964 if (compressed_payload) |payload| {
965 update.payload = payload;
966 update.section = section.section;
967 update.section.?.sh_addralign = @alignOf(Elf_Chdr);
968 update.section.?.sh_size = @intCast(Elf_OffSize, payload.len);
969 update.section.?.sh_flags |= elf.SHF_COMPRESSED;
970 }
971 }
972 }
973
965974 var cmdbuf = std.ArrayList(ElfFileHelper.WriteCmd).init(allocator);
966975 defer cmdbuf.deinit();
967976 try cmdbuf.ensureUnusedCapacity(3 + new_shnum);
......@@ -994,6 +1003,8 @@ fn ElfFile(comptime is_64: bool) type {
9941003 // this code only supports when they are in increasing file order.
9951004 var offset: u64 = eof_offset;
9961005 for (self.sections[1..]) |section| {
1006 if (section.section.sh_type == elf.SHT_NOBITS)
1007 continue;
9971008 if (section.section.sh_offset < offset) {
9981009 fatal("zig objcopy: unsuported ELF file", .{});
9991010 }
......@@ -1025,7 +1036,7 @@ fn ElfFile(comptime is_64: bool) type {
10251036
10261037 const addralign = if (src.sh_addralign == 0 or dest.sh_type == elf.SHT_NOBITS) 1 else src.sh_addralign;
10271038 dest.sh_offset = std.mem.alignForward(Elf_OffSize, eof_offset, addralign);
1028 if (src.sh_offset != dest.sh_offset and section.segment != null and update.action != .empty and dest.sh_type != elf.SHT_NOTE) {
1039 if (src.sh_offset != dest.sh_offset and section.segment != null and update.action != .empty and dest.sh_type != elf.SHT_NOTE and dest.sh_type != elf.SHT_NOBITS) {
10291040 if (src.sh_offset > dest.sh_offset) {
10301041 dest.sh_offset = src.sh_offset; // add padding to avoid modifing the program segments
10311042 } else {
......@@ -1268,6 +1279,49 @@ const ElfFileHelper = struct {
12681279 }
12691280 }
12701281
1282 fn tryCompressSection(allocator: Allocator, in_file: File, offset: u64, size: u64, prefix: []const u8) !?[]align(8) const u8 {
1283 if (size < prefix.len) return null;
1284
1285 try in_file.seekTo(offset);
1286 var section_reader = std.io.limitedReader(in_file.reader(), size);
1287
1288 // allocate as large as decompressed data. if the compression doesn't fit, keep the data uncompressed.
1289 const compressed_data = try allocator.alignedAlloc(u8, 8, @intCast(usize, size));
1290 var compressed_stream = std.io.fixedBufferStream(compressed_data);
1291
1292 try compressed_stream.writer().writeAll(prefix);
1293
1294 {
1295 var compressor = try std.compress.zlib.compressStream(allocator, compressed_stream.writer(), .{});
1296 defer compressor.deinit();
1297
1298 var buf: [8000]u8 = undefined;
1299 while (true) {
1300 const bytes_read = try section_reader.read(&buf);
1301 if (bytes_read == 0) break;
1302 const bytes_written = compressor.write(buf[0..bytes_read]) catch |err| switch (err) {
1303 error.NoSpaceLeft => {
1304 allocator.free(compressed_data);
1305 return null;
1306 },
1307 else => return err,
1308 };
1309 std.debug.assert(bytes_written == bytes_read);
1310 }
1311 compressor.finish() catch |err| switch (err) {
1312 error.NoSpaceLeft => {
1313 allocator.free(compressed_data);
1314 return null;
1315 },
1316 else => return err,
1317 };
1318 }
1319
1320 const compressed_len = @intCast(usize, compressed_stream.getPos() catch unreachable);
1321 const data = allocator.realloc(compressed_data, compressed_len) catch compressed_data;
1322 return data[0..compressed_len];
1323 }
1324
12711325 fn createDebugLink(path: []const u8) DebugLink {
12721326 const file = std.fs.cwd().openFile(path, .{}) catch |err| {
12731327 fatal("zig objcopy: could not open `{s}`: {s}\n", .{ path, @errorName(err) });