authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-12-15 15:10:35+01:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-12-16 18:05:58+01:00
log660270b7a9c492dbd7c0b76a823bcba5a13da71c
tree959e27c42fcdeb5f71b0649e9c501f763cba972f
parent09dee744145fc423feb2b74ffa22cc1679a2749e

macho: calculate UUID excluding stabs and part of contributing strtab

5 files changed, 102 insertions(+), 74 deletions(-)

CMakeLists.txt-1
...@@ -594,7 +594,6 @@ set(ZIG_STAGE2_SOURCES...@@ -594,7 +594,6 @@ set(ZIG_STAGE2_SOURCES
594 "${CMAKE_SOURCE_DIR}/src/link/MachO/hasher.zig"594 "${CMAKE_SOURCE_DIR}/src/link/MachO/hasher.zig"
595 "${CMAKE_SOURCE_DIR}/src/link/MachO/load_commands.zig"595 "${CMAKE_SOURCE_DIR}/src/link/MachO/load_commands.zig"
596 "${CMAKE_SOURCE_DIR}/src/link/MachO/thunks.zig"596 "${CMAKE_SOURCE_DIR}/src/link/MachO/thunks.zig"
597 "${CMAKE_SOURCE_DIR}/src/link/MachO/uuid.zig"
598 "${CMAKE_SOURCE_DIR}/src/link/MachO/zld.zig"597 "${CMAKE_SOURCE_DIR}/src/link/MachO/zld.zig"
599 "${CMAKE_SOURCE_DIR}/src/link/Plan9.zig"598 "${CMAKE_SOURCE_DIR}/src/link/Plan9.zig"
600 "${CMAKE_SOURCE_DIR}/src/link/Plan9/aout.zig"599 "${CMAKE_SOURCE_DIR}/src/link/Plan9/aout.zig"
src/link/MachO.zig+8
...@@ -39,6 +39,7 @@ const Object = @import("MachO/Object.zig");...@@ -39,6 +39,7 @@ const Object = @import("MachO/Object.zig");
39const LibStub = @import("tapi.zig").LibStub;39const LibStub = @import("tapi.zig").LibStub;
40const Liveness = @import("../Liveness.zig");40const Liveness = @import("../Liveness.zig");
41const LlvmObject = @import("../codegen/llvm.zig").Object;41const LlvmObject = @import("../codegen/llvm.zig").Object;
42const Md5 = std.crypto.hash.Md5;
42const Module = @import("../Module.zig");43const Module = @import("../Module.zig");
43const Relocation = @import("MachO/Relocation.zig");44const Relocation = @import("MachO/Relocation.zig");
44const StringTable = @import("strtab.zig").StringTable;45const StringTable = @import("strtab.zig").StringTable;
...@@ -598,6 +599,8 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No...@@ -598,6 +599,8 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No
598599
599 if (self.cold_start) {600 if (self.cold_start) {
600 std.crypto.random.bytes(&self.uuid_cmd.uuid);601 std.crypto.random.bytes(&self.uuid_cmd.uuid);
602 Md5.hash(&self.uuid_cmd.uuid, &self.uuid_cmd.uuid, .{});
603 conformUuid(&self.uuid_cmd.uuid);
601 }604 }
602 try lc_writer.writeStruct(self.uuid_cmd);605 try lc_writer.writeStruct(self.uuid_cmd);
603606
...@@ -662,6 +665,11 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No...@@ -662,6 +665,11 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No
662665
663 self.cold_start = false;666 self.cold_start = false;
664}667}
668inline fn conformUuid(out: *[Md5.digest_length]u8) void {
669 // LC_UUID uuids should conform to RFC 4122 UUID version 4 & UUID version 5 formats
670 out[6] = (out[6] & 0x0F) | (3 << 4);
671 out[8] = (out[8] & 0x3F) | 0x80;
672}
665673
666pub fn resolveLibSystem(674pub fn resolveLibSystem(
667 arena: Allocator,675 arena: Allocator,
src/link/MachO/hasher.zig+9-1
...@@ -13,6 +13,7 @@ pub fn ParallelHasher(comptime Hasher: type) type {...@@ -13,6 +13,7 @@ pub fn ParallelHasher(comptime Hasher: type) type {
13 return struct {13 return struct {
14 pub fn hash(self: @This(), gpa: Allocator, pool: *ThreadPool, file: fs.File, out: [][hash_size]u8, opts: struct {14 pub fn hash(self: @This(), gpa: Allocator, pool: *ThreadPool, file: fs.File, out: [][hash_size]u8, opts: struct {
15 chunk_size: u16 = 0x4000,15 chunk_size: u16 = 0x4000,
16 file_pos: u64 = 0,
16 max_file_size: ?u64 = null,17 max_file_size: ?u64 = null,
17 }) !void {18 }) !void {
18 _ = self;19 _ = self;
...@@ -38,7 +39,14 @@ pub fn ParallelHasher(comptime Hasher: type) type {...@@ -38,7 +39,14 @@ pub fn ParallelHasher(comptime Hasher: type) type {
38 const fstart = i * opts.chunk_size;39 const fstart = i * opts.chunk_size;
39 const fsize = if (fstart + opts.chunk_size > file_size) file_size - fstart else opts.chunk_size;40 const fsize = if (fstart + opts.chunk_size > file_size) file_size - fstart else opts.chunk_size;
40 wg.start();41 wg.start();
41 try pool.spawn(worker, .{ file, fstart, buffer[fstart..][0..fsize], &out[i], &results[i], &wg });42 try pool.spawn(worker, .{
43 file,
44 fstart + opts.file_pos,
45 buffer[fstart..][0..fsize],
46 &out[i],
47 &results[i],
48 &wg,
49 });
42 }50 }
43 }51 }
44 for (results) |result| _ = try result;52 for (results) |result| _ = try result;
src/link/MachO/uuid.zig deleted-69
...@@ -1,69 +0,0 @@
1const std = @import("std");
2const fs = std.fs;
3const mem = std.mem;
4
5const Allocator = mem.Allocator;
6const Compilation = @import("../../Compilation.zig");
7const Md5 = std.crypto.hash.Md5;
8const Hasher = @import("hasher.zig").ParallelHasher;
9
10/// Somewhat random chunk size for MD5 hash calculation.
11pub const chunk_size = 0x4000;
12
13/// Calculates Md5 hash of the file contents.
14/// Hash is calculated in a streaming manner which may be slow.
15pub fn calcUuidStreaming(file: fs.File, file_size: u64, out: *[Md5.digest_length]u8) !void {
16 const total_num_chunks = mem.alignForward(file_size, chunk_size) / chunk_size;
17
18 var hasher = Md5.init(.{});
19 var buffer: [chunk_size]u8 = undefined;
20
21 var i: usize = 0;
22 while (i < total_num_chunks) : (i += 1) {
23 const start = i * chunk_size;
24 const size = if (start + chunk_size > file_size)
25 file_size - start
26 else
27 chunk_size;
28 const amt = try file.preadAll(&buffer, start);
29 if (amt != size) return error.InputOutput;
30
31 hasher.update(buffer[0..size]);
32 }
33
34 hasher.final(out);
35 conform(out);
36}
37
38/// Calculates Md5 hash of each chunk in parallel and then hashes all Md5 hashes to produce
39/// the final digest.
40/// While this is NOT a correct MD5 hash of the contents, this methodology is used by LLVM/LLD
41/// and we will use it too as it seems accepted by Apple OSes.
42pub fn calcUuidParallel(comp: *const Compilation, file: fs.File, file_size: u64, out: *[Md5.digest_length]u8) !void {
43 const total_hashes = mem.alignForward(file_size, chunk_size) / chunk_size;
44
45 const hashes = try comp.gpa.alloc([Md5.digest_length]u8, total_hashes);
46 defer comp.gpa.free(hashes);
47
48 var hasher = Hasher(Md5){};
49 try hasher.hash(comp.gpa, comp.thread_pool, file, hashes, .{
50 .chunk_size = chunk_size,
51 .max_file_size = file_size,
52 });
53
54 const final_buffer = try comp.gpa.alloc(u8, total_hashes * Md5.digest_length);
55 defer comp.gpa.free(final_buffer);
56
57 for (hashes) |hash, i| {
58 mem.copy(u8, final_buffer[i * Md5.digest_length ..][0..Md5.digest_length], &hash);
59 }
60
61 Md5.hash(final_buffer, out, .{});
62 conform(out);
63}
64
65inline fn conform(out: *[Md5.digest_length]u8) void {
66 // LC_UUID uuids should conform to RFC 4122 UUID version 4 & UUID version 5 formats
67 out[6] = (out[6] & 0x0F) | (3 << 4);
68 out[8] = (out[8] & 0x3F) | 0x80;
69}
src/link/MachO/zld.zig+85-3
...@@ -16,7 +16,6 @@ const link = @import("../../link.zig");...@@ -16,7 +16,6 @@ const link = @import("../../link.zig");
16const load_commands = @import("load_commands.zig");16const load_commands = @import("load_commands.zig");
17const thunks = @import("thunks.zig");17const thunks = @import("thunks.zig");
18const trace = @import("../../tracy.zig").trace;18const trace = @import("../../tracy.zig").trace;
19const uuid = @import("uuid.zig");
2019
21const Allocator = mem.Allocator;20const Allocator = mem.Allocator;
22const Archive = @import("Archive.zig");21const Archive = @import("Archive.zig");
...@@ -26,7 +25,9 @@ const CodeSignature = @import("CodeSignature.zig");...@@ -26,7 +25,9 @@ const CodeSignature = @import("CodeSignature.zig");
26const Compilation = @import("../../Compilation.zig");25const Compilation = @import("../../Compilation.zig");
27const DwarfInfo = @import("DwarfInfo.zig");26const DwarfInfo = @import("DwarfInfo.zig");
28const Dylib = @import("Dylib.zig");27const Dylib = @import("Dylib.zig");
28const Hasher = @import("hasher.zig").ParallelHasher;
29const MachO = @import("../MachO.zig");29const MachO = @import("../MachO.zig");
30const Md5 = std.crypto.hash.Md5;
30const LibStub = @import("../tapi.zig").LibStub;31const LibStub = @import("../tapi.zig").LibStub;
31const Object = @import("Object.zig");32const Object = @import("Object.zig");
32const StringTable = @import("../strtab.zig").StringTable;33const StringTable = @import("../strtab.zig").StringTable;
...@@ -2680,17 +2681,98 @@ pub const Zld = struct {...@@ -2680,17 +2681,98 @@ pub const Zld = struct {
2680 // In Debug we don't really care about reproducibility, so put in a random value2681 // In Debug we don't really care about reproducibility, so put in a random value
2681 // and be done with it.2682 // and be done with it.
2682 std.crypto.random.bytes(&self.uuid_cmd.uuid);2683 std.crypto.random.bytes(&self.uuid_cmd.uuid);
2684 Md5.hash(&self.uuid_cmd.uuid, &self.uuid_cmd.uuid, .{});
2685 conformUuid(&self.uuid_cmd.uuid);
2683 },2686 },
2684 else => {2687 else => {
2685 const seg = self.getLinkeditSegmentPtr();2688 const seg = self.getLinkeditSegmentPtr();
2686 const file_size = seg.fileoff + seg.filesize;2689 const max_file_size = @intCast(u32, seg.fileoff + seg.filesize);
2687 try uuid.calcUuidParallel(comp, self.file, file_size, &self.uuid_cmd.uuid);2690
2691 var hashes = std.ArrayList([Md5.digest_length]u8).init(self.gpa);
2692 defer hashes.deinit();
2693
2694 if (!self.options.strip) {
2695 // First exclusion region will comprise all symbol stabs.
2696 const nlocals = self.dysymtab_cmd.nlocalsym;
2697
2698 const locals_buf = try self.gpa.alloc(u8, nlocals * @sizeOf(macho.nlist_64));
2699 defer self.gpa.free(locals_buf);
2700
2701 const amt = try self.file.preadAll(locals_buf, self.symtab_cmd.symoff);
2702 if (amt != locals_buf.len) return error.InputOutput;
2703 const locals = @ptrCast([*]macho.nlist_64, @alignCast(@alignOf(macho.nlist_64), locals_buf))[0..nlocals];
2704
2705 const istab: usize = for (locals) |local, i| {
2706 if (local.stab()) break i;
2707 } else locals.len;
2708 const nstabs = locals.len - istab;
2709
2710 // Next, a subsection of the strtab.
2711 // We do not care about anything succeeding strtab as it is the code signature data which is
2712 // not part of the UUID calculation anyway.
2713 const stab_stroff = locals[istab].n_strx;
2714
2715 const first_cut = FileSubsection{
2716 .start = 0,
2717 .end = @intCast(u32, self.symtab_cmd.symoff + istab * @sizeOf(macho.nlist_64)),
2718 };
2719 const second_cut = FileSubsection{
2720 .start = first_cut.end + @intCast(u32, nstabs * @sizeOf(macho.nlist_64)),
2721 .end = self.symtab_cmd.stroff + stab_stroff,
2722 };
2723
2724 for (&[_]FileSubsection{ first_cut, second_cut }) |cut| {
2725 try self.calcUuidHashes(comp, cut, &hashes);
2726 }
2727 } else {
2728 try self.calcUuidHashes(comp, .{ .start = 0, .end = max_file_size }, &hashes);
2729 }
2730
2731 const final_buffer = try self.gpa.alloc(u8, hashes.items.len * Md5.digest_length);
2732 defer self.gpa.free(final_buffer);
2733
2734 for (hashes.items) |hash, i| {
2735 mem.copy(u8, final_buffer[i * Md5.digest_length ..][0..Md5.digest_length], &hash);
2736 }
2737
2738 Md5.hash(final_buffer, &self.uuid_cmd.uuid, .{});
2739 conformUuid(&self.uuid_cmd.uuid);
2688 },2740 },
2689 }2741 }
2742
2690 const in_file = @sizeOf(macho.mach_header_64) + offset + @sizeOf(macho.load_command);2743 const in_file = @sizeOf(macho.mach_header_64) + offset + @sizeOf(macho.load_command);
2691 try self.file.pwriteAll(&self.uuid_cmd.uuid, in_file);2744 try self.file.pwriteAll(&self.uuid_cmd.uuid, in_file);
2692 }2745 }
26932746
2747 inline fn conformUuid(out: *[Md5.digest_length]u8) void {
2748 // LC_UUID uuids should conform to RFC 4122 UUID version 4 & UUID version 5 formats
2749 out[6] = (out[6] & 0x0F) | (3 << 4);
2750 out[8] = (out[8] & 0x3F) | 0x80;
2751 }
2752
2753 const FileSubsection = struct {
2754 start: u32,
2755 end: u32,
2756 };
2757
2758 fn calcUuidHashes(
2759 self: *Zld,
2760 comp: *const Compilation,
2761 cut: FileSubsection,
2762 hashes: *std.ArrayList([Md5.digest_length]u8),
2763 ) !void {
2764 const chunk_size = 0x4000;
2765 const total_hashes = mem.alignForward(cut.end - cut.start, chunk_size) / chunk_size;
2766 try hashes.resize(hashes.items.len + total_hashes);
2767
2768 var hasher = Hasher(Md5){};
2769 try hasher.hash(self.gpa, comp.thread_pool, self.file, hashes.items, .{
2770 .chunk_size = chunk_size,
2771 .file_pos = cut.start,
2772 .max_file_size = cut.end - cut.start,
2773 });
2774 }
2775
2694 fn writeCodeSignaturePadding(self: *Zld, code_sig: *CodeSignature) !void {2776 fn writeCodeSignaturePadding(self: *Zld, code_sig: *CodeSignature) !void {
2695 const seg = self.getLinkeditSegmentPtr();2777 const seg = self.getLinkeditSegmentPtr();
2696 // Code signature data has to be 16-bytes aligned for Apple tools to recognize the file2778 // Code signature data has to be 16-bytes aligned for Apple tools to recognize the file