authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-12-15 22:12:29+01:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-12-16 18:05:58+01:00
logd00094dd459f37d30b3297939bed6c320470fba8
tree724e14b1dd5862f2362f12cca490e867ae7f9d17
parentb323e14b1c50d731b643180972361552f8e5f5ec

macho: exclude all content of the binary that could cause non-deterministic UUID

1 files changed, 179 insertions(+), 47 deletions(-)

src/link/MachO/zld.zig+179-47
...@@ -25,7 +25,6 @@ const CodeSignature = @import("CodeSignature.zig");...@@ -25,7 +25,6 @@ const CodeSignature = @import("CodeSignature.zig");
25const Compilation = @import("../../Compilation.zig");25const Compilation = @import("../../Compilation.zig");
26const DwarfInfo = @import("DwarfInfo.zig");26const DwarfInfo = @import("DwarfInfo.zig");
27const Dylib = @import("Dylib.zig");27const Dylib = @import("Dylib.zig");
28const Hasher = @import("hasher.zig").ParallelHasher;
29const MachO = @import("../MachO.zig");28const MachO = @import("../MachO.zig");
30const Md5 = std.crypto.hash.Md5;29const Md5 = std.crypto.hash.Md5;
31const LibStub = @import("../tapi.zig").LibStub;30const LibStub = @import("../tapi.zig").LibStub;
...@@ -44,7 +43,9 @@ pub const Zld = struct {...@@ -44,7 +43,9 @@ pub const Zld = struct {
44 dysymtab_cmd: macho.dysymtab_command = .{},43 dysymtab_cmd: macho.dysymtab_command = .{},
45 function_starts_cmd: macho.linkedit_data_command = .{ .cmd = .FUNCTION_STARTS },44 function_starts_cmd: macho.linkedit_data_command = .{ .cmd = .FUNCTION_STARTS },
46 data_in_code_cmd: macho.linkedit_data_command = .{ .cmd = .DATA_IN_CODE },45 data_in_code_cmd: macho.linkedit_data_command = .{ .cmd = .DATA_IN_CODE },
47 uuid_cmd: macho.uuid_command = .{},46 uuid_cmd: macho.uuid_command = .{
47 .uuid = [_]u8{0} ** 16,
48 },
48 codesig_cmd: macho.linkedit_data_command = .{ .cmd = .CODE_SIGNATURE },49 codesig_cmd: macho.linkedit_data_command = .{ .cmd = .CODE_SIGNATURE },
4950
50 objects: std.ArrayListUnmanaged(Object) = .{},51 objects: std.ArrayListUnmanaged(Object) = .{},
...@@ -2679,7 +2680,9 @@ pub const Zld = struct {...@@ -2679,7 +2680,9 @@ pub const Zld = struct {
2679 linkedit_cmd_offset: u32,2680 linkedit_cmd_offset: u32,
2680 symtab_cmd_offset: u32,2681 symtab_cmd_offset: u32,
2681 uuid_cmd_offset: u32,2682 uuid_cmd_offset: u32,
2683 codesig_cmd_offset: ?u32,
2682 }) !void {2684 }) !void {
2685 _ = comp;
2683 switch (self.options.optimize_mode) {2686 switch (self.options.optimize_mode) {
2684 .Debug => {2687 .Debug => {
2685 // In Debug we don't really care about reproducibility, so put in a random value2688 // In Debug we don't really care about reproducibility, so put in a random value
...@@ -2689,27 +2692,34 @@ pub const Zld = struct {...@@ -2689,27 +2692,34 @@ pub const Zld = struct {
2689 conformUuid(&self.uuid_cmd.uuid);2692 conformUuid(&self.uuid_cmd.uuid);
2690 },2693 },
2691 else => {2694 else => {
2692 const seg = self.getLinkeditSegmentPtr();2695 const max_file_size = self.symtab_cmd.stroff + self.symtab_cmd.strsize;
2693 const max_file_size = @intCast(u32, seg.fileoff + seg.filesize);
26942696
2695 var hashes = std.ArrayList([Md5.digest_length]u8).init(self.gpa);2697 var subsections: [5]FileSubsection = undefined;
2696 defer hashes.deinit();2698 var count: usize = 0;
2697
2698 var subsections: [4]FileSubsection = undefined;
2699 var count: usize = 2;
27002699
2701 // Exclude LINKEDIT segment command as it contains file size that includes stabs contribution2700 // Exclude LINKEDIT segment command as it contains file size that includes stabs contribution
2702 // and code signature.2701 // and code signature.
2703 subsections[0] = .{2702 subsections[count] = .{
2704 .start = 0,2703 .start = 0,
2705 .end = args.linkedit_cmd_offset,2704 .end = args.linkedit_cmd_offset,
2706 };2705 };
2706 count += 1;
27072707
2708 // Exclude SYMTAB and DYSYMTAB commands for the same reason.2708 // Exclude SYMTAB and DYSYMTAB commands for the same reason.
2709 subsections[1] = .{2709 subsections[count] = .{
2710 .start = args.linkedit_cmd_offset + @sizeOf(macho.segment_command_64),2710 .start = subsections[count - 1].end + @sizeOf(macho.segment_command_64),
2711 .end = args.symtab_cmd_offset,2711 .end = args.symtab_cmd_offset,
2712 };2712 };
2713 count += 1;
2714
2715 // Exclude CODE_SIGNATURE command (if present).
2716 if (args.codesig_cmd_offset) |offset| {
2717 subsections[count] = .{
2718 .start = subsections[count - 1].end + @sizeOf(macho.symtab_command) + @sizeOf(macho.dysymtab_command),
2719 .end = offset,
2720 };
2721 count += 1;
2722 }
27132723
2714 if (!self.options.strip) {2724 if (!self.options.strip) {
2715 // Exclude region comprising all symbol stabs.2725 // Exclude region comprising all symbol stabs.
...@@ -2726,9 +2736,13 @@ pub const Zld = struct {...@@ -2726,9 +2736,13 @@ pub const Zld = struct {
2726 if (local.stab()) break i;2736 if (local.stab()) break i;
2727 } else locals.len;2737 } else locals.len;
2728 const nstabs = locals.len - istab;2738 const nstabs = locals.len - istab;
2739
2729 if (nstabs == 0) {2740 if (nstabs == 0) {
2730 subsections[2] = .{2741 subsections[count] = .{
2731 .start = args.symtab_cmd_offset + @sizeOf(macho.symtab_command) + @sizeOf(macho.dysymtab_command),2742 .start = subsections[count - 1].end + if (args.codesig_cmd_offset == null)
2743 @as(u32, @sizeOf(macho.symtab_command) + @sizeOf(macho.dysymtab_command))
2744 else
2745 @sizeOf(macho.linkedit_data_command),
2732 .end = max_file_size,2746 .end = max_file_size,
2733 };2747 };
2734 count += 1;2748 count += 1;
...@@ -2738,38 +2752,80 @@ pub const Zld = struct {...@@ -2738,38 +2752,80 @@ pub const Zld = struct {
2738 // not part of the UUID calculation anyway.2752 // not part of the UUID calculation anyway.
2739 const stab_stroff = locals[istab].n_strx;2753 const stab_stroff = locals[istab].n_strx;
27402754
2741 subsections[2] = .{2755 subsections[count] = .{
2742 .start = args.symtab_cmd_offset + @sizeOf(macho.symtab_command) + @sizeOf(macho.dysymtab_command),2756 .start = subsections[count - 1].end + if (args.codesig_cmd_offset == null)
2757 @as(u32, @sizeOf(macho.symtab_command) + @sizeOf(macho.dysymtab_command))
2758 else
2759 @sizeOf(macho.linkedit_data_command),
2743 .end = @intCast(u32, self.symtab_cmd.symoff + istab * @sizeOf(macho.nlist_64)),2760 .end = @intCast(u32, self.symtab_cmd.symoff + istab * @sizeOf(macho.nlist_64)),
2744 };2761 };
2745 subsections[3] = .{2762 count += 1;
2746 .start = subsections[2].end + @intCast(u32, nstabs * @sizeOf(macho.nlist_64)),2763
2764 subsections[count] = .{
2765 .start = subsections[count - 1].end + @intCast(u32, nstabs * @sizeOf(macho.nlist_64)),
2747 .end = self.symtab_cmd.stroff + stab_stroff,2766 .end = self.symtab_cmd.stroff + stab_stroff,
2748 };2767 };
27492768 count += 1;
2750 count += 2;
2751 }2769 }
2752 } else {2770 } else {
2753 subsections[2] = .{2771 subsections[count] = .{
2754 .start = args.symtab_cmd_offset + @sizeOf(macho.symtab_command) + @sizeOf(macho.dysymtab_command),2772 .start = subsections[count - 1].end + if (args.codesig_cmd_offset == null)
2773 @as(u32, @sizeOf(macho.symtab_command) + @sizeOf(macho.dysymtab_command))
2774 else
2775 @sizeOf(macho.linkedit_data_command),
2755 .end = max_file_size,2776 .end = max_file_size,
2756 };2777 };
2757 count += 1;2778 count += 1;
2758 }2779 }
27592780
2781 const chunk_size = 0x4000;
2782
2783 var rb = RingBuffer{};
2784 var hasher = Md5.init(.{});
2785 var buffer: [chunk_size]u8 = undefined;
2786 var hashed: usize = 0;
2787
2760 for (subsections[0..count]) |cut| {2788 for (subsections[0..count]) |cut| {
2761 std.debug.print("{x} - {x}\n", .{ cut.start, cut.end });2789 // std.debug.print("{x} - {x}, {x}\n", .{ cut.start, cut.end, cut.end - cut.start });
2762 try self.calcUuidHashes(comp, cut, &hashes);2790
2791 const size = cut.end - cut.start;
2792 const num_chunks = mem.alignForward(size, chunk_size) / chunk_size;
2793
2794 var i: usize = 0;
2795 while (i < num_chunks) : (i += 1) {
2796 const fstart = cut.start + i * chunk_size;
2797 const fsize = if (fstart + chunk_size > cut.end)
2798 cut.end - fstart
2799 else
2800 chunk_size;
2801 // std.debug.print("fstart {x}, fsize {x}\n", .{ fstart, fsize });
2802 const amt = try self.file.preadAll(buffer[0..fsize], fstart);
2803 if (amt != fsize) return error.InputOutput;
2804
2805 // try formatBinaryBlob(buffer[0..fsize], .{ .fmt_as_str = false }, std.io.getStdOut().writer());
2806
2807 var leftover = rb.append(buffer[0..fsize]);
2808 while (leftover > 0) {
2809 if (rb.full()) {
2810 hasher.update(rb.getBuffer());
2811 hashed += rb.getBuffer().len;
2812 rb.clear();
2813 }
2814 leftover = rb.append(buffer[fsize - leftover ..]);
2815 }
2816 }
2763 }2817 }
27642818
2765 const final_buffer = try self.gpa.alloc(u8, hashes.items.len * Md5.digest_length);2819 if (!rb.empty()) {
2766 defer self.gpa.free(final_buffer);2820 // try formatBinaryBlob(rb.getBuffer(), .{ .fmt_as_str = false }, std.io.getStdOut().writer());
27672821 hasher.update(rb.getBuffer());
2768 for (hashes.items) |hash, i| {2822 hashed += rb.getBuffer().len;
2769 mem.copy(u8, final_buffer[i * Md5.digest_length ..][0..Md5.digest_length], &hash);2823 rb.clear();
2770 }2824 }
27712825
2772 Md5.hash(final_buffer, &self.uuid_cmd.uuid, .{});2826 // std.debug.print("hashed {x}\n", .{hashed});
2827
2828 hasher.final(&self.uuid_cmd.uuid);
2773 conformUuid(&self.uuid_cmd.uuid);2829 conformUuid(&self.uuid_cmd.uuid);
2774 },2830 },
2775 }2831 }
...@@ -2778,6 +2834,79 @@ pub const Zld = struct {...@@ -2778,6 +2834,79 @@ pub const Zld = struct {
2778 try self.file.pwriteAll(&self.uuid_cmd.uuid, in_file);2834 try self.file.pwriteAll(&self.uuid_cmd.uuid, in_file);
2779 }2835 }
27802836
2837 const FmtBinaryBlobOpts = struct {
2838 fmt_as_str: bool = true,
2839 escape_str: bool = false,
2840 };
2841
2842 fn formatBinaryBlob(blob: []const u8, opts: FmtBinaryBlobOpts, writer: anytype) !void {
2843 // Format as 16-by-16-by-8 with two left column in hex, and right in ascii:
2844 // xxxxxxxxxxxxxxxx xxxxxxxxxxxxxxxx xxxxxxxx
2845 var i: usize = 0;
2846 const step = 16;
2847 var tmp_buf: [step]u8 = undefined;
2848 while (i < blob.len) : (i += step) {
2849 const end = if (blob[i..].len >= step) step else blob[i..].len;
2850 const padding = step - blob[i .. i + end].len;
2851 if (padding > 0) {
2852 mem.set(u8, &tmp_buf, 0);
2853 }
2854 mem.copy(u8, &tmp_buf, blob[i .. i + end]);
2855 try writer.print("{x} {x:<016} {x:<016}", .{
2856 i, std.fmt.fmtSliceHexLower(tmp_buf[0 .. step / 2]), std.fmt.fmtSliceHexLower(tmp_buf[step / 2 .. step]),
2857 });
2858 if (opts.fmt_as_str) {
2859 if (opts.escape_str) {
2860 try writer.print(" {s}", .{std.fmt.fmtSliceEscapeLower(tmp_buf[0..step])});
2861 } else {
2862 try writer.print(" {s}", .{tmp_buf[0..step]});
2863 }
2864 }
2865 try writer.writeByte('\n');
2866 }
2867 }
2868
2869 const RingBuffer = struct {
2870 buffer: [chunk_size]u8 = undefined,
2871 pos: usize = 0,
2872
2873 const chunk_size = 0x4000;
2874
2875 fn append(rb: *RingBuffer, data: []u8) usize {
2876 const cpy_size = if (data.len > rb.available())
2877 data.len - rb.available()
2878 else
2879 data.len;
2880 // std.debug.print(" appending {x} of {x} (pos {x})\n", .{ cpy_size, data.len, rb.pos });
2881 mem.copy(u8, rb.buffer[rb.pos..], data[0..cpy_size]);
2882 rb.pos += cpy_size;
2883 const leftover = data.len - cpy_size;
2884 // std.debug.print(" leftover {x}\n", .{leftover});
2885 // std.debug.print(" buffer {x} full\n", .{rb.pos});
2886 return leftover;
2887 }
2888
2889 fn available(rb: RingBuffer) usize {
2890 return rb.buffer.len - rb.pos;
2891 }
2892
2893 fn clear(rb: *RingBuffer) void {
2894 rb.pos = 0;
2895 }
2896
2897 fn full(rb: RingBuffer) bool {
2898 return rb.buffer.len == rb.pos;
2899 }
2900
2901 fn empty(rb: RingBuffer) bool {
2902 return rb.pos == 0;
2903 }
2904
2905 fn getBuffer(rb: *const RingBuffer) []const u8 {
2906 return rb.buffer[0..rb.pos];
2907 }
2908 };
2909
2781 inline fn conformUuid(out: *[Md5.digest_length]u8) void {2910 inline fn conformUuid(out: *[Md5.digest_length]u8) void {
2782 // LC_UUID uuids should conform to RFC 4122 UUID version 4 & UUID version 5 formats2911 // LC_UUID uuids should conform to RFC 4122 UUID version 4 & UUID version 5 formats
2783 out[6] = (out[6] & 0x0F) | (3 << 4);2912 out[6] = (out[6] & 0x0F) | (3 << 4);
...@@ -2789,23 +2918,23 @@ pub const Zld = struct {...@@ -2789,23 +2918,23 @@ pub const Zld = struct {
2789 end: u32,2918 end: u32,
2790 };2919 };
27912920
2792 fn calcUuidHashes(2921 // fn calcUuidHashes(
2793 self: *Zld,2922 // self: *Zld,
2794 comp: *const Compilation,2923 // comp: *const Compilation,
2795 cut: FileSubsection,2924 // cut: FileSubsection,
2796 hashes: *std.ArrayList([Md5.digest_length]u8),2925 // hashes: *std.ArrayList([Md5.digest_length]u8),
2797 ) !void {2926 // ) !void {
2798 const chunk_size = 0x4000;2927 // const chunk_size = 0x4000;
2799 const total_hashes = mem.alignForward(cut.end - cut.start, chunk_size) / chunk_size;2928 // const total_hashes = mem.alignForward(cut.end - cut.start, chunk_size) / chunk_size;
2800 try hashes.resize(hashes.items.len + total_hashes);2929 // try hashes.resize(hashes.items.len + total_hashes);
28012930
2802 var hasher = Hasher(Md5){};2931 // var hasher = Hasher(Md5){};
2803 try hasher.hash(self.gpa, comp.thread_pool, self.file, hashes.items, .{2932 // try hasher.hash(self.gpa, comp.thread_pool, self.file, hashes.items, .{
2804 .chunk_size = chunk_size,2933 // .chunk_size = chunk_size,
2805 .file_pos = cut.start,2934 // .file_pos = cut.start,
2806 .max_file_size = cut.end - cut.start,2935 // .max_file_size = cut.end - cut.start,
2807 });2936 // });
2808 }2937 // }
28092938
2810 fn writeCodeSignaturePadding(self: *Zld, code_sig: *CodeSignature) !void {2939 fn writeCodeSignaturePadding(self: *Zld, code_sig: *CodeSignature) !void {
2811 const seg = self.getLinkeditSegmentPtr();2940 const seg = self.getLinkeditSegmentPtr();
...@@ -4133,6 +4262,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr...@@ -4133,6 +4262,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr
4133 if (cpu_arch == .aarch64 and (os_tag == .macos or abi == .simulator)) break :blk true;4262 if (cpu_arch == .aarch64 and (os_tag == .macos or abi == .simulator)) break :blk true;
4134 break :blk false;4263 break :blk false;
4135 };4264 };
4265 var codesig_cmd_offset: ?u32 = null;
4136 var codesig: ?CodeSignature = if (requires_codesig) blk: {4266 var codesig: ?CodeSignature = if (requires_codesig) blk: {
4137 // Preallocate space for the code signature.4267 // Preallocate space for the code signature.
4138 // We need to do this at this stage so that we have the load commands with proper values4268 // We need to do this at this stage so that we have the load commands with proper values
...@@ -4145,6 +4275,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr...@@ -4145,6 +4275,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr
4145 try codesig.addEntitlements(gpa, path);4275 try codesig.addEntitlements(gpa, path);
4146 }4276 }
4147 try zld.writeCodeSignaturePadding(&codesig);4277 try zld.writeCodeSignaturePadding(&codesig);
4278 codesig_cmd_offset = @sizeOf(macho.mach_header_64) + @intCast(u32, lc_buffer.items.len);
4148 try lc_writer.writeStruct(zld.codesig_cmd);4279 try lc_writer.writeStruct(zld.codesig_cmd);
4149 break :blk codesig;4280 break :blk codesig;
4150 } else null;4281 } else null;
...@@ -4158,6 +4289,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr...@@ -4158,6 +4289,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr
4158 .linkedit_cmd_offset = linkedit_cmd_offset,4289 .linkedit_cmd_offset = linkedit_cmd_offset,
4159 .symtab_cmd_offset = symtab_cmd_offset,4290 .symtab_cmd_offset = symtab_cmd_offset,
4160 .uuid_cmd_offset = uuid_cmd_offset,4291 .uuid_cmd_offset = uuid_cmd_offset,
4292 .codesig_cmd_offset = codesig_cmd_offset,
4161 });4293 });
41624294
4163 if (codesig) |*csig| {4295 if (codesig) |*csig| {