authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-12-15 23:43:50+01:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-12-16 18:05:58+01:00
log585c21e54d339f207028d871a45546da2d1b6871
tree0b830739f4a388e4767b54d55a7f070e2209ece4
parent1928ed7dab3949db964d96dad82beb776e69554f

macho: move parallel file hashing back to CodeSignature

I need to think some more how to calculate UUID in parallel, if it is even possible, to preserve UUID's determinism.

4 files changed, 67 insertions(+), 101 deletions(-)

CMakeLists.txt-1
...@@ -591,7 +591,6 @@ set(ZIG_STAGE2_SOURCES...@@ -591,7 +591,6 @@ set(ZIG_STAGE2_SOURCES
591 "${CMAKE_SOURCE_DIR}/src/link/MachO/bind.zig"591 "${CMAKE_SOURCE_DIR}/src/link/MachO/bind.zig"
592 "${CMAKE_SOURCE_DIR}/src/link/MachO/dead_strip.zig"592 "${CMAKE_SOURCE_DIR}/src/link/MachO/dead_strip.zig"
593 "${CMAKE_SOURCE_DIR}/src/link/MachO/fat.zig"593 "${CMAKE_SOURCE_DIR}/src/link/MachO/fat.zig"
594 "${CMAKE_SOURCE_DIR}/src/link/MachO/hasher.zig"
595 "${CMAKE_SOURCE_DIR}/src/link/MachO/load_commands.zig"594 "${CMAKE_SOURCE_DIR}/src/link/MachO/load_commands.zig"
596 "${CMAKE_SOURCE_DIR}/src/link/MachO/thunks.zig"595 "${CMAKE_SOURCE_DIR}/src/link/MachO/thunks.zig"
597 "${CMAKE_SOURCE_DIR}/src/link/MachO/zld.zig"596 "${CMAKE_SOURCE_DIR}/src/link/MachO/zld.zig"
src/link/MachO/CodeSignature.zig+59-6
...@@ -10,8 +10,9 @@ const testing = std.testing;...@@ -10,8 +10,9 @@ const testing = std.testing;
1010
11const Allocator = mem.Allocator;11const Allocator = mem.Allocator;
12const Compilation = @import("../../Compilation.zig");12const Compilation = @import("../../Compilation.zig");
13const Hasher = @import("hasher.zig").ParallelHasher;
14const Sha256 = std.crypto.hash.sha2.Sha256;13const Sha256 = std.crypto.hash.sha2.Sha256;
14const ThreadPool = @import("../../ThreadPool.zig");
15const WaitGroup = @import("../../WaitGroup.zig");
1516
16const hash_size = Sha256.digest_length;17const hash_size = Sha256.digest_length;
1718
...@@ -288,11 +289,7 @@ pub fn writeAdhocSignature(...@@ -288,11 +289,7 @@ pub fn writeAdhocSignature(
288 self.code_directory.inner.nCodeSlots = total_pages;289 self.code_directory.inner.nCodeSlots = total_pages;
289290
290 // Calculate hash for each page (in file) and write it to the buffer291 // Calculate hash for each page (in file) and write it to the buffer
291 var hasher = Hasher(Sha256){};292 try self.parallelHash(gpa, comp.thread_pool, opts.file, opts.file_size);
292 try hasher.hash(gpa, comp.thread_pool, opts.file, self.code_directory.code_slots.items, .{
293 .chunk_size = self.page_size,
294 .max_file_size = opts.file_size,
295 });
296293
297 try blobs.append(.{ .code_directory = &self.code_directory });294 try blobs.append(.{ .code_directory = &self.code_directory });
298 header.length += @sizeOf(macho.BlobIndex);295 header.length += @sizeOf(macho.BlobIndex);
...@@ -351,6 +348,62 @@ pub fn writeAdhocSignature(...@@ -351,6 +348,62 @@ pub fn writeAdhocSignature(
351 }348 }
352}349}
353350
351fn parallelHash(
352 self: *CodeSignature,
353 gpa: Allocator,
354 pool: *ThreadPool,
355 file: fs.File,
356 file_size: u64,
357) !void {
358 var wg: WaitGroup = .{};
359
360 const total_num_chunks = mem.alignForward(file_size, self.page_size) / self.page_size;
361 assert(self.code_directory.code_slots.items.len >= total_num_chunks);
362
363 const buffer = try gpa.alloc(u8, self.page_size * total_num_chunks);
364 defer gpa.free(buffer);
365
366 const results = try gpa.alloc(fs.File.PReadError!usize, total_num_chunks);
367 defer gpa.free(results);
368
369 {
370 wg.reset();
371 defer wg.wait();
372
373 var i: usize = 0;
374 while (i < total_num_chunks) : (i += 1) {
375 const fstart = i * self.page_size;
376 const fsize = if (fstart + self.page_size > file_size)
377 file_size - fstart
378 else
379 self.page_size;
380 wg.start();
381 try pool.spawn(worker, .{
382 file,
383 fstart,
384 buffer[fstart..][0..fsize],
385 &self.code_directory.code_slots.items[i],
386 &results[i],
387 &wg,
388 });
389 }
390 }
391 for (results) |result| _ = try result;
392}
393
394fn worker(
395 file: fs.File,
396 fstart: usize,
397 buffer: []u8,
398 out: *[hash_size]u8,
399 err: *fs.File.PReadError!usize,
400 wg: *WaitGroup,
401) void {
402 defer wg.finish();
403 err.* = file.preadAll(buffer, fstart);
404 Sha256.hash(buffer, out, .{});
405}
406
354pub fn size(self: CodeSignature) u32 {407pub fn size(self: CodeSignature) u32 {
355 var ssize: u32 = @sizeOf(macho.SuperBlob) + @sizeOf(macho.BlobIndex) + self.code_directory.size();408 var ssize: u32 = @sizeOf(macho.SuperBlob) + @sizeOf(macho.BlobIndex) + self.code_directory.size();
356 if (self.requirements) |req| {409 if (self.requirements) |req| {
src/link/MachO/hasher.zig deleted-68
...@@ -1,68 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const fs = std.fs;
4const mem = std.mem;
5
6const Allocator = mem.Allocator;
7const ThreadPool = @import("../../ThreadPool.zig");
8const WaitGroup = @import("../../WaitGroup.zig");
9
10pub fn ParallelHasher(comptime Hasher: type) type {
11 const hash_size = Hasher.digest_length;
12
13 return struct {
14 pub fn hash(self: @This(), gpa: Allocator, pool: *ThreadPool, file: fs.File, out: [][hash_size]u8, opts: struct {
15 chunk_size: u16 = 0x4000,
16 file_pos: u64 = 0,
17 max_file_size: ?u64 = null,
18 }) !void {
19 _ = self;
20
21 var wg: WaitGroup = .{};
22
23 const file_size = opts.max_file_size orelse try file.getEndPos();
24 const total_num_chunks = mem.alignForward(file_size, opts.chunk_size) / opts.chunk_size;
25 assert(out.len >= total_num_chunks);
26
27 const buffer = try gpa.alloc(u8, opts.chunk_size * total_num_chunks);
28 defer gpa.free(buffer);
29
30 const results = try gpa.alloc(fs.File.PReadError!usize, total_num_chunks);
31 defer gpa.free(results);
32
33 {
34 wg.reset();
35 defer wg.wait();
36
37 var i: usize = 0;
38 while (i < total_num_chunks) : (i += 1) {
39 const fstart = i * opts.chunk_size;
40 const fsize = if (fstart + opts.chunk_size > file_size) file_size - fstart else opts.chunk_size;
41 wg.start();
42 try pool.spawn(worker, .{
43 file,
44 fstart + opts.file_pos,
45 buffer[fstart..][0..fsize],
46 &out[i],
47 &results[i],
48 &wg,
49 });
50 }
51 }
52 for (results) |result| _ = try result;
53 }
54
55 fn worker(
56 file: fs.File,
57 fstart: usize,
58 buffer: []u8,
59 out: *[hash_size]u8,
60 err: *fs.File.PReadError!usize,
61 wg: *WaitGroup,
62 ) void {
63 defer wg.finish();
64 err.* = file.preadAll(buffer, fstart);
65 Hasher.hash(buffer, out, .{});
66 }
67 };
68}
src/link/MachO/zld.zig+8-26
...@@ -2692,7 +2692,12 @@ pub const Zld = struct {...@@ -2692,7 +2692,12 @@ pub const Zld = struct {
2692 conformUuid(&self.uuid_cmd.uuid);2692 conformUuid(&self.uuid_cmd.uuid);
2693 },2693 },
2694 else => {2694 else => {
2695 const max_file_size = self.symtab_cmd.stroff + self.symtab_cmd.strsize;2695 const max_file_end = self.symtab_cmd.stroff + self.symtab_cmd.strsize;
2696
2697 const FileSubsection = struct {
2698 start: u32,
2699 end: u32,
2700 };
26962701
2697 var subsections: [5]FileSubsection = undefined;2702 var subsections: [5]FileSubsection = undefined;
2698 var count: usize = 0;2703 var count: usize = 0;
...@@ -2743,7 +2748,7 @@ pub const Zld = struct {...@@ -2743,7 +2748,7 @@ pub const Zld = struct {
2743 @as(u32, @sizeOf(macho.symtab_command) + @sizeOf(macho.dysymtab_command))2748 @as(u32, @sizeOf(macho.symtab_command) + @sizeOf(macho.dysymtab_command))
2744 else2749 else
2745 @sizeOf(macho.linkedit_data_command),2750 @sizeOf(macho.linkedit_data_command),
2746 .end = max_file_size,2751 .end = max_file_end,
2747 };2752 };
2748 count += 1;2753 count += 1;
2749 } else {2754 } else {
...@@ -2773,7 +2778,7 @@ pub const Zld = struct {...@@ -2773,7 +2778,7 @@ pub const Zld = struct {
2773 @as(u32, @sizeOf(macho.symtab_command) + @sizeOf(macho.dysymtab_command))2778 @as(u32, @sizeOf(macho.symtab_command) + @sizeOf(macho.dysymtab_command))
2774 else2779 else
2775 @sizeOf(macho.linkedit_data_command),2780 @sizeOf(macho.linkedit_data_command),
2776 .end = max_file_size,2781 .end = max_file_end,
2777 };2782 };
2778 count += 1;2783 count += 1;
2779 }2784 }
...@@ -2816,29 +2821,6 @@ pub const Zld = struct {...@@ -2816,29 +2821,6 @@ pub const Zld = struct {
2816 out[8] = (out[8] & 0x3F) | 0x80;2821 out[8] = (out[8] & 0x3F) | 0x80;
2817 }2822 }
28182823
2819 const FileSubsection = struct {
2820 start: u32,
2821 end: u32,
2822 };
2823
2824 // fn calcUuidHashes(
2825 // self: *Zld,
2826 // comp: *const Compilation,
2827 // cut: FileSubsection,
2828 // hashes: *std.ArrayList([Md5.digest_length]u8),
2829 // ) !void {
2830 // const chunk_size = 0x4000;
2831 // const total_hashes = mem.alignForward(cut.end - cut.start, chunk_size) / chunk_size;
2832 // try hashes.resize(hashes.items.len + total_hashes);
2833
2834 // var hasher = Hasher(Md5){};
2835 // try hasher.hash(self.gpa, comp.thread_pool, self.file, hashes.items, .{
2836 // .chunk_size = chunk_size,
2837 // .file_pos = cut.start,
2838 // .max_file_size = cut.end - cut.start,
2839 // });
2840 // }
2841
2842 fn writeCodeSignaturePadding(self: *Zld, code_sig: *CodeSignature) !void {2824 fn writeCodeSignaturePadding(self: *Zld, code_sig: *CodeSignature) !void {
2843 const seg = self.getLinkeditSegmentPtr();2825 const seg = self.getLinkeditSegmentPtr();
2844 // Code signature data has to be 16-bytes aligned for Apple tools to recognize the file2826 // Code signature data has to be 16-bytes aligned for Apple tools to recognize the file