| ... | @@ -7,22 +7,19 @@ const assert = std.debug.assert; | ... | @@ -7,22 +7,19 @@ const assert = std.debug.assert; |
| 7 | const tmpDir = std.testing.tmpDir; | 7 | const tmpDir = std.testing.tmpDir; |
| 8 | | 8 | |
| 9 | const Allocator = mem.Allocator; | 9 | const Allocator = mem.Allocator; |
| 10 | const Blake3 = std.crypto.hash.Blake3; | | |
| 11 | const OsTag = std.Target.Os.Tag; | 10 | const OsTag = std.Target.Os.Tag; |
| 12 | | 11 | |
| 13 | var general_purpose_allocator = std.heap.GeneralPurposeAllocator(.{}){}; | 12 | var general_purpose_allocator = std.heap.GeneralPurposeAllocator(.{}){}; |
| 14 | const gpa = general_purpose_allocator.allocator(); | 13 | const gpa = general_purpose_allocator.allocator(); |
| 15 | | 14 | |
| 16 | const Arch = enum { | 15 | const Arch = enum { |
| 17 | any, | | |
| 18 | aarch64, | 16 | aarch64, |
| 19 | x86_64, | 17 | x86_64, |
| 20 | }; | 18 | }; |
| 21 | | 19 | |
| 22 | const Abi = enum { any, none }; | 20 | const Abi = enum { none }; |
| 23 | | 21 | |
| 24 | const OsVer = enum(u32) { | 22 | const OsVer = enum(u32) { |
| 25 | any = 0, | | |
| 26 | catalina = 10, | 23 | catalina = 10, |
| 27 | big_sur = 11, | 24 | big_sur = 11, |
| 28 | monterey = 12, | 25 | monterey = 12, |
| ... | @@ -37,22 +34,6 @@ const Target = struct { | ... | @@ -37,22 +34,6 @@ const Target = struct { |
| 37 | os_ver: OsVer, | 34 | os_ver: OsVer, |
| 38 | abi: Abi = .none, | 35 | abi: Abi = .none, |
| 39 | | 36 | |
| 40 | fn hash(a: Target) u32 { | | |
| 41 | var hasher = std.hash.Wyhash.init(0); | | |
| 42 | std.hash.autoHash(&hasher, a.arch); | | |
| 43 | std.hash.autoHash(&hasher, a.os); | | |
| 44 | std.hash.autoHash(&hasher, a.os_ver); | | |
| 45 | std.hash.autoHash(&hasher, a.abi); | | |
| 46 | return @as(u32, @truncate(hasher.final())); | | |
| 47 | } | | |
| 48 | | | |
| 49 | fn eql(a: Target, b: Target) bool { | | |
| 50 | return a.arch == b.arch and | | |
| 51 | a.os == b.os and | | |
| 52 | a.os_ver == b.os_ver and | | |
| 53 | a.abi == b.abi; | | |
| 54 | } | | |
| 55 | | | |
| 56 | fn name(self: Target, allocator: Allocator) ![]const u8 { | 37 | fn name(self: Target, allocator: Allocator) ![]const u8 { |
| 57 | return std.fmt.allocPrint(allocator, "{s}-{s}-{s}", .{ | 38 | return std.fmt.allocPrint(allocator, "{s}-{s}-{s}", .{ |
| 58 | @tagName(self.arch), | 39 | @tagName(self.arch), |
| ... | @@ -62,7 +43,6 @@ const Target = struct { | ... | @@ -62,7 +43,6 @@ const Target = struct { |
| 62 | } | 43 | } |
| 63 | | 44 | |
| 64 | fn fullName(self: Target, allocator: Allocator) ![]const u8 { | 45 | fn fullName(self: Target, allocator: Allocator) ![]const u8 { |
| 65 | if (self.os_ver == .any) return self.name(allocator); | | |
| 66 | return std.fmt.allocPrint(allocator, "{s}-{s}.{d}-{s}", .{ | 46 | return std.fmt.allocPrint(allocator, "{s}-{s}.{d}-{s}", .{ |
| 67 | @tagName(self.arch), | 47 | @tagName(self.arch), |
| 68 | @tagName(self.os), | 48 | @tagName(self.os), |
| ... | @@ -72,123 +52,13 @@ const Target = struct { | ... | @@ -72,123 +52,13 @@ const Target = struct { |
| 72 | } | 52 | } |
| 73 | }; | 53 | }; |
| 74 | | 54 | |
| 75 | const targets = [_]Target{ | | |
| 76 | Target{ | | |
| 77 | .arch = .any, | | |
| 78 | .abi = .any, | | |
| 79 | .os_ver = .any, | | |
| 80 | }, | | |
| 81 | Target{ | | |
| 82 | .arch = .aarch64, | | |
| 83 | .os_ver = .any, | | |
| 84 | }, | | |
| 85 | Target{ | | |
| 86 | .arch = .x86_64, | | |
| 87 | .os_ver = .any, | | |
| 88 | }, | | |
| 89 | Target{ | | |
| 90 | .arch = .x86_64, | | |
| 91 | .os_ver = .catalina, | | |
| 92 | }, | | |
| 93 | Target{ | | |
| 94 | .arch = .x86_64, | | |
| 95 | .os_ver = .big_sur, | | |
| 96 | }, | | |
| 97 | Target{ | | |
| 98 | .arch = .x86_64, | | |
| 99 | .os_ver = .monterey, | | |
| 100 | }, | | |
| 101 | Target{ | | |
| 102 | .arch = .x86_64, | | |
| 103 | .os_ver = .ventura, | | |
| 104 | }, | | |
| 105 | Target{ | | |
| 106 | .arch = .x86_64, | | |
| 107 | .os_ver = .sonoma, | | |
| 108 | }, | | |
| 109 | Target{ | | |
| 110 | .arch = .x86_64, | | |
| 111 | .os_ver = .sequoia, | | |
| 112 | }, | | |
| 113 | Target{ | | |
| 114 | .arch = .aarch64, | | |
| 115 | .os_ver = .big_sur, | | |
| 116 | }, | | |
| 117 | Target{ | | |
| 118 | .arch = .aarch64, | | |
| 119 | .os_ver = .monterey, | | |
| 120 | }, | | |
| 121 | Target{ | | |
| 122 | .arch = .aarch64, | | |
| 123 | .os_ver = .ventura, | | |
| 124 | }, | | |
| 125 | Target{ | | |
| 126 | .arch = .aarch64, | | |
| 127 | .os_ver = .sonoma, | | |
| 128 | }, | | |
| 129 | Target{ | | |
| 130 | .arch = .aarch64, | | |
| 131 | .os_ver = .sequoia, | | |
| 132 | }, | | |
| 133 | }; | | |
| 134 | | | |
| 135 | const headers_source_prefix: []const u8 = "headers"; | 55 | const headers_source_prefix: []const u8 = "headers"; |
| 136 | | 56 | |
| 137 | const Contents = struct { | | |
| 138 | bytes: []const u8, | | |
| 139 | hit_count: usize, | | |
| 140 | hash: []const u8, | | |
| 141 | is_generic: bool, | | |
| 142 | | | |
| 143 | fn hitCountLessThan(context: void, lhs: *const Contents, rhs: *const Contents) bool { | | |
| 144 | _ = context; | | |
| 145 | return lhs.hit_count < rhs.hit_count; | | |
| 146 | } | | |
| 147 | }; | | |
| 148 | | | |
| 149 | const TargetToHashContext = struct { | | |
| 150 | pub fn hash(self: @This(), target: Target) u32 { | | |
| 151 | _ = self; | | |
| 152 | return target.hash(); | | |
| 153 | } | | |
| 154 | pub fn eql(self: @This(), a: Target, b: Target, b_index: usize) bool { | | |
| 155 | _ = self; | | |
| 156 | _ = b_index; | | |
| 157 | return a.eql(b); | | |
| 158 | } | | |
| 159 | }; | | |
| 160 | const TargetToHash = std.ArrayHashMap(Target, []const u8, TargetToHashContext, true); | | |
| 161 | | | |
| 162 | const HashToContents = std.StringHashMap(Contents); | | |
| 163 | const PathTable = std.StringHashMap(*TargetToHash); | | |
| 164 | | | |
| 165 | /// The don't-dedup-list contains file paths with known problematic headers | | |
| 166 | /// which while contain the same contents between architectures, should not be | | |
| 167 | /// deduped since they contain includes, etc. which are relative and thus cannot be separated | | |
| 168 | /// into a shared include dir such as `any-macos-any`. | | |
| 169 | const dont_dedup_list = &[_][]const u8{ | | |
| 170 | "libkern/OSAtomic.h", | | |
| 171 | "libkern/OSAtomicDeprecated.h", | | |
| 172 | "libkern/OSSpinLockDeprecated.h", | | |
| 173 | "libkern/OSAtomicQueue.h", | | |
| 174 | }; | | |
| 175 | | | |
| 176 | fn generateDontDedupMap(arena: Allocator) !std.StringHashMap(void) { | | |
| 177 | var map = std.StringHashMap(void).init(arena); | | |
| 178 | try map.ensureTotalCapacity(dont_dedup_list.len); | | |
| 179 | for (dont_dedup_list) |path| { | | |
| 180 | map.putAssumeCapacityNoClobber(path, {}); | | |
| 181 | } | | |
| 182 | return map; | | |
| 183 | } | | |
| 184 | | | |
| 185 | const usage = | 57 | const usage = |
| 186 | \\fetch_them_macos_headers fetch | 58 | \\fetch_them_macos_headers [options] [cc args] |
| 187 | \\fetch_them_macos_headers dedup | | |
| 188 | \\ | 59 | \\ |
| 189 | \\Commands: | 60 | \\Options: |
| 190 | \\ fetch Fetch libc headers into headers/<arch>-macos.<os_ver> dir | 61 | \\ --sysroot Path to macOS SDK |
| 191 | \\ dedup Generate deduplicated dirs into a given <destination> path | | |
| 192 | \\ | 62 | \\ |
| 193 | \\General Options: | 63 | \\General Options: |
| 194 | \\-h, --help Print this help and exit | 64 | \\-h, --help Print this help and exit |
| ... | @@ -197,70 +67,17 @@ const usage = | ... | @@ -197,70 +67,17 @@ const usage = |
| 197 | pub fn main() anyerror!void { | 67 | pub fn main() anyerror!void { |
| 198 | var arena = std.heap.ArenaAllocator.init(gpa); | 68 | var arena = std.heap.ArenaAllocator.init(gpa); |
| 199 | defer arena.deinit(); | 69 | defer arena.deinit(); |
| | 70 | const allocator = arena.allocator(); |
| 200 | | 71 | |
| 201 | const all_args = try std.process.argsAlloc(arena.allocator()); | 72 | const args = try std.process.argsAlloc(allocator); |
| 202 | const args = all_args[1..]; | | |
| 203 | if (args.len == 0) fatal("no command or option specified", .{}); | | |
| 204 | | | |
| 205 | const cmd = args[0]; | | |
| 206 | if (mem.eql(u8, cmd, "--help") or mem.eql(u8, cmd, "-h")) { | | |
| 207 | return info(usage, .{}); | | |
| 208 | } else if (mem.eql(u8, cmd, "dedup")) { | | |
| 209 | return dedup(arena.allocator(), args[1..]); | | |
| 210 | } else if (mem.eql(u8, cmd, "fetch")) { | | |
| 211 | return fetch(arena.allocator(), args[1..]); | | |
| 212 | } else fatal("unknown command or option: {s}", .{cmd}); | | |
| 213 | } | | |
| 214 | | | |
| 215 | const ArgsIterator = struct { | | |
| 216 | args: []const []const u8, | | |
| 217 | i: usize = 0, | | |
| 218 | | | |
| 219 | fn next(it: *@This()) ?[]const u8 { | | |
| 220 | if (it.i >= it.args.len) { | | |
| 221 | return null; | | |
| 222 | } | | |
| 223 | defer it.i += 1; | | |
| 224 | return it.args[it.i]; | | |
| 225 | } | | |
| 226 | | | |
| 227 | fn nextOrFatal(it: *@This()) []const u8 { | | |
| 228 | const arg = it.next() orelse fatal("expected parameter after '{s}'", .{it.args[it.i - 1]}); | | |
| 229 | return arg; | | |
| 230 | } | | |
| 231 | }; | | |
| 232 | | | |
| 233 | fn info(comptime format: []const u8, args: anytype) void { | | |
| 234 | const msg = std.fmt.allocPrint(gpa, "info: " ++ format ++ "\n", args) catch return; | | |
| 235 | std.io.getStdOut().writeAll(msg) catch {}; | | |
| 236 | } | | |
| 237 | | | |
| 238 | fn fatal(comptime format: []const u8, args: anytype) noreturn { | | |
| 239 | ret: { | | |
| 240 | const msg = std.fmt.allocPrint(gpa, "fatal: " ++ format ++ "\n", args) catch break :ret; | | |
| 241 | std.io.getStdErr().writeAll(msg) catch {}; | | |
| 242 | } | | |
| 243 | std.process.exit(1); | | |
| 244 | } | | |
| 245 | | | |
| 246 | const fetch_usage = | | |
| 247 | \\fetch_them_macos_headers fetch | | |
| 248 | \\ | | |
| 249 | \\Options: | | |
| 250 | \\ --sysroot Path to macOS SDK | | |
| 251 | \\ | | |
| 252 | \\General Options: | | |
| 253 | \\-h, --help Print this help and exit | | |
| 254 | ; | | |
| 255 | | 73 | |
| 256 | fn fetch(arena: Allocator, args: []const []const u8) !void { | 74 | var argv = std.ArrayList([]const u8).init(allocator); |
| 257 | var argv = std.ArrayList([]const u8).init(arena); | | |
| 258 | var sysroot: ?[]const u8 = null; | 75 | var sysroot: ?[]const u8 = null; |
| 259 | | 76 | |
| 260 | var args_iter = ArgsIterator{ .args = args }; | 77 | var args_iter = ArgsIterator{ .args = args[1..] }; |
| 261 | while (args_iter.next()) |arg| { | 78 | while (args_iter.next()) |arg| { |
| 262 | if (mem.eql(u8, arg, "--help") or mem.eql(u8, arg, "-h")) { | 79 | if (mem.eql(u8, arg, "--help") or mem.eql(u8, arg, "-h")) { |
| 263 | return info(fetch_usage, .{}); | 80 | return info(usage, .{}); |
| 264 | } else if (mem.eql(u8, arg, "--sysroot")) { | 81 | } else if (mem.eql(u8, arg, "--sysroot")) { |
| 265 | sysroot = args_iter.nextOrFatal(); | 82 | sysroot = args_iter.nextOrFatal(); |
| 266 | } else try argv.append(arg); | 83 | } else try argv.append(arg); |
| ... | @@ -268,17 +85,17 @@ fn fetch(arena: Allocator, args: []const []const u8) !void { | ... | @@ -268,17 +85,17 @@ fn fetch(arena: Allocator, args: []const []const u8) !void { |
| 268 | | 85 | |
| 269 | const sysroot_path = sysroot orelse blk: { | 86 | const sysroot_path = sysroot orelse blk: { |
| 270 | const target = try std.zig.system.resolveTargetQuery(.{}); | 87 | const target = try std.zig.system.resolveTargetQuery(.{}); |
| 271 | break :blk std.zig.system.darwin.getSdk(arena, target) orelse | 88 | break :blk std.zig.system.darwin.getSdk(allocator, target) orelse |
| 272 | fatal("no SDK found; you can provide one explicitly with '--sysroot' flag", .{}); | 89 | fatal("no SDK found; you can provide one explicitly with '--sysroot' flag", .{}); |
| 273 | }; | 90 | }; |
| 274 | | 91 | |
| 275 | var sdk_dir = try std.fs.cwd().openDir(sysroot_path, .{}); | 92 | var sdk_dir = try std.fs.cwd().openDir(sysroot_path, .{}); |
| 276 | defer sdk_dir.close(); | 93 | defer sdk_dir.close(); |
| 277 | const sdk_info = try sdk_dir.readFileAlloc(arena, "SDKSettings.json", std.math.maxInt(u32)); | 94 | const sdk_info = try sdk_dir.readFileAlloc(allocator, "SDKSettings.json", std.math.maxInt(u32)); |
| 278 | | 95 | |
| 279 | const parsed_json = try std.json.parseFromSlice(struct { | 96 | const parsed_json = try std.json.parseFromSlice(struct { |
| 280 | DefaultProperties: struct { MACOSX_DEPLOYMENT_TARGET: []const u8 }, | 97 | DefaultProperties: struct { MACOSX_DEPLOYMENT_TARGET: []const u8 }, |
| 281 | }, arena, sdk_info, .{ .ignore_unknown_fields = true }); | 98 | }, allocator, sdk_info, .{ .ignore_unknown_fields = true }); |
| 282 | | 99 | |
| 283 | const version = Version.parse(parsed_json.value.DefaultProperties.MACOSX_DEPLOYMENT_TARGET) orelse | 100 | const version = Version.parse(parsed_json.value.DefaultProperties.MACOSX_DEPLOYMENT_TARGET) orelse |
| 284 | fatal("don't know how to parse SDK version: {s}", .{ | 101 | fatal("don't know how to parse SDK version: {s}", .{ |
| ... | @@ -303,7 +120,7 @@ fn fetch(arena: Allocator, args: []const []const u8) !void { | ... | @@ -303,7 +120,7 @@ fn fetch(arena: Allocator, args: []const []const u8) !void { |
| 303 | .arch = arch, | 120 | .arch = arch, |
| 304 | .os_ver = os_ver, | 121 | .os_ver = os_ver, |
| 305 | }; | 122 | }; |
| 306 | try fetchTarget(arena, argv.items, sysroot_path, target, version, tmp); | 123 | try fetchTarget(allocator, argv.items, sysroot_path, target, version, tmp); |
| 307 | } | 124 | } |
| 308 | } | 125 | } |
| 309 | | 126 | |
| ... | @@ -315,8 +132,8 @@ fn fetchTarget( | ... | @@ -315,8 +132,8 @@ fn fetchTarget( |
| 315 | ver: Version, | 132 | ver: Version, |
| 316 | tmp: std.testing.TmpDir, | 133 | tmp: std.testing.TmpDir, |
| 317 | ) !void { | 134 | ) !void { |
| 318 | const tmp_filename = "headers"; | 135 | const tmp_filename = "macos-headers"; |
| 319 | const headers_list_filename = "headers.o.d"; | 136 | const headers_list_filename = "macos-headers.o.d"; |
| 320 | const tmp_path = try tmp.dir.realpathAlloc(arena, "."); | 137 | const tmp_path = try tmp.dir.realpathAlloc(arena, "."); |
| 321 | const tmp_file_path = try fs.path.join(arena, &[_][]const u8{ tmp_path, tmp_filename }); | 138 | const tmp_file_path = try fs.path.join(arena, &[_][]const u8{ tmp_path, tmp_filename }); |
| 322 | const headers_list_path = try fs.path.join(arena, &[_][]const u8{ tmp_path, headers_list_filename }); | 139 | const headers_list_path = try fs.path.join(arena, &[_][]const u8{ tmp_path, headers_list_filename }); |
| ... | @@ -333,7 +150,6 @@ fn fetchTarget( | ... | @@ -333,7 +150,6 @@ fn fetchTarget( |
| 333 | switch (target.arch) { | 150 | switch (target.arch) { |
| 334 | .x86_64 => "x86_64", | 151 | .x86_64 => "x86_64", |
| 335 | .aarch64 => "arm64", | 152 | .aarch64 => "arm64", |
| 336 | else => unreachable, | | |
| 337 | }, | 153 | }, |
| 338 | macos_version, | 154 | macos_version, |
| 339 | "-isysroot", | 155 | "-isysroot", |
| ... | @@ -350,8 +166,6 @@ fn fetchTarget( | ... | @@ -350,8 +166,6 @@ fn fetchTarget( |
| 350 | }); | 166 | }); |
| 351 | try cc_argv.appendSlice(args); | 167 | try cc_argv.appendSlice(args); |
| 352 | | 168 | |
| 353 | // TODO instead of calling `cc` as a child process here, | | |
| 354 | // hook in directly to `zig cc` API. | | |
| 355 | const res = try std.process.Child.run(.{ | 169 | const res = try std.process.Child.run(.{ |
| 356 | .allocator = arena, | 170 | .allocator = arena, |
| 357 | .argv = cc_argv.items, | 171 | .argv = cc_argv.items, |
| ... | @@ -361,7 +175,7 @@ fn fetchTarget( | ... | @@ -361,7 +175,7 @@ fn fetchTarget( |
| 361 | std.log.err("{s}", .{res.stderr}); | 175 | std.log.err("{s}", .{res.stderr}); |
| 362 | } | 176 | } |
| 363 | | 177 | |
| 364 | // Read in the contents of `upgrade.o.d` | 178 | // Read in the contents of `macos-headers.o.d` |
| 365 | const headers_list_file = try tmp.dir.openFile(headers_list_filename, .{}); | 179 | const headers_list_file = try tmp.dir.openFile(headers_list_filename, .{}); |
| 366 | defer headers_list_file.close(); | 180 | defer headers_list_file.close(); |
| 367 | | 181 | |
| ... | @@ -413,295 +227,35 @@ fn fetchTarget( | ... | @@ -413,295 +227,35 @@ fn fetchTarget( |
| 413 | } | 227 | } |
| 414 | } | 228 | } |
| 415 | | 229 | |
| 416 | const dedup_usage = | 230 | const ArgsIterator = struct { |
| 417 | \\fetch_them_macos_headers dedup [path] | 231 | args: []const []const u8, |
| 418 | \\ | 232 | i: usize = 0, |
| 419 | \\General Options: | | |
| 420 | \\-h, --help Print this help and exit | | |
| 421 | ; | | |
| 422 | | | |
| 423 | /// Dedups libs headers assuming the following layered structure: | | |
| 424 | /// layer 1: x86_64-macos.10 x86_64-macos.11 x86_64-macos.12 aarch64-macos.11 aarch64-macos.12 | | |
| 425 | /// layer 2: any-macos.10 any-macos.11 any-macos.12 | | |
| 426 | /// layer 3: any-macos | | |
| 427 | /// | | |
| 428 | /// The first layer consists of headers specific to a CPU architecture AND macOS version. The second | | |
| 429 | /// layer consists of headers common to a macOS version across CPU architectures, and the final | | |
| 430 | /// layer consists of headers common to all libc headers. | | |
| 431 | fn dedup(arena: Allocator, args: []const []const u8) !void { | | |
| 432 | var path: ?[]const u8 = null; | | |
| 433 | var args_iter = ArgsIterator{ .args = args }; | | |
| 434 | while (args_iter.next()) |arg| { | | |
| 435 | if (mem.eql(u8, arg, "--help") or mem.eql(u8, arg, "-h")) { | | |
| 436 | return info(dedup_usage, .{}); | | |
| 437 | } else { | | |
| 438 | if (path != null) fatal("too many arguments", .{}); | | |
| 439 | path = arg; | | |
| 440 | } | | |
| 441 | } | | |
| 442 | | | |
| 443 | const dest_path = path orelse fatal("no destination path specified", .{}); | | |
| 444 | var dest_dir = fs.cwd().makeOpenPath(dest_path, .{}) catch |err| switch (err) { | | |
| 445 | error.NotDir => fatal("path '{s}' not a directory", .{dest_path}), | | |
| 446 | else => return err, | | |
| 447 | }; | | |
| 448 | defer dest_dir.close(); | | |
| 449 | | | |
| 450 | var dont_dedup_map = try generateDontDedupMap(arena); | | |
| 451 | var layer_2_targets = std.ArrayList(TargetWithPrefix).init(arena); | | |
| 452 | | | |
| 453 | for (&[_]OsVer{ .catalina, .big_sur, .monterey, .ventura, .sonoma, .sequoia }) |os_ver| { | | |
| 454 | var layer_1_targets = std.ArrayList(TargetWithPrefix).init(arena); | | |
| 455 | | | |
| 456 | for (targets) |target| { | | |
| 457 | if (target.os_ver != os_ver) continue; | | |
| 458 | try layer_1_targets.append(.{ | | |
| 459 | .prefix = headers_source_prefix, | | |
| 460 | .target = target, | | |
| 461 | }); | | |
| 462 | } | | |
| 463 | | | |
| 464 | if (layer_1_targets.items.len < 2) { | | |
| 465 | try layer_2_targets.appendSlice(layer_1_targets.items); | | |
| 466 | continue; | | |
| 467 | } | | |
| 468 | | | |
| 469 | const layer_2_target = try dedupDirs(arena, .{ | | |
| 470 | .os_ver = os_ver, | | |
| 471 | .dest_path = dest_path, | | |
| 472 | .dest_dir = dest_dir, | | |
| 473 | .targets = layer_1_targets.items, | | |
| 474 | .dont_dedup_map = &dont_dedup_map, | | |
| 475 | }); | | |
| 476 | try layer_2_targets.append(layer_2_target); | | |
| 477 | } | | |
| 478 | | | |
| 479 | const layer_3_target = try dedupDirs(arena, .{ | | |
| 480 | .os_ver = .any, | | |
| 481 | .dest_path = dest_path, | | |
| 482 | .dest_dir = dest_dir, | | |
| 483 | .targets = layer_2_targets.items, | | |
| 484 | .dont_dedup_map = &dont_dedup_map, | | |
| 485 | }); | | |
| 486 | assert(layer_3_target.target.eql(targets[0])); | | |
| 487 | } | | |
| 488 | | | |
| 489 | const TargetWithPrefix = struct { | | |
| 490 | prefix: []const u8, | | |
| 491 | target: Target, | | |
| 492 | }; | | |
| 493 | | | |
| 494 | const DedupDirsArgs = struct { | | |
| 495 | os_ver: OsVer, | | |
| 496 | dest_path: []const u8, | | |
| 497 | dest_dir: fs.Dir, | | |
| 498 | targets: []const TargetWithPrefix, | | |
| 499 | dont_dedup_map: *const std.StringHashMap(void), | | |
| 500 | }; | | |
| 501 | | | |
| 502 | fn dedupDirs(arena: Allocator, args: DedupDirsArgs) !TargetWithPrefix { | | |
| 503 | var tmp = tmpDir(.{ .iterate = true }); | | |
| 504 | defer tmp.cleanup(); | | |
| 505 | | | |
| 506 | var path_table = PathTable.init(arena); | | |
| 507 | var hash_to_contents = HashToContents.init(arena); | | |
| 508 | | | |
| 509 | var savings = FindResult{}; | | |
| 510 | for (args.targets) |target| { | | |
| 511 | const res = try findDuplicates(target.target, arena, target.prefix, &path_table, &hash_to_contents); | | |
| 512 | savings.max_bytes_saved += res.max_bytes_saved; | | |
| 513 | savings.total_bytes += res.total_bytes; | | |
| 514 | } | | |
| 515 | | | |
| 516 | info("summary: {} could be reduced to {}", .{ | | |
| 517 | std.fmt.fmtIntSizeBin(savings.total_bytes), | | |
| 518 | std.fmt.fmtIntSizeBin(savings.total_bytes - savings.max_bytes_saved), | | |
| 519 | }); | | |
| 520 | | 233 | |
| 521 | const output_target = Target{ | 234 | fn next(it: *@This()) ?[]const u8 { |
| 522 | .arch = .any, | 235 | if (it.i >= it.args.len) { |
| 523 | .abi = .any, | 236 | return null; |
| 524 | .os_ver = args.os_ver, | | |
| 525 | }; | | |
| 526 | const common_name = try output_target.fullName(arena); | | |
| 527 | | | |
| 528 | var missed_opportunity_bytes: usize = 0; | | |
| 529 | // Iterate path_table. For each path, put all the hashes into a list. Sort by hit_count. | | |
| 530 | // The hash with the highest hit_count gets to be the "generic" one. Everybody else | | |
| 531 | // gets their header in a separate arch directory. | | |
| 532 | var path_it = path_table.iterator(); | | |
| 533 | while (path_it.next()) |path_kv| { | | |
| 534 | if (!args.dont_dedup_map.contains(path_kv.key_ptr.*)) { | | |
| 535 | var contents_list = std.ArrayList(*Contents).init(arena); | | |
| 536 | { | | |
| 537 | var hash_it = path_kv.value_ptr.*.iterator(); | | |
| 538 | while (hash_it.next()) |hash_kv| { | | |
| 539 | const contents = &hash_to_contents.getEntry(hash_kv.value_ptr.*).?.value_ptr.*; | | |
| 540 | try contents_list.append(contents); | | |
| 541 | } | | |
| 542 | } | | |
| 543 | std.mem.sort(*Contents, contents_list.items, {}, Contents.hitCountLessThan); | | |
| 544 | const best_contents = contents_list.popOrNull().?; | | |
| 545 | if (best_contents.hit_count > 1) { | | |
| 546 | // Put it in `any-macos-none`. | | |
| 547 | const full_path = try fs.path.join(arena, &[_][]const u8{ common_name, path_kv.key_ptr.* }); | | |
| 548 | try tmp.dir.makePath(fs.path.dirname(full_path).?); | | |
| 549 | try tmp.dir.writeFile(.{ .sub_path = full_path, .data = best_contents.bytes }); | | |
| 550 | best_contents.is_generic = true; | | |
| 551 | while (contents_list.popOrNull()) |contender| { | | |
| 552 | if (contender.hit_count > 1) { | | |
| 553 | const this_missed_bytes = contender.hit_count * contender.bytes.len; | | |
| 554 | missed_opportunity_bytes += this_missed_bytes; | | |
| 555 | info("Missed opportunity ({}): {s}", .{ | | |
| 556 | std.fmt.fmtIntSizeBin(this_missed_bytes), | | |
| 557 | path_kv.key_ptr.*, | | |
| 558 | }); | | |
| 559 | } else break; | | |
| 560 | } | | |
| 561 | } | | |
| 562 | } | | |
| 563 | var hash_it = path_kv.value_ptr.*.iterator(); | | |
| 564 | while (hash_it.next()) |hash_kv| { | | |
| 565 | const contents = &hash_to_contents.getEntry(hash_kv.value_ptr.*).?.value_ptr.*; | | |
| 566 | if (contents.is_generic) continue; | | |
| 567 | | | |
| 568 | const target = hash_kv.key_ptr.*; | | |
| 569 | const target_name = try target.fullName(arena); | | |
| 570 | const full_path = try fs.path.join(arena, &[_][]const u8{ target_name, path_kv.key_ptr.* }); | | |
| 571 | try tmp.dir.makePath(fs.path.dirname(full_path).?); | | |
| 572 | try tmp.dir.writeFile(.{ .sub_path = full_path, .data = contents.bytes }); | | |
| 573 | } | 237 | } |
| | 238 | defer it.i += 1; |
| | 239 | return it.args[it.i]; |
| 574 | } | 240 | } |
| 575 | | 241 | |
| 576 | for (args.targets) |target| { | 242 | fn nextOrFatal(it: *@This()) []const u8 { |
| 577 | const target_name = try target.target.fullName(arena); | 243 | const arg = it.next() orelse fatal("expected parameter after '{s}'", .{it.args[it.i - 1]}); |
| 578 | try args.dest_dir.deleteTree(target_name); | 244 | return arg; |
| 579 | } | | |
| 580 | try args.dest_dir.deleteTree(common_name); | | |
| 581 | | | |
| 582 | var tmp_it = tmp.dir.iterate(); | | |
| 583 | while (try tmp_it.next()) |entry| { | | |
| 584 | switch (entry.kind) { | | |
| 585 | .directory => { | | |
| 586 | const sub_dir = try tmp.dir.openDir(entry.name, .{ .iterate = true }); | | |
| 587 | const dest_sub_dir = try args.dest_dir.makeOpenPath(entry.name, .{}); | | |
| 588 | try copyDirAll(sub_dir, dest_sub_dir); | | |
| 589 | }, | | |
| 590 | else => info("unexpected file format: not a directory: '{s}'", .{entry.name}), | | |
| 591 | } | | |
| 592 | } | 245 | } |
| 593 | | | |
| 594 | return TargetWithPrefix{ | | |
| 595 | .prefix = args.dest_path, | | |
| 596 | .target = output_target, | | |
| 597 | }; | | |
| 598 | } | | |
| 599 | | | |
| 600 | const FindResult = struct { | | |
| 601 | max_bytes_saved: usize = 0, | | |
| 602 | total_bytes: usize = 0, | | |
| 603 | }; | 246 | }; |
| 604 | | 247 | |
| 605 | fn findDuplicates( | 248 | fn info(comptime format: []const u8, args: anytype) void { |
| 606 | target: Target, | 249 | const msg = std.fmt.allocPrint(gpa, "info: " ++ format ++ "\n", args) catch return; |
| 607 | arena: Allocator, | 250 | std.io.getStdOut().writeAll(msg) catch {}; |
| 608 | dest_path: []const u8, | | |
| 609 | path_table: *PathTable, | | |
| 610 | hash_to_contents: *HashToContents, | | |
| 611 | ) !FindResult { | | |
| 612 | var result = FindResult{}; | | |
| 613 | | | |
| 614 | const target_name = try target.fullName(arena); | | |
| 615 | const target_include_dir = try fs.path.join(arena, &[_][]const u8{ dest_path, target_name }); | | |
| 616 | var dir_stack = std.ArrayList([]const u8).init(arena); | | |
| 617 | try dir_stack.append(target_include_dir); | | |
| 618 | | | |
| 619 | while (dir_stack.popOrNull()) |full_dir_name| { | | |
| 620 | var dir = fs.cwd().openDir(full_dir_name, .{ .iterate = true }) catch |err| switch (err) { | | |
| 621 | error.FileNotFound => break, | | |
| 622 | error.AccessDenied => break, | | |
| 623 | else => return err, | | |
| 624 | }; | | |
| 625 | defer dir.close(); | | |
| 626 | | | |
| 627 | var dir_it = dir.iterate(); | | |
| 628 | | | |
| 629 | while (try dir_it.next()) |entry| { | | |
| 630 | const full_path = try fs.path.join(arena, &[_][]const u8{ full_dir_name, entry.name }); | | |
| 631 | switch (entry.kind) { | | |
| 632 | .directory => try dir_stack.append(full_path), | | |
| 633 | .file => { | | |
| 634 | const rel_path = try fs.path.relative(arena, target_include_dir, full_path); | | |
| 635 | const max_size = 2 * 1024 * 1024 * 1024; | | |
| 636 | const raw_bytes = try fs.cwd().readFileAlloc(arena, full_path, max_size); | | |
| 637 | const trimmed = mem.trim(u8, raw_bytes, " \r\n\t"); | | |
| 638 | result.total_bytes += raw_bytes.len; | | |
| 639 | const hash = try arena.alloc(u8, 32); | | |
| 640 | var hasher = Blake3.init(.{}); | | |
| 641 | hasher.update(rel_path); | | |
| 642 | hasher.update(trimmed); | | |
| 643 | hasher.final(hash); | | |
| 644 | const gop = try hash_to_contents.getOrPut(hash); | | |
| 645 | if (gop.found_existing) { | | |
| 646 | result.max_bytes_saved += raw_bytes.len; | | |
| 647 | gop.value_ptr.hit_count += 1; | | |
| 648 | info("duplicate: {s} {s} ({})", .{ | | |
| 649 | target_name, | | |
| 650 | rel_path, | | |
| 651 | std.fmt.fmtIntSizeBin(raw_bytes.len), | | |
| 652 | }); | | |
| 653 | } else { | | |
| 654 | gop.value_ptr.* = Contents{ | | |
| 655 | .bytes = trimmed, | | |
| 656 | .hit_count = 1, | | |
| 657 | .hash = hash, | | |
| 658 | .is_generic = false, | | |
| 659 | }; | | |
| 660 | } | | |
| 661 | const path_gop = try path_table.getOrPut(rel_path); | | |
| 662 | const target_to_hash = if (path_gop.found_existing) path_gop.value_ptr.* else blk: { | | |
| 663 | const ptr = try arena.create(TargetToHash); | | |
| 664 | ptr.* = TargetToHash.init(arena); | | |
| 665 | path_gop.value_ptr.* = ptr; | | |
| 666 | break :blk ptr; | | |
| 667 | }; | | |
| 668 | try target_to_hash.putNoClobber(target, hash); | | |
| 669 | }, | | |
| 670 | else => info("unexpected file: {s}", .{full_path}), | | |
| 671 | } | | |
| 672 | } | | |
| 673 | } | | |
| 674 | | | |
| 675 | return result; | | |
| 676 | } | 251 | } |
| 677 | | 252 | |
| 678 | fn copyDirAll(source: fs.Dir, dest: fs.Dir) anyerror!void { | 253 | fn fatal(comptime format: []const u8, args: anytype) noreturn { |
| 679 | var it = source.iterate(); | 254 | ret: { |
| 680 | while (try it.next()) |next| { | 255 | const msg = std.fmt.allocPrint(gpa, "fatal: " ++ format ++ "\n", args) catch break :ret; |
| 681 | switch (next.kind) { | 256 | std.io.getStdErr().writeAll(msg) catch {}; |
| 682 | .directory => { | | |
| 683 | var sub_dir = try dest.makeOpenPath(next.name, .{}); | | |
| 684 | var sub_source = try source.openDir(next.name, .{ .iterate = true }); | | |
| 685 | defer { | | |
| 686 | sub_dir.close(); | | |
| 687 | sub_source.close(); | | |
| 688 | } | | |
| 689 | try copyDirAll(sub_source, sub_dir); | | |
| 690 | }, | | |
| 691 | .file => { | | |
| 692 | var source_file = try source.openFile(next.name, .{}); | | |
| 693 | var dest_file = try dest.createFile(next.name, .{}); | | |
| 694 | defer { | | |
| 695 | source_file.close(); | | |
| 696 | dest_file.close(); | | |
| 697 | } | | |
| 698 | const stat = try source_file.stat(); | | |
| 699 | const ncopied = try source_file.copyRangeAll(0, dest_file, 0, stat.size); | | |
| 700 | assert(ncopied == stat.size); | | |
| 701 | }, | | |
| 702 | else => |kind| info("unexpected file kind '{s}' will be ignored", .{@tagName(kind)}), | | |
| 703 | } | | |
| 704 | } | 257 | } |
| | 258 | std.process.exit(1); |
| 705 | } | 259 | } |
| 706 | | 260 | |
| 707 | const Version = struct { | 261 | const Version = struct { |
Alex Rønne Petersen <alex@alexrp.com> 2024-12-23 04:44:19+01:00
GitHub <noreply@github.com> 2024-12-23 04:44:19+01:00
Signed by PGP key B5690EEEBB952194