authorgravatar for stephen@hexops.comStephen Gutekanst <stephen@hexops.com> 2022-02-20 11:58:58-07:00
committergravatar for stephen@hexops.comStephen Gutekanst <stephen@hexops.com> 2022-02-20 14:44:39-07:00
log2bd10f4db98fd313c6743477ad36b139652a8dd5
tree9c935031ca3027b48e763a63b7fb7cd7b6f582b6
parente41d4df93cf78b7ecccd959472dc653dc089f8aa

std: have Builder use response files if zig build args exceed OS limits

In Mach engine we're seeing command line arguments to `zig build-lib` exceed the 32 KiB limit that Windows imposes, due to the number of sources and compiler flags we must pass in order to build gpu-dawn. This change fixes the issue by having `Builder` check if the arguments to a `zig build-*` command are >30 KiB and, if so, writes the arguments to a file `zig-cache/args/<SHA2 of args>`. Then the command invocation merely becomes `zig build-lib @<that file>`. Fixes #10693 Fixes hexops/mach#167 Signed-off-by: Stephen Gutekanst <stephen@hexops.com>

1 files changed, 43 insertions(+), 0 deletions(-)

lib/std/build.zig+43
......@@ -17,6 +17,7 @@ const fmt_lib = std.fmt;
1717const File = std.fs.File;
1818const CrossTarget = std.zig.CrossTarget;
1919const NativeTargetInfo = std.zig.system.NativeTargetInfo;
20const Sha256 = std.crypto.hash.sha2.Sha256;
2021
2122pub const FmtStep = @import("build/FmtStep.zig");
2223pub const TranslateCStep = @import("build/TranslateCStep.zig");
......@@ -2888,6 +2889,48 @@ pub const LibExeObjStep = struct {
28882889
28892890 try zig_args.append("--enable-cache");
28902891
2892 const is_build = switch (self.kind) {
2893 .lib => true,
2894 .exe => true,
2895 .obj => true,
2896 else => false,
2897 };
2898 if (is_build) {
2899 // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux
2900 // 2,097,152. If our args exceed 30 KiB, we instead write them to a "response file" and
2901 // pass that to zig, e.g. via 'zig build-lib @args.rsp'
2902 var args_length: usize = 0;
2903 for (zig_args.items) |arg| {
2904 args_length += arg.len;
2905 }
2906 if (args_length >= 30 * 1024) {
2907 const args_dir = try fs.path.join(
2908 builder.allocator,
2909 &[_][]const u8{ builder.pathFromRoot("zig-cache"), "args" },
2910 );
2911 try std.fs.cwd().makePath(args_dir);
2912
2913 // Write the args to zig-cache/args/<SHA256 hash of args> to avoid conflicts with
2914 // other zig build commands running in parallel.
2915 const partially_quoted = try std.mem.join(builder.allocator, "\" \"", zig_args.items[2..]);
2916 const args = try std.mem.concat(builder.allocator, u8, &[_][]const u8{ "\"", partially_quoted, "\"" });
2917
2918 var args_hash: [Sha256.digest_length]u8 = undefined;
2919 Sha256.hash(args, &args_hash, .{});
2920 var args_hex_hash: [Sha256.digest_length * 2]u8 = undefined;
2921 _ = try std.fmt.bufPrint(
2922 &args_hex_hash,
2923 "{s}",
2924 .{std.fmt.fmtSliceHexLower(&args_hash)},
2925 );
2926
2927 const args_file = try fs.path.join(builder.allocator, &[_][]const u8{ args_dir, args_hex_hash[0..] });
2928 try std.fs.cwd().writeFile(args_file, args);
2929
2930 zig_args.shrinkRetainingCapacity(2);
2931 try zig_args.append(try std.mem.concat(builder.allocator, u8, &[_][]const u8{ "@", args_file }));
2932 }
2933 }
28912934 const output_dir_nl = try builder.execFromStep(zig_args.items, &self.step);
28922935 const build_output_dir = mem.trimRight(u8, output_dir_nl, "\r\n");
28932936