authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-02-09 10:01:01-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-02-13 06:42:26-07:00
logd97042ad2e41b173334ec542eb4b07e81864d10e
treebf7f000a33e84ad37190b69963ac187d785d1ccd
parent066632261492ee7624117ad09269f57526aca4c0

std.Build: start using the cache system with RunStep

* Use std.Build.Cache.Directory instead of a string for storing the cache roots and build roots. * Set up a std.Build.Cache in build_runner.zig and use it in std.Build.RunStep for avoiding redundant work.

9 files changed, 198 insertions(+), 117 deletions(-)

build.zig+3-6
...@@ -40,11 +40,8 @@ pub fn build(b: *std.Build) !void {...@@ -40,11 +40,8 @@ pub fn build(b: *std.Build) !void {
40 });40 });
41 docgen_exe.single_threaded = single_threaded;41 docgen_exe.single_threaded = single_threaded;
4242
43 const rel_zig_exe = try fs.path.relative(b.allocator, b.build_root, b.zig_exe);43 const rel_zig_exe = try b.build_root.join(b.allocator, &.{b.zig_exe});
44 const langref_out_path = fs.path.join(44 const langref_out_path = try b.cache_root.join(b.allocator, &.{"langref.html"});
45 b.allocator,
46 &[_][]const u8{ b.cache_root, "langref.html" },
47 ) catch unreachable;
48 const docgen_cmd = docgen_exe.run();45 const docgen_cmd = docgen_exe.run();
49 docgen_cmd.addArgs(&[_][]const u8{46 docgen_cmd.addArgs(&[_][]const u8{
50 "--zig",47 "--zig",
...@@ -215,7 +212,7 @@ pub fn build(b: *std.Build) !void {...@@ -215,7 +212,7 @@ pub fn build(b: *std.Build) !void {
215212
216 var code: u8 = undefined;213 var code: u8 = undefined;
217 const git_describe_untrimmed = b.execAllowFail(&[_][]const u8{214 const git_describe_untrimmed = b.execAllowFail(&[_][]const u8{
218 "git", "-C", b.build_root, "describe", "--match", "*.*.*", "--tags",215 "git", "-C", b.build_root.path orelse ".", "describe", "--match", "*.*.*", "--tags",
219 }, &code, .Ignore) catch {216 }, &code, .Ignore) catch {
220 break :v version_string;217 break :v version_string;
221 };218 };
lib/build_runner.zig+31-4
...@@ -43,13 +43,40 @@ pub fn main() !void {...@@ -43,13 +43,40 @@ pub fn main() !void {
4343
44 const host = try std.zig.system.NativeTargetInfo.detect(.{});44 const host = try std.zig.system.NativeTargetInfo.detect(.{});
4545
46 const build_root_directory: std.Build.Cache.Directory = .{
47 .path = build_root,
48 .handle = try std.fs.cwd().openDir(build_root, .{}),
49 };
50
51 const local_cache_directory: std.Build.Cache.Directory = .{
52 .path = try std.fs.path.relative(allocator, build_root, cache_root),
53 .handle = try std.fs.cwd().makeOpenPath(cache_root, .{}),
54 };
55
56 const global_cache_directory: std.Build.Cache.Directory = .{
57 .path = try std.fs.path.relative(allocator, build_root, global_cache_root),
58 .handle = try std.fs.cwd().makeOpenPath(global_cache_root, .{}),
59 };
60
61 var cache: std.Build.Cache = .{
62 .gpa = allocator,
63 .manifest_dir = try local_cache_directory.handle.makeOpenPath("h", .{}),
64 };
65 cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() });
66 cache.addPrefix(build_root_directory);
67 cache.addPrefix(local_cache_directory);
68 cache.addPrefix(global_cache_directory);
69
70 //cache.hash.addBytes(builtin.zig_version);
71
46 const builder = try std.Build.create(72 const builder = try std.Build.create(
47 allocator,73 allocator,
48 zig_exe,74 zig_exe,
49 build_root,75 build_root_directory,
50 cache_root,76 local_cache_directory,
51 global_cache_root,77 global_cache_directory,
52 host,78 host,
79 &cache,
53 );80 );
54 defer builder.destroy();81 defer builder.destroy();
5582
...@@ -138,7 +165,7 @@ pub fn main() !void {...@@ -138,7 +165,7 @@ pub fn main() !void {
138 return usageAndErr(builder, false, stderr_stream);165 return usageAndErr(builder, false, stderr_stream);
139 };166 };
140 } else if (mem.eql(u8, arg, "--zig-lib-dir")) {167 } else if (mem.eql(u8, arg, "--zig-lib-dir")) {
141 builder.override_lib_dir = nextArg(args, &arg_idx) orelse {168 builder.zig_lib_dir = nextArg(args, &arg_idx) orelse {
142 std.debug.print("Expected argument after --zig-lib-dir\n\n", .{});169 std.debug.print("Expected argument after --zig-lib-dir\n\n", .{});
143 return usageAndErr(builder, false, stderr_stream);170 return usageAndErr(builder, false, stderr_stream);
144 };171 };
lib/std/Build.zig+31-27
...@@ -79,11 +79,12 @@ search_prefixes: ArrayList([]const u8),...@@ -79,11 +79,12 @@ search_prefixes: ArrayList([]const u8),
79libc_file: ?[]const u8 = null,79libc_file: ?[]const u8 = null,
80installed_files: ArrayList(InstalledFile),80installed_files: ArrayList(InstalledFile),
81/// Path to the directory containing build.zig.81/// Path to the directory containing build.zig.
82build_root: []const u8,82build_root: Cache.Directory,
83cache_root: []const u8,83cache_root: Cache.Directory,
84global_cache_root: []const u8,84global_cache_root: Cache.Directory,
85/// zig lib dir85cache: *Cache,
86override_lib_dir: ?[]const u8,86/// If non-null, overrides the default zig lib dir.
87zig_lib_dir: ?[]const u8,
87vcpkg_root: VcpkgRoot = .unattempted,88vcpkg_root: VcpkgRoot = .unattempted,
88pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null,89pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null,
89args: ?[][]const u8 = null,90args: ?[][]const u8 = null,
...@@ -187,10 +188,11 @@ pub const DirList = struct {...@@ -187,10 +188,11 @@ pub const DirList = struct {
187pub fn create(188pub fn create(
188 allocator: Allocator,189 allocator: Allocator,
189 zig_exe: []const u8,190 zig_exe: []const u8,
190 build_root: []const u8,191 build_root: Cache.Directory,
191 cache_root: []const u8,192 cache_root: Cache.Directory,
192 global_cache_root: []const u8,193 global_cache_root: Cache.Directory,
193 host: NativeTargetInfo,194 host: NativeTargetInfo,
195 cache: *Cache,
194) !*Build {196) !*Build {
195 const env_map = try allocator.create(EnvMap);197 const env_map = try allocator.create(EnvMap);
196 env_map.* = try process.getEnvMap(allocator);198 env_map.* = try process.getEnvMap(allocator);
...@@ -199,8 +201,9 @@ pub fn create(...@@ -199,8 +201,9 @@ pub fn create(
199 self.* = Build{201 self.* = Build{
200 .zig_exe = zig_exe,202 .zig_exe = zig_exe,
201 .build_root = build_root,203 .build_root = build_root,
202 .cache_root = try fs.path.relative(allocator, build_root, cache_root),204 .cache_root = cache_root,
203 .global_cache_root = global_cache_root,205 .global_cache_root = global_cache_root,
206 .cache = cache,
204 .verbose = false,207 .verbose = false,
205 .verbose_link = false,208 .verbose_link = false,
206 .verbose_cc = false,209 .verbose_cc = false,
...@@ -232,7 +235,7 @@ pub fn create(...@@ -232,7 +235,7 @@ pub fn create(
232 .step = Step.init(.top_level, "uninstall", allocator, makeUninstall),235 .step = Step.init(.top_level, "uninstall", allocator, makeUninstall),
233 .description = "Remove build artifacts from prefix path",236 .description = "Remove build artifacts from prefix path",
234 },237 },
235 .override_lib_dir = null,238 .zig_lib_dir = null,
236 .install_path = undefined,239 .install_path = undefined,
237 .args = null,240 .args = null,
238 .host = host,241 .host = host,
...@@ -247,7 +250,7 @@ pub fn create(...@@ -247,7 +250,7 @@ pub fn create(
247fn createChild(250fn createChild(
248 parent: *Build,251 parent: *Build,
249 dep_name: []const u8,252 dep_name: []const u8,
250 build_root: []const u8,253 build_root: Cache.Directory,
251 args: anytype,254 args: anytype,
252) !*Build {255) !*Build {
253 const child = try createChildOnly(parent, dep_name, build_root);256 const child = try createChildOnly(parent, dep_name, build_root);
...@@ -255,7 +258,7 @@ fn createChild(...@@ -255,7 +258,7 @@ fn createChild(
255 return child;258 return child;
256}259}
257260
258fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: []const u8) !*Build {261fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Directory) !*Build {
259 const allocator = parent.allocator;262 const allocator = parent.allocator;
260 const child = try allocator.create(Build);263 const child = try allocator.create(Build);
261 child.* = .{264 child.* = .{
...@@ -299,7 +302,8 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: []const u8)...@@ -299,7 +302,8 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: []const u8)
299 .build_root = build_root,302 .build_root = build_root,
300 .cache_root = parent.cache_root,303 .cache_root = parent.cache_root,
301 .global_cache_root = parent.global_cache_root,304 .global_cache_root = parent.global_cache_root,
302 .override_lib_dir = parent.override_lib_dir,305 .cache = parent.cache,
306 .zig_lib_dir = parent.zig_lib_dir,
303 .debug_log_scopes = parent.debug_log_scopes,307 .debug_log_scopes = parent.debug_log_scopes,
304 .debug_compile_errors = parent.debug_compile_errors,308 .debug_compile_errors = parent.debug_compile_errors,
305 .enable_darling = parent.enable_darling,309 .enable_darling = parent.enable_darling,
...@@ -381,7 +385,7 @@ fn applyArgs(b: *Build, args: anytype) !void {...@@ -381,7 +385,7 @@ fn applyArgs(b: *Build, args: anytype) !void {
381 _ = std.fmt.bufPrint(&hash_basename, "{s}", .{std.fmt.fmtSliceHexLower(&digest)}) catch385 _ = std.fmt.bufPrint(&hash_basename, "{s}", .{std.fmt.fmtSliceHexLower(&digest)}) catch
382 unreachable;386 unreachable;
383387
384 const install_prefix = b.pathJoin(&.{ b.cache_root, "i", &hash_basename });388 const install_prefix = try b.cache_root.join(b.allocator, &.{ "i", &hash_basename });
385 b.resolveInstallPrefix(install_prefix, .{});389 b.resolveInstallPrefix(install_prefix, .{});
386}390}
387391
...@@ -398,7 +402,7 @@ pub fn resolveInstallPrefix(self: *Build, install_prefix: ?[]const u8, dir_list:...@@ -398,7 +402,7 @@ pub fn resolveInstallPrefix(self: *Build, install_prefix: ?[]const u8, dir_list:
398 self.install_path = self.pathJoin(&.{ dest_dir, self.install_prefix });402 self.install_path = self.pathJoin(&.{ dest_dir, self.install_prefix });
399 } else {403 } else {
400 self.install_prefix = install_prefix orelse404 self.install_prefix = install_prefix orelse
401 (self.pathJoin(&.{ self.build_root, "zig-out" }));405 (self.build_root.join(self.allocator, &.{"zig-out"}) catch @panic("unhandled error"));
402 self.install_path = self.install_prefix;406 self.install_path = self.install_prefix;
403 }407 }
404408
...@@ -698,8 +702,6 @@ pub fn addTranslateC(self: *Build, options: TranslateCStep.Options) *TranslateCS...@@ -698,8 +702,6 @@ pub fn addTranslateC(self: *Build, options: TranslateCStep.Options) *TranslateCS
698}702}
699703
700pub fn make(self: *Build, step_names: []const []const u8) !void {704pub fn make(self: *Build, step_names: []const []const u8) !void {
701 try self.makePath(self.cache_root);
702
703 var wanted_steps = ArrayList(*Step).init(self.allocator);705 var wanted_steps = ArrayList(*Step).init(self.allocator);
704 defer wanted_steps.deinit();706 defer wanted_steps.deinit();
705707
...@@ -1225,13 +1227,6 @@ pub fn spawnChildEnvMap(self: *Build, cwd: ?[]const u8, env_map: *const EnvMap,...@@ -1225,13 +1227,6 @@ pub fn spawnChildEnvMap(self: *Build, cwd: ?[]const u8, env_map: *const EnvMap,
1225 }1227 }
1226}1228}
12271229
1228pub fn makePath(self: *Build, path: []const u8) !void {
1229 fs.cwd().makePath(self.pathFromRoot(path)) catch |err| {
1230 log.err("Unable to create path {s}: {s}", .{ path, @errorName(err) });
1231 return err;
1232 };
1233}
1234
1235pub fn installArtifact(self: *Build, artifact: *CompileStep) void {1230pub fn installArtifact(self: *Build, artifact: *CompileStep) void {
1236 self.getInstallStep().dependOn(&self.addInstallArtifact(artifact).step);1231 self.getInstallStep().dependOn(&self.addInstallArtifact(artifact).step);
1237}1232}
...@@ -1346,8 +1341,8 @@ pub fn truncateFile(self: *Build, dest_path: []const u8) !void {...@@ -1346,8 +1341,8 @@ pub fn truncateFile(self: *Build, dest_path: []const u8) !void {
1346 src_file.close();1341 src_file.close();
1347}1342}
13481343
1349pub fn pathFromRoot(self: *Build, rel_path: []const u8) []u8 {1344pub fn pathFromRoot(b: *Build, p: []const u8) []u8 {
1350 return fs.path.resolve(self.allocator, &[_][]const u8{ self.build_root, rel_path }) catch @panic("OOM");1345 return fs.path.resolve(b.allocator, &.{ b.build_root.path orelse ".", p }) catch @panic("OOM");
1351}1346}
13521347
1353pub fn pathJoin(self: *Build, paths: []const []const u8) []u8 {1348pub fn pathJoin(self: *Build, paths: []const []const u8) []u8 {
...@@ -1568,10 +1563,19 @@ pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {...@@ -1568,10 +1563,19 @@ pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {
1568fn dependencyInner(1563fn dependencyInner(
1569 b: *Build,1564 b: *Build,
1570 name: []const u8,1565 name: []const u8,
1571 build_root: []const u8,1566 build_root_string: []const u8,
1572 comptime build_zig: type,1567 comptime build_zig: type,
1573 args: anytype,1568 args: anytype,
1574) *Dependency {1569) *Dependency {
1570 const build_root: std.Build.Cache.Directory = .{
1571 .path = build_root_string,
1572 .handle = std.fs.cwd().openDir(build_root_string, .{}) catch |err| {
1573 std.debug.print("unable to open '{s}': {s}\n", .{
1574 build_root_string, @errorName(err),
1575 });
1576 std.process.exit(1);
1577 },
1578 };
1575 const sub_builder = b.createChild(name, build_root, args) catch @panic("unhandled error");1579 const sub_builder = b.createChild(name, build_root, args) catch @panic("unhandled error");
1576 sub_builder.runBuild(build_zig) catch @panic("unhandled error");1580 sub_builder.runBuild(build_zig) catch @panic("unhandled error");
15771581
lib/std/Build/CompileStep.zig+19-22
...@@ -83,7 +83,7 @@ max_memory: ?u64 = null,...@@ -83,7 +83,7 @@ max_memory: ?u64 = null,
83shared_memory: bool = false,83shared_memory: bool = false,
84global_base: ?u64 = null,84global_base: ?u64 = null,
85c_std: std.Build.CStd,85c_std: std.Build.CStd,
86override_lib_dir: ?[]const u8,86zig_lib_dir: ?[]const u8,
87main_pkg_path: ?[]const u8,87main_pkg_path: ?[]const u8,
88exec_cmd_args: ?[]const ?[]const u8,88exec_cmd_args: ?[]const ?[]const u8,
89name_prefix: []const u8,89name_prefix: []const u8,
...@@ -344,7 +344,7 @@ pub fn create(builder: *std.Build, options: Options) *CompileStep {...@@ -344,7 +344,7 @@ pub fn create(builder: *std.Build, options: Options) *CompileStep {
344 .installed_headers = ArrayList(*Step).init(builder.allocator),344 .installed_headers = ArrayList(*Step).init(builder.allocator),
345 .object_src = undefined,345 .object_src = undefined,
346 .c_std = std.Build.CStd.C99,346 .c_std = std.Build.CStd.C99,
347 .override_lib_dir = null,347 .zig_lib_dir = null,
348 .main_pkg_path = null,348 .main_pkg_path = null,
349 .exec_cmd_args = null,349 .exec_cmd_args = null,
350 .name_prefix = "",350 .name_prefix = "",
...@@ -857,7 +857,7 @@ pub fn setVerboseCC(self: *CompileStep, value: bool) void {...@@ -857,7 +857,7 @@ pub fn setVerboseCC(self: *CompileStep, value: bool) void {
857}857}
858858
859pub fn overrideZigLibDir(self: *CompileStep, dir_path: []const u8) void {859pub fn overrideZigLibDir(self: *CompileStep, dir_path: []const u8) void {
860 self.override_lib_dir = self.builder.dupePath(dir_path);860 self.zig_lib_dir = self.builder.dupePath(dir_path);
861}861}
862862
863pub fn setMainPkgPath(self: *CompileStep, dir_path: []const u8) void {863pub fn setMainPkgPath(self: *CompileStep, dir_path: []const u8) void {
...@@ -1350,10 +1350,10 @@ fn make(step: *Step) !void {...@@ -1350,10 +1350,10 @@ fn make(step: *Step) !void {
1350 }1350 }
13511351
1352 try zig_args.append("--cache-dir");1352 try zig_args.append("--cache-dir");
1353 try zig_args.append(builder.pathFromRoot(builder.cache_root));1353 try zig_args.append(builder.pathFromRoot(builder.cache_root.path orelse "."));
13541354
1355 try zig_args.append("--global-cache-dir");1355 try zig_args.append("--global-cache-dir");
1356 try zig_args.append(builder.pathFromRoot(builder.global_cache_root));1356 try zig_args.append(builder.pathFromRoot(builder.global_cache_root.path orelse "."));
13571357
1358 try zig_args.append("--name");1358 try zig_args.append("--name");
1359 try zig_args.append(self.name);1359 try zig_args.append(self.name);
...@@ -1703,12 +1703,12 @@ fn make(step: *Step) !void {...@@ -1703,12 +1703,12 @@ fn make(step: *Step) !void {
1703 try addFlag(&zig_args, "each-lib-rpath", self.each_lib_rpath);1703 try addFlag(&zig_args, "each-lib-rpath", self.each_lib_rpath);
1704 try addFlag(&zig_args, "build-id", self.build_id);1704 try addFlag(&zig_args, "build-id", self.build_id);
17051705
1706 if (self.override_lib_dir) |dir| {1706 if (self.zig_lib_dir) |dir| {
1707 try zig_args.append("--zig-lib-dir");1707 try zig_args.append("--zig-lib-dir");
1708 try zig_args.append(builder.pathFromRoot(dir));1708 try zig_args.append(builder.pathFromRoot(dir));
1709 } else if (builder.override_lib_dir) |dir| {1709 } else if (builder.zig_lib_dir) |dir| {
1710 try zig_args.append("--zig-lib-dir");1710 try zig_args.append("--zig-lib-dir");
1711 try zig_args.append(builder.pathFromRoot(dir));1711 try zig_args.append(dir);
1712 }1712 }
17131713
1714 if (self.main_pkg_path) |dir| {1714 if (self.main_pkg_path) |dir| {
...@@ -1745,23 +1745,15 @@ fn make(step: *Step) !void {...@@ -1745,23 +1745,15 @@ fn make(step: *Step) !void {
1745 args_length += arg.len + 1; // +1 to account for null terminator1745 args_length += arg.len + 1; // +1 to account for null terminator
1746 }1746 }
1747 if (args_length >= 30 * 1024) {1747 if (args_length >= 30 * 1024) {
1748 const args_dir = try fs.path.join(1748 try builder.cache_root.handle.makePath("args");
1749 builder.allocator,
1750 &[_][]const u8{ builder.pathFromRoot("zig-cache"), "args" },
1751 );
1752 try std.fs.cwd().makePath(args_dir);
1753
1754 var args_arena = std.heap.ArenaAllocator.init(builder.allocator);
1755 defer args_arena.deinit();
17561749
1757 const args_to_escape = zig_args.items[2..];1750 const args_to_escape = zig_args.items[2..];
1758 var escaped_args = try ArrayList([]const u8).initCapacity(args_arena.allocator(), args_to_escape.len);1751 var escaped_args = try ArrayList([]const u8).initCapacity(builder.allocator, args_to_escape.len);
1759
1760 arg_blk: for (args_to_escape) |arg| {1752 arg_blk: for (args_to_escape) |arg| {
1761 for (arg) |c, arg_idx| {1753 for (arg) |c, arg_idx| {
1762 if (c == '\\' or c == '"') {1754 if (c == '\\' or c == '"') {
1763 // Slow path for arguments that need to be escaped. We'll need to allocate and copy1755 // Slow path for arguments that need to be escaped. We'll need to allocate and copy
1764 var escaped = try ArrayList(u8).initCapacity(args_arena.allocator(), arg.len + 1);1756 var escaped = try ArrayList(u8).initCapacity(builder.allocator, arg.len + 1);
1765 const writer = escaped.writer();1757 const writer = escaped.writer();
1766 try writer.writeAll(arg[0..arg_idx]);1758 try writer.writeAll(arg[0..arg_idx]);
1767 for (arg[arg_idx..]) |to_escape| {1759 for (arg[arg_idx..]) |to_escape| {
...@@ -1789,11 +1781,16 @@ fn make(step: *Step) !void {...@@ -1789,11 +1781,16 @@ fn make(step: *Step) !void {
1789 .{std.fmt.fmtSliceHexLower(&args_hash)},1781 .{std.fmt.fmtSliceHexLower(&args_hash)},
1790 );1782 );
17911783
1792 const args_file = try fs.path.join(builder.allocator, &[_][]const u8{ args_dir, args_hex_hash[0..] });1784 const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash;
1793 try std.fs.cwd().writeFile(args_file, args);1785 try builder.cache_root.handle.writeFile(args_file, args);
1786
1787 const resolved_args_file = try mem.concat(builder.allocator, u8, &.{
1788 "@",
1789 builder.pathFromRoot(try builder.cache_root.join(builder.allocator, &.{args_file})),
1790 });
17941791
1795 zig_args.shrinkRetainingCapacity(2);1792 zig_args.shrinkRetainingCapacity(2);
1796 try zig_args.append(try std.mem.concat(builder.allocator, u8, &[_][]const u8{ "@", args_file }));1793 try zig_args.append(resolved_args_file);
1797 }1794 }
17981795
1799 const output_dir_nl = try builder.execFromStep(zig_args.items, &self.step);1796 const output_dir_nl = try builder.execFromStep(zig_args.items, &self.step);
lib/std/Build/ConfigHeaderStep.zig+1-3
...@@ -208,9 +208,7 @@ fn make(step: *Step) !void {...@@ -208,9 +208,7 @@ fn make(step: *Step) !void {
208 .{std.fmt.fmtSliceHexLower(&digest)},208 .{std.fmt.fmtSliceHexLower(&digest)},
209 ) catch unreachable;209 ) catch unreachable;
210210
211 const output_dir = try std.fs.path.join(gpa, &[_][]const u8{211 const output_dir = try self.builder.cache_root.join(gpa, &.{ "o", &hash_basename });
212 self.builder.cache_root, "o", &hash_basename,
213 });
214212
215 // If output_path has directory parts, deal with them. Example:213 // If output_path has directory parts, deal with them. Example:
216 // output_dir is zig-cache/o/HASH214 // output_dir is zig-cache/o/HASH
lib/std/Build/OptionsStep.zig+8-14
...@@ -234,26 +234,20 @@ fn make(step: *Step) !void {...@@ -234,26 +234,20 @@ fn make(step: *Step) !void {
234 );234 );
235 }235 }
236236
237 const options_directory = self.builder.pathFromRoot(237 var options_dir = try self.builder.cache_root.handle.makeOpenPath("options", .{});
238 try fs.path.join(238 defer options_dir.close();
239 self.builder.allocator,
240 &[_][]const u8{ self.builder.cache_root, "options" },
241 ),
242 );
243
244 try fs.cwd().makePath(options_directory);
245239
246 const options_file = try fs.path.join(240 const basename = self.hashContentsToFileName();
247 self.builder.allocator,
248 &[_][]const u8{ options_directory, &self.hashContentsToFileName() },
249 );
250241
251 try fs.cwd().writeFile(options_file, self.contents.items);242 try options_dir.writeFile(&basename, self.contents.items);
252243
253 self.generated_file.path = options_file;244 self.generated_file.path = try self.builder.cache_root.join(self.builder.allocator, &.{
245 "options", &basename,
246 });
254}247}
255248
256fn hashContentsToFileName(self: *OptionsStep) [64]u8 {249fn hashContentsToFileName(self: *OptionsStep) [64]u8 {
250 // TODO update to use the cache system instead of this
257 // This implementation is copied from `WriteFileStep.make`251 // This implementation is copied from `WriteFileStep.make`
258252
259 var hash = std.crypto.hash.blake2.Blake2b384.init(.{});253 var hash = std.crypto.hash.blake2.Blake2b384.init(.{});
lib/std/Build/RunStep.zig+101-37
...@@ -44,6 +44,10 @@ print: bool,...@@ -44,6 +44,10 @@ print: bool,
44/// running if all output files are up-to-date.44/// running if all output files are up-to-date.
45condition: enum { output_outdated, always } = .output_outdated,45condition: enum { output_outdated, always } = .output_outdated,
4646
47/// Additional file paths relative to build.zig that, when modified, indicate
48/// that the RunStep should be re-executed.
49extra_file_dependencies: []const []const u8 = &.{},
50
47pub const StdIoAction = union(enum) {51pub const StdIoAction = union(enum) {
48 inherit,52 inherit,
49 ignore,53 ignore,
...@@ -184,63 +188,104 @@ fn stdIoActionToBehavior(action: StdIoAction) std.ChildProcess.StdIo {...@@ -184,63 +188,104 @@ fn stdIoActionToBehavior(action: StdIoAction) std.ChildProcess.StdIo {
184}188}
185189
186fn needOutputCheck(self: RunStep) bool {190fn needOutputCheck(self: RunStep) bool {
187 switch (self.condition) {191 if (self.extra_file_dependencies.len > 0) return true;
188 .always => return false,192
189 .output_outdated => {193 for (self.argv.items) |arg| switch (arg) {
190 for (self.argv.items) |arg| switch (arg) {194 .output => return true,
191 .output => return true,195 else => continue,
192 else => continue,196 };
193 };197
194 return false;198 return switch (self.condition) {
195 },199 .always => false,
196 }200 .output_outdated => true,
201 };
197}202}
198203
199fn make(step: *Step) !void {204fn make(step: *Step) !void {
200 const self = @fieldParentPtr(RunStep, "step", step);205 const self = @fieldParentPtr(RunStep, "step", step);
206 const need_output_check = self.needOutputCheck();
201207
202 var argv_list = ArrayList([]const u8).init(self.builder.allocator);208 var argv_list = ArrayList([]const u8).init(self.builder.allocator);
209 var output_placeholders = ArrayList(struct {
210 index: usize,
211 output: Arg.Output,
212 }).init(self.builder.allocator);
213
214 var man = self.builder.cache.obtain();
215 defer man.deinit();
203216
204 for (self.argv.items) |arg| {217 for (self.argv.items) |arg| {
205 switch (arg) {218 switch (arg) {
206 .bytes => |bytes| try argv_list.append(bytes),219 .bytes => |bytes| {
207 .file_source => |file| try argv_list.append(file.getPath(self.builder)),220 try argv_list.append(bytes);
221 man.hash.addBytes(bytes);
222 },
223 .file_source => |file| {
224 const file_path = file.getPath(self.builder);
225 try argv_list.append(file_path);
226 _ = try man.addFile(file_path, null);
227 },
208 .artifact => |artifact| {228 .artifact => |artifact| {
209 if (artifact.target.isWindows()) {229 if (artifact.target.isWindows()) {
210 // On Windows we don't have rpaths so we have to add .dll search paths to PATH230 // On Windows we don't have rpaths so we have to add .dll search paths to PATH
211 self.addPathForDynLibs(artifact);231 self.addPathForDynLibs(artifact);
212 }232 }
213 const executable_path = artifact.installed_path orelse233 const file_path = artifact.installed_path orelse
214 artifact.getOutputSource().getPath(self.builder);234 artifact.getOutputSource().getPath(self.builder);
215 try argv_list.append(executable_path);235
236 try argv_list.append(file_path);
237
238 _ = try man.addFile(file_path, null);
216 },239 },
217 .output => |output| {240 .output => |output| {
218 // TODO: until the cache system is brought into the build system,241 man.hash.addBytes(output.basename);
219 // we use a temporary directory here for each run.242 // Add a placeholder into the argument list because we need the
220 var digest: [16]u8 = undefined;243 // manifest hash to be updated with all arguments before the
221 std.crypto.random.bytes(&digest);244 // object directory is computed.
222 var hash_basename: [digest.len * 2]u8 = undefined;245 try argv_list.append("");
223 _ = std.fmt.bufPrint(246 try output_placeholders.append(.{
224 &hash_basename,247 .index = argv_list.items.len - 1,
225 "{s}",248 .output = output,
226 .{std.fmt.fmtSliceHexLower(&digest)},
227 ) catch unreachable;
228
229 const output_path = try fs.path.join(self.builder.allocator, &[_][]const u8{
230 self.builder.cache_root, "tmp", &hash_basename, output.basename,
231 });249 });
232 const output_dir = fs.path.dirname(output_path).?;
233 fs.cwd().makePath(output_dir) catch |err| {
234 std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });
235 return err;
236 };
237
238 output.generated_file.path = output_path;
239 try argv_list.append(output_path);
240 },250 },
241 }251 }
242 }252 }
243253
254 if (need_output_check) {
255 for (self.extra_file_dependencies) |file_path| {
256 _ = try man.addFile(self.builder.pathFromRoot(file_path), null);
257 }
258
259 if (man.hit() catch |err| failWithCacheError(man, err)) {
260 // cache hit, skip running command
261 const digest = man.final();
262 for (output_placeholders.items) |placeholder| {
263 placeholder.output.generated_file.path = try self.builder.cache_root.join(
264 self.builder.allocator,
265 &.{ "o", &digest, placeholder.output.basename },
266 );
267 }
268 return;
269 }
270
271 const digest = man.final();
272
273 for (output_placeholders.items) |placeholder| {
274 const output_path = try self.builder.cache_root.join(
275 self.builder.allocator,
276 &.{ "o", &digest, placeholder.output.basename },
277 );
278 const output_dir = fs.path.dirname(output_path).?;
279 fs.cwd().makePath(output_dir) catch |err| {
280 std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });
281 return err;
282 };
283
284 placeholder.output.generated_file.path = output_path;
285 argv_list.items[placeholder.index] = output_path;
286 }
287 }
288
244 try runCommand(289 try runCommand(
245 argv_list.items,290 argv_list.items,
246 self.builder,291 self.builder,
...@@ -252,6 +297,10 @@ fn make(step: *Step) !void {...@@ -252,6 +297,10 @@ fn make(step: *Step) !void {
252 self.cwd,297 self.cwd,
253 self.print,298 self.print,
254 );299 );
300
301 if (need_output_check) {
302 try man.writeManifest();
303 }
255}304}
256305
257pub fn runCommand(306pub fn runCommand(
...@@ -265,11 +314,13 @@ pub fn runCommand(...@@ -265,11 +314,13 @@ pub fn runCommand(
265 maybe_cwd: ?[]const u8,314 maybe_cwd: ?[]const u8,
266 print: bool,315 print: bool,
267) !void {316) !void {
268 const cwd = if (maybe_cwd) |cwd| builder.pathFromRoot(cwd) else builder.build_root;317 const cwd = if (maybe_cwd) |cwd| builder.pathFromRoot(cwd) else builder.build_root.path;
269318
270 if (!std.process.can_spawn) {319 if (!std.process.can_spawn) {
271 const cmd = try std.mem.join(builder.allocator, " ", argv);320 const cmd = try std.mem.join(builder.allocator, " ", argv);
272 std.debug.print("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{ @tagName(builtin.os.tag), cmd });321 std.debug.print("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{
322 @tagName(builtin.os.tag), cmd,
323 });
273 builder.allocator.free(cmd);324 builder.allocator.free(cmd);
274 return ExecError.ExecNotSupported;325 return ExecError.ExecNotSupported;
275 }326 }
...@@ -410,6 +461,19 @@ pub fn runCommand(...@@ -410,6 +461,19 @@ pub fn runCommand(
410 }461 }
411}462}
412463
464fn failWithCacheError(man: std.Build.Cache.Manifest, err: anyerror) noreturn {
465 const i = man.failed_file_index orelse failWithSimpleError(err);
466 const pp = man.files.items[i].prefixed_path orelse failWithSimpleError(err);
467 const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
468 std.debug.print("{s}: {s}/{s}\n", .{ @errorName(err), prefix, pp.sub_path });
469 std.process.exit(1);
470}
471
472fn failWithSimpleError(err: anyerror) noreturn {
473 std.debug.print("{s}\n", .{@errorName(err)});
474 std.process.exit(1);
475}
476
413fn printCmd(cwd: ?[]const u8, argv: []const []const u8) void {477fn printCmd(cwd: ?[]const u8, argv: []const []const u8) void {
414 if (cwd) |yes_cwd| std.debug.print("cd {s} && ", .{yes_cwd});478 if (cwd) |yes_cwd| std.debug.print("cd {s} && ", .{yes_cwd});
415 for (argv) |arg| {479 for (argv) |arg| {
lib/std/Build/WriteFileStep.zig+2-2
...@@ -85,8 +85,8 @@ fn make(step: *Step) !void {...@@ -85,8 +85,8 @@ fn make(step: *Step) !void {
85 .{std.fmt.fmtSliceHexLower(&digest)},85 .{std.fmt.fmtSliceHexLower(&digest)},
86 ) catch unreachable;86 ) catch unreachable;
8787
88 const output_dir = try fs.path.join(self.builder.allocator, &[_][]const u8{88 const output_dir = try self.builder.cache_root.join(self.builder.allocator, &.{
89 self.builder.cache_root, "o", &hash_basename,89 "o", &hash_basename,
90 });90 });
91 var dir = fs.cwd().makeOpenPath(output_dir, .{}) catch |err| {91 var dir = fs.cwd().makeOpenPath(output_dir, .{}) catch |err| {
92 std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });92 std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });
test/tests.zig+2-2
...@@ -570,7 +570,7 @@ pub fn addCliTests(b: *std.Build, test_filter: ?[]const u8, optimize_modes: []co...@@ -570,7 +570,7 @@ pub fn addCliTests(b: *std.Build, test_filter: ?[]const u8, optimize_modes: []co
570 const run_cmd = exe.run();570 const run_cmd = exe.run();
571 run_cmd.addArgs(&[_][]const u8{571 run_cmd.addArgs(&[_][]const u8{
572 fs.realpathAlloc(b.allocator, b.zig_exe) catch unreachable,572 fs.realpathAlloc(b.allocator, b.zig_exe) catch unreachable,
573 b.pathFromRoot(b.cache_root),573 b.pathFromRoot(b.cache_root.path orelse "."),
574 });574 });
575575
576 step.dependOn(&run_cmd.step);576 step.dependOn(&run_cmd.step);
...@@ -1059,7 +1059,7 @@ pub const StandaloneContext = struct {...@@ -1059,7 +1059,7 @@ pub const StandaloneContext = struct {
1059 }1059 }
10601060
1061 var zig_args = ArrayList([]const u8).init(b.allocator);1061 var zig_args = ArrayList([]const u8).init(b.allocator);
1062 const rel_zig_exe = fs.path.relative(b.allocator, b.build_root, b.zig_exe) catch unreachable;1062 const rel_zig_exe = fs.path.relative(b.allocator, b.build_root.path orelse ".", b.zig_exe) catch unreachable;
1063 zig_args.append(rel_zig_exe) catch unreachable;1063 zig_args.append(rel_zig_exe) catch unreachable;
1064 zig_args.append("build") catch unreachable;1064 zig_args.append("build") catch unreachable;
10651065