Commit a9e1cf3049d1d90d7e62950c3b7f4b088081caa8 | zig

CMakeLists.txt+3-2

 @@ -216,6 +216,9 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/lib/std/atomic/stack.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/base64.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/buf_map.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/Build.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/Build/Cache.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/Build/Cache/DepTokenizer.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/builtin.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/c.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/c/linux.zig"
 @@ -523,9 +526,7 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/lib/std/zig/tokenizer.zig"
     "${CMAKE_SOURCE_DIR}/src/Air.zig"
     "${CMAKE_SOURCE_DIR}/src/AstGen.zig"
-    "${CMAKE_SOURCE_DIR}/src/Cache.zig"
     "${CMAKE_SOURCE_DIR}/src/Compilation.zig"
-    "${CMAKE_SOURCE_DIR}/src/DepTokenizer.zig"
     "${CMAKE_SOURCE_DIR}/src/Liveness.zig"
     "${CMAKE_SOURCE_DIR}/src/Module.zig"
     "${CMAKE_SOURCE_DIR}/src/Package.zig"

build.zig+3-7

 @@ -40,15 +40,11 @@ pub fn build(b: *std.Build) !void {
     });
     docgen_exe.single_threaded = single_threaded;
-    const rel_zig_exe = try fs.path.relative(b.allocator, b.build_root, b.zig_exe);
+    const langref_out_path = try b.cache_root.join(b.allocator, &.{"langref.html"});
-    const langref_out_path = fs.path.join(
-        b.allocator,
-        &[_][]const u8{ b.cache_root, "langref.html" },
-    ) catch unreachable;
     const docgen_cmd = docgen_exe.run();
     docgen_cmd.addArgs(&[_][]const u8{
         "--zig",
-        rel_zig_exe,
+        b.zig_exe,
         "doc" ++ fs.path.sep_str ++ "langref.html.in",
         langref_out_path,
     });
 @@ -215,7 +211,7 @@ pub fn build(b: *std.Build) !void {
         var code: u8 = undefined;
         const git_describe_untrimmed = b.execAllowFail(&[_][]const u8{
-            "git", "-C", b.build_root, "describe", "--match", "*.*.*", "--tags",
+            "git", "-C", b.build_root.path orelse ".", "describe", "--match", "*.*.*", "--tags",
         }, &code, .Ignore) catch {
             break :v version_string;
         };

lib/build_runner.zig+31-4

...	@@ -43,13 +43,40 @@ pub fn main() !void {	...	@@ -43,13 +43,40 @@ pub fn main() !void {
43		43
44	const host = try std.zig.system.NativeTargetInfo.detect(.{});	44	const host = try std.zig.system.NativeTargetInfo.detect(.{});
45		45
		46	const build_root_directory: std.Build.Cache.Directory = .{
		47	.path = build_root,
		48	.handle = try std.fs.cwd().openDir(build_root, .{}),
		49	};
		50
		51	const local_cache_directory: std.Build.Cache.Directory = .{
		52	.path = cache_root,
		53	.handle = try std.fs.cwd().makeOpenPath(cache_root, .{}),
		54	};
		55
		56	const global_cache_directory: std.Build.Cache.Directory = .{
		57	.path = global_cache_root,
		58	.handle = try std.fs.cwd().makeOpenPath(global_cache_root, .{}),
		59	};
		60
		61	var cache: std.Build.Cache = .{
		62	.gpa = allocator,
		63	.manifest_dir = try local_cache_directory.handle.makeOpenPath("h", .{}),
		64	};
		65	cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() });
		66	cache.addPrefix(build_root_directory);
		67	cache.addPrefix(local_cache_directory);
		68	cache.addPrefix(global_cache_directory);
		69
		70	//cache.hash.addBytes(builtin.zig_version);
		71
46	const builder = try std.Build.create(	72	const builder = try std.Build.create(
47	allocator,	73	allocator,
48	zig_exe,	74	zig_exe,
49	build_root,	75	build_root_directory,
50	cache_root,	76	local_cache_directory,
51	global_cache_root,	77	global_cache_directory,
52	host,	78	host,
		79	&cache,
53	);	80	);
54	defer builder.destroy();	81	defer builder.destroy();
55		82
...	@@ -138,7 +165,7 @@ pub fn main() !void {	...	@@ -138,7 +165,7 @@ pub fn main() !void {
138	return usageAndErr(builder, false, stderr_stream);	165	return usageAndErr(builder, false, stderr_stream);
139	};	166	};
140	} else if (mem.eql(u8, arg, "--zig-lib-dir")) {	167	} else if (mem.eql(u8, arg, "--zig-lib-dir")) {
141	builder.override_lib_dir = nextArg(args, &arg_idx) orelse {	168	builder.zig_lib_dir = nextArg(args, &arg_idx) orelse {
142	std.debug.print("Expected argument after --zig-lib-dir\n\n", .{});	169	std.debug.print("Expected argument after --zig-lib-dir\n\n", .{});
143	return usageAndErr(builder, false, stderr_stream);	170	return usageAndErr(builder, false, stderr_stream);
144	};	171	};

lib/std/Build.zig+56-49

 @@ -19,6 +19,8 @@ const NativeTargetInfo = std.zig.system.NativeTargetInfo;
 const Sha256 = std.crypto.hash.sha2.Sha256;
 const Build = @This();
+pub const Cache = @import("Build/Cache.zig");
 /// deprecated: use `CompileStep`.
 pub const LibExeObjStep = CompileStep;
 /// deprecated: use `Build`.
 @@ -77,11 +79,12 @@ search_prefixes: ArrayList([]const u8),
 libc_file: ?[]const u8 = null,
 installed_files: ArrayList(InstalledFile),
 /// Path to the directory containing build.zig.
-build_root: []const u8,
+build_root: Cache.Directory,
-cache_root: []const u8,
+cache_root: Cache.Directory,
-global_cache_root: []const u8,
+global_cache_root: Cache.Directory,
-/// zig lib dir
+cache: *Cache,
-override_lib_dir: ?[]const u8,
+/// If non-null, overrides the default zig lib dir.
+zig_lib_dir: ?[]const u8,
 vcpkg_root: VcpkgRoot = .unattempted,
 pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null,
 args: ?[][]const u8 = null,
 @@ -185,10 +188,11 @@ pub const DirList = struct {
 pub fn create(
     allocator: Allocator,
     zig_exe: []const u8,
-    build_root: []const u8,
+    build_root: Cache.Directory,
-    cache_root: []const u8,
+    cache_root: Cache.Directory,
-    global_cache_root: []const u8,
+    global_cache_root: Cache.Directory,
     host: NativeTargetInfo,
+    cache: *Cache,
 ) !*Build {
     const env_map = try allocator.create(EnvMap);
     env_map.* = try process.getEnvMap(allocator);
 @@ -197,8 +201,9 @@ pub fn create(
     self.* = Build{
         .zig_exe = zig_exe,
         .build_root = build_root,
-        .cache_root = try fs.path.relative(allocator, build_root, cache_root),
+        .cache_root = cache_root,
         .global_cache_root = global_cache_root,
+        .cache = cache,
         .verbose = false,
         .verbose_link = false,
         .verbose_cc = false,
 @@ -230,7 +235,7 @@ pub fn create(
             .step = Step.init(.top_level, "uninstall", allocator, makeUninstall),
             .description = "Remove build artifacts from prefix path",
         },
-        .override_lib_dir = null,
+        .zig_lib_dir = null,
         .install_path = undefined,
         .args = null,
         .host = host,
 @@ -245,7 +250,7 @@ pub fn create(
 fn createChild(
     parent: *Build,
     dep_name: []const u8,
-    build_root: []const u8,
+    build_root: Cache.Directory,
     args: anytype,
 ) !*Build {
     const child = try createChildOnly(parent, dep_name, build_root);
 @@ -253,7 +258,7 @@ fn createChild(
     return child;
 }
-fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: []const u8) !*Build {
+fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Directory) !*Build {
     const allocator = parent.allocator;
     const child = try allocator.create(Build);
     child.* = .{
 @@ -297,7 +302,8 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: []const u8)
         .build_root = build_root,
         .cache_root = parent.cache_root,
         .global_cache_root = parent.global_cache_root,
-        .override_lib_dir = parent.override_lib_dir,
+        .cache = parent.cache,
+        .zig_lib_dir = parent.zig_lib_dir,
         .debug_log_scopes = parent.debug_log_scopes,
         .debug_compile_errors = parent.debug_compile_errors,
         .enable_darling = parent.enable_darling,
 @@ -348,7 +354,7 @@ fn applyArgs(b: *Build, args: anytype) !void {
                         .used = false,
                     });
                 },
-                .Enum => {
+                .Enum, .EnumLiteral => {
                     try b.user_input_options.put(field.name, .{
                         .name = field.name,
                         .value = .{ .scalar = @tagName(v) },
 @@ -379,7 +385,7 @@ fn applyArgs(b: *Build, args: anytype) !void {
     _ = std.fmt.bufPrint(&hash_basename, "{s}", .{std.fmt.fmtSliceHexLower(&digest)}) catch
         unreachable;
-    const install_prefix = b.pathJoin(&.{ b.cache_root, "i", &hash_basename });
+    const install_prefix = try b.cache_root.join(b.allocator, &.{ "i", &hash_basename });
     b.resolveInstallPrefix(install_prefix, .{});
 }
 @@ -396,7 +402,7 @@ pub fn resolveInstallPrefix(self: *Build, install_prefix: ?[]const u8, dir_list:
         self.install_path = self.pathJoin(&.{ dest_dir, self.install_prefix });
     } else {
         self.install_prefix = install_prefix orelse
-            (self.pathJoin(&.{ self.build_root, "zig-out" }));
+            (self.build_root.join(self.allocator, &.{"zig-out"}) catch @panic("unhandled error"));
         self.install_path = self.install_prefix;
     }
 @@ -599,6 +605,28 @@ pub fn addSystemCommand(self: *Build, argv: []const []const u8) *RunStep {
     return run_step;
 }
+/// Creates a `RunStep` with an executable built with `addExecutable`.
+/// Add command line arguments with methods of `RunStep`.
+pub fn addRunArtifact(b: *Build, exe: *CompileStep) *RunStep {
+    assert(exe.kind == .exe or exe.kind == .test_exe);
+    // It doesn't have to be native. We catch that if you actually try to run it.
+    // Consider that this is declarative; the run step may not be run unless a user
+    // option is supplied.
+    const run_step = RunStep.create(b, b.fmt("run {s}", .{exe.step.name}));
+    run_step.addArtifactArg(exe);
+    if (exe.kind == .test_exe) {
+        run_step.addArg(b.zig_exe);
+    }
+    if (exe.vcpkg_bin_path) |path| {
+        run_step.addPathDir(path);
+    }
+    return run_step;
+}
 /// Using the `values` provided, produces a C header file, possibly based on a
 /// template input file (e.g. config.h.in).
 /// When an input template file is provided, this function will fail the build
 @@ -674,8 +702,6 @@ pub fn addTranslateC(self: *Build, options: TranslateCStep.Options) *TranslateCS
 }
 pub fn make(self: *Build, step_names: []const []const u8) !void {
-    try self.makePath(self.cache_root);
     var wanted_steps = ArrayList(*Step).init(self.allocator);
     defer wanted_steps.deinit();
 @@ -1201,13 +1227,6 @@ pub fn spawnChildEnvMap(self: *Build, cwd: ?[]const u8, env_map: *const EnvMap,
     }
 }
-pub fn makePath(self: *Build, path: []const u8) !void {
-    fs.cwd().makePath(self.pathFromRoot(path)) catch |err| {
-        log.err("Unable to create path {s}: {s}", .{ path, @errorName(err) });
-        return err;
-    };
-}
 pub fn installArtifact(self: *Build, artifact: *CompileStep) void {
     self.getInstallStep().dependOn(&self.addInstallArtifact(artifact).step);
 }
 @@ -1322,8 +1341,8 @@ pub fn truncateFile(self: *Build, dest_path: []const u8) !void {
     src_file.close();
 }
-pub fn pathFromRoot(self: *Build, rel_path: []const u8) []u8 {
+pub fn pathFromRoot(b: *Build, p: []const u8) []u8 {
-    return fs.path.resolve(self.allocator, &[_][]const u8{ self.build_root, rel_path }) catch @panic("OOM");
+    return fs.path.resolve(b.allocator, &.{ b.build_root.path orelse ".", p }) catch @panic("OOM");
 }
 pub fn pathJoin(self: *Build, paths: []const []const u8) []u8 {
 @@ -1544,10 +1563,19 @@ pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {
 fn dependencyInner(
     b: *Build,
     name: []const u8,
-    build_root: []const u8,
+    build_root_string: []const u8,
     comptime build_zig: type,
     args: anytype,
 ) *Dependency {
+    const build_root: std.Build.Cache.Directory = .{
+        .path = build_root_string,
+        .handle = std.fs.cwd().openDir(build_root_string, .{}) catch |err| {
+            std.debug.print("unable to open '{s}': {s}\n", .{
+                build_root_string, @errorName(err),
+            });
+            std.process.exit(1);
+        },
+    };
     const sub_builder = b.createChild(name, build_root, args) catch @panic("unhandled error");
     sub_builder.runBuild(build_zig) catch @panic("unhandled error");
 @@ -1568,26 +1596,6 @@ pub fn runBuild(b: *Build, build_zig: anytype) anyerror!void {
     }
 }
-test "builder.findProgram compiles" {
-    if (builtin.os.tag == .wasi) return error.SkipZigTest;
-    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
-    defer arena.deinit();
-    const host = try NativeTargetInfo.detect(.{});
-    const builder = try Build.create(
-        arena.allocator(),
-        "zig",
-        "zig-cache",
-        "zig-cache",
-        "zig-cache",
-        host,
-    );
-    defer builder.destroy();
-    _ = builder.findProgram(&[_][]const u8{}, &[_][]const u8{}) catch null;
-}
 pub const Module = struct {
     builder: *Build,
     /// This could either be a generated file, in which case the module
 @@ -1616,7 +1624,6 @@ pub const GeneratedFile = struct {
 };
 /// A file source is a reference to an existing or future file.
-///
 pub const FileSource = union(enum) {
     /// A plain file path, relative to build root or absolute.
     path: []const u8,

lib/std/Build/Cache.zig created+1253

...	@@ -0,0 +1,1253 @@
1	//! Manages `zig-cache` directories.
2	//! This is not a general-purpose cache. It is designed to be fast and simple,
3	//! not to withstand attacks using specially-crafted input.
4
5	pub const Directory = struct {
6	/// This field is redundant for operations that can act on the open directory handle
7	/// directly, but it is needed when passing the directory to a child process.
8	/// `null` means cwd.
9	path: ?[]const u8,
10	handle: std.fs.Dir,
11
12	pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
13	if (self.path) \|p\| {
14	// TODO clean way to do this with only 1 allocation
15	const part2 = try std.fs.path.join(allocator, paths);
16	defer allocator.free(part2);
17	return std.fs.path.join(allocator, &[_][]const u8{ p, part2 });
18	} else {
19	return std.fs.path.join(allocator, paths);
20	}
21	}
22
23	pub fn joinZ(self: Directory, allocator: Allocator, paths: []const []const u8) ![:0]u8 {
24	if (self.path) \|p\| {
25	// TODO clean way to do this with only 1 allocation
26	const part2 = try std.fs.path.join(allocator, paths);
27	defer allocator.free(part2);
28	return std.fs.path.joinZ(allocator, &[_][]const u8{ p, part2 });
29	} else {
30	return std.fs.path.joinZ(allocator, paths);
31	}
32	}
33
34	/// Whether or not the handle should be closed, or the path should be freed
35	/// is determined by usage, however this function is provided for convenience
36	/// if it happens to be what the caller needs.
37	pub fn closeAndFree(self: *Directory, gpa: Allocator) void {
38	self.handle.close();
39	if (self.path) \|p\| gpa.free(p);
40	self.* = undefined;
41	}
42	};
43
44	gpa: Allocator,
45	manifest_dir: fs.Dir,
46	hash: HashHelper = .{},
47	/// This value is accessed from multiple threads, protected by mutex.
48	recent_problematic_timestamp: i128 = 0,
49	mutex: std.Thread.Mutex = .{},
50
51	/// A set of strings such as the zig library directory or project source root, which
52	/// are stripped from the file paths before putting into the cache. They
53	/// are replaced with single-character indicators. This is not to save
54	/// space but to eliminate absolute file paths. This improves portability
55	/// and usefulness of the cache for advanced use cases.
56	prefixes_buffer: [4]Directory = undefined,
57	prefixes_len: usize = 0,
58
59	pub const DepTokenizer = @import("Cache/DepTokenizer.zig");
60
61	const Cache = @This();
62	const std = @import("std");
63	const builtin = @import("builtin");
64	const crypto = std.crypto;
65	const fs = std.fs;
66	const assert = std.debug.assert;
67	const testing = std.testing;
68	const mem = std.mem;
69	const fmt = std.fmt;
70	const Allocator = std.mem.Allocator;
71	const log = std.log.scoped(.cache);
72
73	pub fn addPrefix(cache: *Cache, directory: Directory) void {
74	cache.prefixes_buffer[cache.prefixes_len] = directory;
75	cache.prefixes_len += 1;
76	}
77
78	/// Be sure to call `Manifest.deinit` after successful initialization.
79	pub fn obtain(cache: *Cache) Manifest {
80	return Manifest{
81	.cache = cache,
82	.hash = cache.hash,
83	.manifest_file = null,
84	.manifest_dirty = false,
85	.hex_digest = undefined,
86	};
87	}
88
89	pub fn prefixes(cache: *const Cache) []const Directory {
90	return cache.prefixes_buffer[0..cache.prefixes_len];
91	}
92
93	const PrefixedPath = struct {
94	prefix: u8,
95	sub_path: []u8,
96	};
97
98	fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath {
99	const gpa = cache.gpa;
100	const resolved_path = try fs.path.resolve(gpa, &[_][]const u8{file_path});
101	errdefer gpa.free(resolved_path);
102	return findPrefixResolved(cache, resolved_path);
103	}
104
105	/// Takes ownership of `resolved_path` on success.
106	fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath {
107	const gpa = cache.gpa;
108	const prefixes_slice = cache.prefixes();
109	var i: u8 = 1; // Start at 1 to skip over checking the null prefix.
110	while (i < prefixes_slice.len) : (i += 1) {
111	const p = prefixes_slice[i].path.?;
112	if (mem.startsWith(u8, resolved_path, p)) {
113	// +1 to skip over the path separator here
114	const sub_path = try gpa.dupe(u8, resolved_path[p.len + 1 ..]);
115	gpa.free(resolved_path);
116	return PrefixedPath{
117	.prefix = @intCast(u8, i),
118	.sub_path = sub_path,
119	};
120	}
121	}
122
123	return PrefixedPath{
124	.prefix = 0,
125	.sub_path = resolved_path,
126	};
127	}
128
129	/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
130	pub const bin_digest_len = 16;
131	pub const hex_digest_len = bin_digest_len * 2;
132	pub const BinDigest = [bin_digest_len]u8;
133
134	const manifest_file_size_max = 50 * 1024 * 1024;
135
136	/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it
137	/// provides enough collision resistance for the Manifest use cases, while being one of our
138	/// fastest options right now.
139	pub const Hasher = crypto.auth.siphash.SipHash128(1, 3);
140
141	/// Initial state, that can be copied.
142	pub const hasher_init: Hasher = Hasher.init(&[_]u8{0} ** Hasher.key_length);
143
144	pub const File = struct {
145	prefixed_path: ?PrefixedPath,
146	max_file_size: ?usize,
147	stat: Stat,
148	bin_digest: BinDigest,
149	contents: ?[]const u8,
150
151	pub const Stat = struct {
152	inode: fs.File.INode,
153	size: u64,
154	mtime: i128,
155	};
156
157	pub fn deinit(self: *File, gpa: Allocator) void {
158	if (self.prefixed_path) \|pp\| {
159	gpa.free(pp.sub_path);
160	self.prefixed_path = null;
161	}
162	if (self.contents) \|contents\| {
163	gpa.free(contents);
164	self.contents = null;
165	}
166	self.* = undefined;
167	}
168	};
169
170	pub const HashHelper = struct {
171	hasher: Hasher = hasher_init,
172
173	/// Record a slice of bytes as an dependency of the process being cached
174	pub fn addBytes(hh: *HashHelper, bytes: []const u8) void {
175	hh.hasher.update(mem.asBytes(&bytes.len));
176	hh.hasher.update(bytes);
177	}
178
179	pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void {
180	hh.add(optional_bytes != null);
181	hh.addBytes(optional_bytes orelse return);
182	}
183
184	pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void {
185	hh.add(list_of_bytes.len);
186	for (list_of_bytes) \|bytes\| hh.addBytes(bytes);
187	}
188
189	/// Convert the input value into bytes and record it as a dependency of the process being cached.
190	pub fn add(hh: *HashHelper, x: anytype) void {
191	switch (@TypeOf(x)) {
192	std.builtin.Version => {
193	hh.add(x.major);
194	hh.add(x.minor);
195	hh.add(x.patch);
196	},
197	std.Target.Os.TaggedVersionRange => {
198	switch (x) {
199	.linux => \|linux\| {
200	hh.add(linux.range.min);
201	hh.add(linux.range.max);
202	hh.add(linux.glibc);
203	},
204	.windows => \|windows\| {
205	hh.add(windows.min);
206	hh.add(windows.max);
207	},
208	.semver => \|semver\| {
209	hh.add(semver.min);
210	hh.add(semver.max);
211	},
212	.none => {},
213	}
214	},
215	else => switch (@typeInfo(@TypeOf(x))) {
216	.Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)),
217	else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))),
218	},
219	}
220	}
221
222	pub fn addOptional(hh: *HashHelper, optional: anytype) void {
223	hh.add(optional != null);
224	hh.add(optional orelse return);
225	}
226
227	/// Returns a hex encoded hash of the inputs, without modifying state.
228	pub fn peek(hh: HashHelper) [hex_digest_len]u8 {
229	var copy = hh;
230	return copy.final();
231	}
232
233	pub fn peekBin(hh: HashHelper) BinDigest {
234	var copy = hh;
235	var bin_digest: BinDigest = undefined;
236	copy.hasher.final(&bin_digest);
237	return bin_digest;
238	}
239
240	/// Returns a hex encoded hash of the inputs, mutating the state of the hasher.
241	pub fn final(hh: *HashHelper) [hex_digest_len]u8 {
242	var bin_digest: BinDigest = undefined;
243	hh.hasher.final(&bin_digest);
244
245	var out_digest: [hex_digest_len]u8 = undefined;
246	_ = std.fmt.bufPrint(
247	&out_digest,
248	"{s}",
249	.{std.fmt.fmtSliceHexLower(&bin_digest)},
250	) catch unreachable;
251	return out_digest;
252	}
253	};
254
255	pub const Lock = struct {
256	manifest_file: fs.File,
257
258	pub fn release(lock: *Lock) void {
259	if (builtin.os.tag == .windows) {
260	// Windows does not guarantee that locks are immediately unlocked when
261	// the file handle is closed. See LockFileEx documentation.
262	lock.manifest_file.unlock();
263	}
264
265	lock.manifest_file.close();
266	lock.* = undefined;
267	}
268	};
269
270	pub const Manifest = struct {
271	cache: *Cache,
272	/// Current state for incremental hashing.
273	hash: HashHelper,
274	manifest_file: ?fs.File,
275	manifest_dirty: bool,
276	/// Set this flag to true before calling hit() in order to indicate that
277	/// upon a cache hit, the code using the cache will not modify the files
278	/// within the cache directory. This allows multiple processes to utilize
279	/// the same cache directory at the same time.
280	want_shared_lock: bool = true,
281	have_exclusive_lock: bool = false,
282	// Indicate that we want isProblematicTimestamp to perform a filesystem write in
283	// order to obtain a problematic timestamp for the next call. Calls after that
284	// will then use the same timestamp, to avoid unnecessary filesystem writes.
285	want_refresh_timestamp: bool = true,
286	files: std.ArrayListUnmanaged(File) = .{},
287	hex_digest: [hex_digest_len]u8,
288	/// Populated when hit() returns an error because of one
289	/// of the files listed in the manifest.
290	failed_file_index: ?usize = null,
291	/// Keeps track of the last time we performed a file system write to observe
292	/// what time the file system thinks it is, according to its own granularity.
293	recent_problematic_timestamp: i128 = 0,
294
295	/// Add a file as a dependency of process being cached. When `hit` is
296	/// called, the file's contents will be checked to ensure that it matches
297	/// the contents from previous times.
298	///
299	/// Max file size will be used to determine the amount of space the file contents
300	/// are allowed to take up in memory. If max_file_size is null, then the contents
301	/// will not be loaded into memory.
302	///
303	/// Returns the index of the entry in the `files` array list. You can use it
304	/// to access the contents of the file after calling `hit()` like so:
305	///
306	/// ```
307	/// var file_contents = cache_hash.files.items[file_index].contents.?;
308	/// ```
309	pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize {
310	assert(self.manifest_file == null);
311
312	const gpa = self.cache.gpa;
313	try self.files.ensureUnusedCapacity(gpa, 1);
314	const prefixed_path = try self.cache.findPrefix(file_path);
315	errdefer gpa.free(prefixed_path.sub_path);
316
317	self.files.addOneAssumeCapacity().* = .{
318	.prefixed_path = prefixed_path,
319	.contents = null,
320	.max_file_size = max_file_size,
321	.stat = undefined,
322	.bin_digest = undefined,
323	};
324
325	self.hash.add(prefixed_path.prefix);
326	self.hash.addBytes(prefixed_path.sub_path);
327
328	return self.files.items.len - 1;
329	}
330
331	pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void {
332	self.hash.add(optional_file_path != null);
333	const file_path = optional_file_path orelse return;
334	_ = try self.addFile(file_path, null);
335	}
336
337	pub fn addListOfFiles(self: *Manifest, list_of_files: []const []const u8) !void {
338	self.hash.add(list_of_files.len);
339	for (list_of_files) \|file_path\| {
340	_ = try self.addFile(file_path, null);
341	}
342	}
343
344	/// Check the cache to see if the input exists in it. If it exists, returns `true`.
345	/// A hex encoding of its hash is available by calling `final`.
346	///
347	/// This function will also acquire an exclusive lock to the manifest file. This means
348	/// that a process holding a Manifest will block any other process attempting to
349	/// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the
350	/// manifest file to be locked in shared mode, and a cache miss guarantees the manifest
351	/// file to be locked in exclusive mode.
352	///
353	/// The lock on the manifest file is released when `deinit` is called. As another
354	/// option, one may call `toOwnedLock` to obtain a smaller object which can represent
355	/// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
356	pub fn hit(self: *Manifest) !bool {
357	const gpa = self.cache.gpa;
358	assert(self.manifest_file == null);
359
360	self.failed_file_index = null;
361
362	const ext = ".txt";
363	var manifest_file_path: [self.hex_digest.len + ext.len]u8 = undefined;
364
365	var bin_digest: BinDigest = undefined;
366	self.hash.hasher.final(&bin_digest);
367
368	_ = std.fmt.bufPrint(
369	&self.hex_digest,
370	"{s}",
371	.{std.fmt.fmtSliceHexLower(&bin_digest)},
372	) catch unreachable;
373
374	self.hash.hasher = hasher_init;
375	self.hash.hasher.update(&bin_digest);
376
377	mem.copy(u8, &manifest_file_path, &self.hex_digest);
378	manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*;
379
380	if (self.files.items.len == 0) {
381	// If there are no file inputs, we check if the manifest file exists instead of
382	// comparing the hashes on the files used for the cached item
383	while (true) {
384	if (self.cache.manifest_dir.openFile(&manifest_file_path, .{
385	.mode = .read_write,
386	.lock = .Exclusive,
387	.lock_nonblocking = self.want_shared_lock,
388	})) \|manifest_file\| {
389	self.manifest_file = manifest_file;
390	self.have_exclusive_lock = true;
391	break;
392	} else \|open_err\| switch (open_err) {
393	error.WouldBlock => {
394	self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
395	.lock = .Shared,
396	});
397	break;
398	},
399	error.FileNotFound => {
400	if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
401	.read = true,
402	.truncate = false,
403	.lock = .Exclusive,
404	.lock_nonblocking = self.want_shared_lock,
405	})) \|manifest_file\| {
406	self.manifest_file = manifest_file;
407	self.manifest_dirty = true;
408	self.have_exclusive_lock = true;
409	return false; // cache miss; exclusive lock already held
410	} else \|err\| switch (err) {
411	error.WouldBlock => continue,
412	else => \|e\| return e,
413	}
414	},
415	else => \|e\| return e,
416	}
417	}
418	} else {
419	if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
420	.read = true,
421	.truncate = false,
422	.lock = .Exclusive,
423	.lock_nonblocking = self.want_shared_lock,
424	})) \|manifest_file\| {
425	self.manifest_file = manifest_file;
426	self.have_exclusive_lock = true;
427	} else \|err\| switch (err) {
428	error.WouldBlock => {
429	self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
430	.lock = .Shared,
431	});
432	},
433	else => \|e\| return e,
434	}
435	}
436
437	self.want_refresh_timestamp = true;
438
439	const file_contents = try self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max);
440	defer gpa.free(file_contents);
441
442	const input_file_count = self.files.items.len;
443	var any_file_changed = false;
444	var line_iter = mem.tokenize(u8, file_contents, "\n");
445	var idx: usize = 0;
446	while (line_iter.next()) \|line\| {
447	defer idx += 1;
448
449	const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: {
450	const new = try self.files.addOne(gpa);
451	new.* = .{
452	.prefixed_path = null,
453	.contents = null,
454	.max_file_size = null,
455	.stat = undefined,
456	.bin_digest = undefined,
457	};
458	break :blk new;
459	};
460
461	var iter = mem.tokenize(u8, line, " ");
462	const size = iter.next() orelse return error.InvalidFormat;
463	const inode = iter.next() orelse return error.InvalidFormat;
464	const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
465	const digest_str = iter.next() orelse return error.InvalidFormat;
466	const prefix_str = iter.next() orelse return error.InvalidFormat;
467	const file_path = iter.rest();
468
469	cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat;
470	cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat;
471	cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
472	_ = std.fmt.hexToBytes(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;
473	const prefix = fmt.parseInt(u8, prefix_str, 10) catch return error.InvalidFormat;
474	if (prefix >= self.cache.prefixes_len) return error.InvalidFormat;
475
476	if (file_path.len == 0) {
477	return error.InvalidFormat;
478	}
479	if (cache_hash_file.prefixed_path) \|pp\| {
480	if (pp.prefix != prefix or !mem.eql(u8, file_path, pp.sub_path)) {
481	return error.InvalidFormat;
482	}
483	}
484
485	if (cache_hash_file.prefixed_path == null) {
486	cache_hash_file.prefixed_path = .{
487	.prefix = prefix,
488	.sub_path = try gpa.dupe(u8, file_path),
489	};
490	}
491
492	const pp = cache_hash_file.prefixed_path.?;
493	const dir = self.cache.prefixes()[pp.prefix].handle;
494	const this_file = dir.openFile(pp.sub_path, .{ .mode = .read_only }) catch \|err\| switch (err) {
495	error.FileNotFound => {
496	try self.upgradeToExclusiveLock();
497	return false;
498	},
499	else => return error.CacheUnavailable,
500	};
501	defer this_file.close();
502
503	const actual_stat = this_file.stat() catch \|err\| {
504	self.failed_file_index = idx;
505	return err;
506	};
507	const size_match = actual_stat.size == cache_hash_file.stat.size;
508	const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime;
509	const inode_match = actual_stat.inode == cache_hash_file.stat.inode;
510
511	if (!size_match or !mtime_match or !inode_match) {
512	self.manifest_dirty = true;
513
514	cache_hash_file.stat = .{
515	.size = actual_stat.size,
516	.mtime = actual_stat.mtime,
517	.inode = actual_stat.inode,
518	};
519
520	if (self.isProblematicTimestamp(cache_hash_file.stat.mtime)) {
521	// The actual file has an unreliable timestamp, force it to be hashed
522	cache_hash_file.stat.mtime = 0;
523	cache_hash_file.stat.inode = 0;
524	}
525
526	var actual_digest: BinDigest = undefined;
527	hashFile(this_file, &actual_digest) catch \|err\| {
528	self.failed_file_index = idx;
529	return err;
530	};
531
532	if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
533	cache_hash_file.bin_digest = actual_digest;
534	// keep going until we have the input file digests
535	any_file_changed = true;
536	}
537	}
538
539	if (!any_file_changed) {
540	self.hash.hasher.update(&cache_hash_file.bin_digest);
541	}
542	}
543
544	if (any_file_changed) {
545	// cache miss
546	// keep the manifest file open
547	self.unhit(bin_digest, input_file_count);
548	try self.upgradeToExclusiveLock();
549	return false;
550	}
551
552	if (idx < input_file_count) {
553	self.manifest_dirty = true;
554	while (idx < input_file_count) : (idx += 1) {
555	const ch_file = &self.files.items[idx];
556	self.populateFileHash(ch_file) catch \|err\| {
557	self.failed_file_index = idx;
558	return err;
559	};
560	}
561	try self.upgradeToExclusiveLock();
562	return false;
563	}
564
565	if (self.want_shared_lock) {
566	try self.downgradeToSharedLock();
567	}
568
569	return true;
570	}
571
572	pub fn unhit(self: *Manifest, bin_digest: BinDigest, input_file_count: usize) void {
573	// Reset the hash.
574	self.hash.hasher = hasher_init;
575	self.hash.hasher.update(&bin_digest);
576
577	// Remove files not in the initial hash.
578	for (self.files.items[input_file_count..]) \|*file\| {
579	file.deinit(self.cache.gpa);
580	}
581	self.files.shrinkRetainingCapacity(input_file_count);
582
583	for (self.files.items) \|file\| {
584	self.hash.hasher.update(&file.bin_digest);
585	}
586	}
587
588	fn isProblematicTimestamp(man: *Manifest, file_time: i128) bool {
589	// If the file_time is prior to the most recent problematic timestamp
590	// then we don't need to access the filesystem.
591	if (file_time < man.recent_problematic_timestamp)
592	return false;
593
594	// Next we will check the globally shared Cache timestamp, which is accessed
595	// from multiple threads.
596	man.cache.mutex.lock();
597	defer man.cache.mutex.unlock();
598
599	// Save the global one to our local one to avoid locking next time.
600	man.recent_problematic_timestamp = man.cache.recent_problematic_timestamp;
601	if (file_time < man.recent_problematic_timestamp)
602	return false;
603
604	// This flag prevents multiple filesystem writes for the same hit() call.
605	if (man.want_refresh_timestamp) {
606	man.want_refresh_timestamp = false;
607
608	var file = man.cache.manifest_dir.createFile("timestamp", .{
609	.read = true,
610	.truncate = true,
611	}) catch return true;
612	defer file.close();
613
614	// Save locally and also save globally (we still hold the global lock).
615	man.recent_problematic_timestamp = (file.stat() catch return true).mtime;
616	man.cache.recent_problematic_timestamp = man.recent_problematic_timestamp;
617	}
618
619	return file_time >= man.recent_problematic_timestamp;
620	}
621
622	fn populateFileHash(self: Manifest, ch_file: File) !void {
623	const pp = ch_file.prefixed_path.?;
624	const dir = self.cache.prefixes()[pp.prefix].handle;
625	const file = try dir.openFile(pp.sub_path, .{});
626	defer file.close();
627
628	const actual_stat = try file.stat();
629	ch_file.stat = .{
630	.size = actual_stat.size,
631	.mtime = actual_stat.mtime,
632	.inode = actual_stat.inode,
633	};
634
635	if (self.isProblematicTimestamp(ch_file.stat.mtime)) {
636	// The actual file has an unreliable timestamp, force it to be hashed
637	ch_file.stat.mtime = 0;
638	ch_file.stat.inode = 0;
639	}
640
641	if (ch_file.max_file_size) \|max_file_size\| {
642	if (ch_file.stat.size > max_file_size) {
643	return error.FileTooBig;
644	}
645
646	const contents = try self.cache.gpa.alloc(u8, @intCast(usize, ch_file.stat.size));
647	errdefer self.cache.gpa.free(contents);
648
649	// Hash while reading from disk, to keep the contents in the cpu cache while
650	// doing hashing.
651	var hasher = hasher_init;
652	var off: usize = 0;
653	while (true) {
654	// give me everything you've got, captain
655	const bytes_read = try file.read(contents[off..]);
656	if (bytes_read == 0) break;
657	hasher.update(contents[off..][0..bytes_read]);
658	off += bytes_read;
659	}
660	hasher.final(&ch_file.bin_digest);
661
662	ch_file.contents = contents;
663	} else {
664	try hashFile(file, &ch_file.bin_digest);
665	}
666
667	self.hash.hasher.update(&ch_file.bin_digest);
668	}
669
670	/// Add a file as a dependency of process being cached, after the initial hash has been
671	/// calculated. This is useful for processes that don't know all the files that
672	/// are depended on ahead of time. For example, a source file that can import other files
673	/// will need to be recompiled if the imported file is changed.
674	pub fn addFilePostFetch(self: *Manifest, file_path: []const u8, max_file_size: usize) ![]const u8 {
675	assert(self.manifest_file != null);
676
677	const gpa = self.cache.gpa;
678	const prefixed_path = try self.cache.findPrefix(file_path);
679	errdefer gpa.free(prefixed_path.sub_path);
680
681	const new_ch_file = try self.files.addOne(gpa);
682	new_ch_file.* = .{
683	.prefixed_path = prefixed_path,
684	.max_file_size = max_file_size,
685	.stat = undefined,
686	.bin_digest = undefined,
687	.contents = null,
688	};
689	errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
690
691	try self.populateFileHash(new_ch_file);
692
693	return new_ch_file.contents.?;
694	}
695
696	/// Add a file as a dependency of process being cached, after the initial hash has been
697	/// calculated. This is useful for processes that don't know the all the files that
698	/// are depended on ahead of time. For example, a source file that can import other files
699	/// will need to be recompiled if the imported file is changed.
700	pub fn addFilePost(self: *Manifest, file_path: []const u8) !void {
701	assert(self.manifest_file != null);
702
703	const gpa = self.cache.gpa;
704	const prefixed_path = try self.cache.findPrefix(file_path);
705	errdefer gpa.free(prefixed_path.sub_path);
706
707	const new_ch_file = try self.files.addOne(gpa);
708	new_ch_file.* = .{
709	.prefixed_path = prefixed_path,
710	.max_file_size = null,
711	.stat = undefined,
712	.bin_digest = undefined,
713	.contents = null,
714	};
715	errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
716
717	try self.populateFileHash(new_ch_file);
718	}
719
720	/// Like `addFilePost` but when the file contents have already been loaded from disk.
721	/// On success, cache takes ownership of `resolved_path`.
722	pub fn addFilePostContents(
723	self: *Manifest,
724	resolved_path: []u8,
725	bytes: []const u8,
726	stat: File.Stat,
727	) error{OutOfMemory}!void {
728	assert(self.manifest_file != null);
729	const gpa = self.cache.gpa;
730
731	const ch_file = try self.files.addOne(gpa);
732	errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
733
734	const prefixed_path = try self.cache.findPrefixResolved(resolved_path);
735	errdefer gpa.free(prefixed_path.sub_path);
736
737	ch_file.* = .{
738	.prefixed_path = prefixed_path,
739	.max_file_size = null,
740	.stat = stat,
741	.bin_digest = undefined,
742	.contents = null,
743	};
744
745	if (self.isProblematicTimestamp(ch_file.stat.mtime)) {
746	// The actual file has an unreliable timestamp, force it to be hashed
747	ch_file.stat.mtime = 0;
748	ch_file.stat.inode = 0;
749	}
750
751	{
752	var hasher = hasher_init;
753	hasher.update(bytes);
754	hasher.final(&ch_file.bin_digest);
755	}
756
757	self.hash.hasher.update(&ch_file.bin_digest);
758	}
759
760	pub fn addDepFilePost(self: *Manifest, dir: fs.Dir, dep_file_basename: []const u8) !void {
761	assert(self.manifest_file != null);
762
763	const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max);
764	defer self.cache.gpa.free(dep_file_contents);
765
766	var error_buf = std.ArrayList(u8).init(self.cache.gpa);
767	defer error_buf.deinit();
768
769	var it: DepTokenizer = .{ .bytes = dep_file_contents };
770
771	// Skip first token: target.
772	switch (it.next() orelse return) { // Empty dep file OK.
773	.target, .target_must_resolve, .prereq => {},
774	else => \|err\| {
775	try err.printError(error_buf.writer());
776	log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
777	return error.InvalidDepFile;
778	},
779	}
780	// Process 0+ preqreqs.
781	// Clang is invoked in single-source mode so we never get more targets.
782	while (true) {
783	switch (it.next() orelse return) {
784	.target, .target_must_resolve => return,
785	.prereq => \|file_path\| try self.addFilePost(file_path),
786	else => \|err\| {
787	try err.printError(error_buf.writer());
788	log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
789	return error.InvalidDepFile;
790	},
791	}
792	}
793	}
794
795	/// Returns a hex encoded hash of the inputs.
796	pub fn final(self: *Manifest) [hex_digest_len]u8 {
797	assert(self.manifest_file != null);
798
799	// We don't close the manifest file yet, because we want to
800	// keep it locked until the API user is done using it.
801	// We also don't write out the manifest yet, because until
802	// cache_release is called we still might be working on creating
803	// the artifacts to cache.
804
805	var bin_digest: BinDigest = undefined;
806	self.hash.hasher.final(&bin_digest);
807
808	var out_digest: [hex_digest_len]u8 = undefined;
809	_ = std.fmt.bufPrint(
810	&out_digest,
811	"{s}",
812	.{std.fmt.fmtSliceHexLower(&bin_digest)},
813	) catch unreachable;
814
815	return out_digest;
816	}
817
818	/// If `want_shared_lock` is true, this function automatically downgrades the
819	/// lock from exclusive to shared.
820	pub fn writeManifest(self: *Manifest) !void {
821	assert(self.have_exclusive_lock);
822
823	const manifest_file = self.manifest_file.?;
824	if (self.manifest_dirty) {
825	self.manifest_dirty = false;
826
827	var contents = std.ArrayList(u8).init(self.cache.gpa);
828	defer contents.deinit();
829
830	const writer = contents.writer();
831	var encoded_digest: [hex_digest_len]u8 = undefined;
832
833	for (self.files.items) \|file\| {
834	_ = std.fmt.bufPrint(
835	&encoded_digest,
836	"{s}",
837	.{std.fmt.fmtSliceHexLower(&file.bin_digest)},
838	) catch unreachable;
839	try writer.print("{d} {d} {d} {s} {d} {s}\n", .{
840	file.stat.size,
841	file.stat.inode,
842	file.stat.mtime,
843	&encoded_digest,
844	file.prefixed_path.?.prefix,
845	file.prefixed_path.?.sub_path,
846	});
847	}
848
849	try manifest_file.setEndPos(contents.items.len);
850	try manifest_file.pwriteAll(contents.items, 0);
851	}
852
853	if (self.want_shared_lock) {
854	try self.downgradeToSharedLock();
855	}
856	}
857
858	fn downgradeToSharedLock(self: *Manifest) !void {
859	if (!self.have_exclusive_lock) return;
860
861	// WASI does not currently support flock, so we bypass it here.
862	// TODO: If/when flock is supported on WASI, this check should be removed.
863	// See https://github.com/WebAssembly/wasi-filesystem/issues/2
864	if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
865	const manifest_file = self.manifest_file.?;
866	try manifest_file.downgradeLock();
867	}
868
869	self.have_exclusive_lock = false;
870	}
871
872	fn upgradeToExclusiveLock(self: *Manifest) !void {
873	if (self.have_exclusive_lock) return;
874	assert(self.manifest_file != null);
875
876	// WASI does not currently support flock, so we bypass it here.
877	// TODO: If/when flock is supported on WASI, this check should be removed.
878	// See https://github.com/WebAssembly/wasi-filesystem/issues/2
879	if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
880	const manifest_file = self.manifest_file.?;
881	// Here we intentionally have a period where the lock is released, in case there are
882	// other processes holding a shared lock.
883	manifest_file.unlock();
884	try manifest_file.lock(.Exclusive);
885	}
886	self.have_exclusive_lock = true;
887	}
888
889	/// Obtain only the data needed to maintain a lock on the manifest file.
890	/// The `Manifest` remains safe to deinit.
891	/// Don't forget to call `writeManifest` before this!
892	pub fn toOwnedLock(self: *Manifest) Lock {
893	const lock: Lock = .{
894	.manifest_file = self.manifest_file.?,
895	};
896
897	self.manifest_file = null;
898	return lock;
899	}
900
901	/// Releases the manifest file and frees any memory the Manifest was using.
902	/// `Manifest.hit` must be called first.
903	/// Don't forget to call `writeManifest` before this!
904	pub fn deinit(self: *Manifest) void {
905	if (self.manifest_file) \|file\| {
906	if (builtin.os.tag == .windows) {
907	// See Lock.release for why this is required on Windows
908	file.unlock();
909	}
910
911	file.close();
912	}
913	for (self.files.items) \|*file\| {
914	file.deinit(self.cache.gpa);
915	}
916	self.files.deinit(self.cache.gpa);
917	}
918	};
919
920	/// On operating systems that support symlinks, does a readlink. On other operating systems,
921	/// uses the file contents. Windows supports symlinks but only with elevated privileges, so
922	/// it is treated as not supporting symlinks.
923	pub fn readSmallFile(dir: fs.Dir, sub_path: []const u8, buffer: []u8) ![]u8 {
924	if (builtin.os.tag == .windows) {
925	return dir.readFile(sub_path, buffer);
926	} else {
927	return dir.readLink(sub_path, buffer);
928	}
929	}
930
931	/// On operating systems that support symlinks, does a symlink. On other operating systems,
932	/// uses the file contents. Windows supports symlinks but only with elevated privileges, so
933	/// it is treated as not supporting symlinks.
934	/// `data` must be a valid UTF-8 encoded file path and 255 bytes or fewer.
935	pub fn writeSmallFile(dir: fs.Dir, sub_path: []const u8, data: []const u8) !void {
936	assert(data.len <= 255);
937	if (builtin.os.tag == .windows) {
938	return dir.writeFile(sub_path, data);
939	} else {
940	return dir.symLink(data, sub_path, .{});
941	}
942	}
943
944	fn hashFile(file: fs.File, bin_digest: *[Hasher.mac_length]u8) !void {
945	var buf: [1024]u8 = undefined;
946
947	var hasher = hasher_init;
948	while (true) {
949	const bytes_read = try file.read(&buf);
950	if (bytes_read == 0) break;
951	hasher.update(buf[0..bytes_read]);
952	}
953
954	hasher.final(bin_digest);
955	}
956
957	// Create/Write a file, close it, then grab its stat.mtime timestamp.
958	fn testGetCurrentFileTimestamp() !i128 {
959	var file = try fs.cwd().createFile("test-filetimestamp.tmp", .{
960	.read = true,
961	.truncate = true,
962	});
963	defer file.close();
964
965	return (try file.stat()).mtime;
966	}
967
968	test "cache file and then recall it" {
969	if (builtin.os.tag == .wasi) {
970	// https://github.com/ziglang/zig/issues/5437
971	return error.SkipZigTest;
972	}
973
974	const cwd = fs.cwd();
975
976	const temp_file = "test.txt";
977	const temp_manifest_dir = "temp_manifest_dir";
978
979	try cwd.writeFile(temp_file, "Hello, world!\n");
980
981	// Wait for file timestamps to tick
982	const initial_time = try testGetCurrentFileTimestamp();
983	while ((try testGetCurrentFileTimestamp()) == initial_time) {
984	std.time.sleep(1);
985	}
986
987	var digest1: [hex_digest_len]u8 = undefined;
988	var digest2: [hex_digest_len]u8 = undefined;
989
990	{
991	var cache = Cache{
992	.gpa = testing.allocator,
993	.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
994	};
995	cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
996	defer cache.manifest_dir.close();
997
998	{
999	var ch = cache.obtain();
1000	defer ch.deinit();
1001
1002	ch.hash.add(true);
1003	ch.hash.add(@as(u16, 1234));
1004	ch.hash.addBytes("1234");
1005	_ = try ch.addFile(temp_file, null);
1006
1007	// There should be nothing in the cache
1008	try testing.expectEqual(false, try ch.hit());
1009
1010	digest1 = ch.final();
1011	try ch.writeManifest();
1012	}
1013	{
1014	var ch = cache.obtain();
1015	defer ch.deinit();
1016
1017	ch.hash.add(true);
1018	ch.hash.add(@as(u16, 1234));
1019	ch.hash.addBytes("1234");
1020	_ = try ch.addFile(temp_file, null);
1021
1022	// Cache hit! We just "built" the same file
1023	try testing.expect(try ch.hit());
1024	digest2 = ch.final();
1025
1026	try testing.expectEqual(false, ch.have_exclusive_lock);
1027	}
1028
1029	try testing.expectEqual(digest1, digest2);
1030	}
1031
1032	try cwd.deleteTree(temp_manifest_dir);
1033	try cwd.deleteFile(temp_file);
1034	}
1035
1036	test "check that changing a file makes cache fail" {
1037	if (builtin.os.tag == .wasi) {
1038	// https://github.com/ziglang/zig/issues/5437
1039	return error.SkipZigTest;
1040	}
1041	const cwd = fs.cwd();
1042
1043	const temp_file = "cache_hash_change_file_test.txt";
1044	const temp_manifest_dir = "cache_hash_change_file_manifest_dir";
1045	const original_temp_file_contents = "Hello, world!\n";
1046	const updated_temp_file_contents = "Hello, world; but updated!\n";
1047
1048	try cwd.deleteTree(temp_manifest_dir);
1049	try cwd.deleteTree(temp_file);
1050
1051	try cwd.writeFile(temp_file, original_temp_file_contents);
1052
1053	// Wait for file timestamps to tick
1054	const initial_time = try testGetCurrentFileTimestamp();
1055	while ((try testGetCurrentFileTimestamp()) == initial_time) {
1056	std.time.sleep(1);
1057	}
1058
1059	var digest1: [hex_digest_len]u8 = undefined;
1060	var digest2: [hex_digest_len]u8 = undefined;
1061
1062	{
1063	var cache = Cache{
1064	.gpa = testing.allocator,
1065	.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
1066	};
1067	cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
1068	defer cache.manifest_dir.close();
1069
1070	{
1071	var ch = cache.obtain();
1072	defer ch.deinit();
1073
1074	ch.hash.addBytes("1234");
1075	const temp_file_idx = try ch.addFile(temp_file, 100);
1076
1077	// There should be nothing in the cache
1078	try testing.expectEqual(false, try ch.hit());
1079
1080	try testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
1081
1082	digest1 = ch.final();
1083
1084	try ch.writeManifest();
1085	}
1086
1087	try cwd.writeFile(temp_file, updated_temp_file_contents);
1088
1089	{
1090	var ch = cache.obtain();
1091	defer ch.deinit();
1092
1093	ch.hash.addBytes("1234");
1094	const temp_file_idx = try ch.addFile(temp_file, 100);
1095
1096	// A file that we depend on has been updated, so the cache should not contain an entry for it
1097	try testing.expectEqual(false, try ch.hit());
1098
1099	// The cache system does not keep the contents of re-hashed input files.
1100	try testing.expect(ch.files.items[temp_file_idx].contents == null);
1101
1102	digest2 = ch.final();
1103
1104	try ch.writeManifest();
1105	}
1106
1107	try testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));
1108	}
1109
1110	try cwd.deleteTree(temp_manifest_dir);
1111	try cwd.deleteTree(temp_file);
1112	}
1113
1114	test "no file inputs" {
1115	if (builtin.os.tag == .wasi) {
1116	// https://github.com/ziglang/zig/issues/5437
1117	return error.SkipZigTest;
1118	}
1119	const cwd = fs.cwd();
1120	const temp_manifest_dir = "no_file_inputs_manifest_dir";
1121	defer cwd.deleteTree(temp_manifest_dir) catch {};
1122
1123	var digest1: [hex_digest_len]u8 = undefined;
1124	var digest2: [hex_digest_len]u8 = undefined;
1125
1126	var cache = Cache{
1127	.gpa = testing.allocator,
1128	.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
1129	};
1130	cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
1131	defer cache.manifest_dir.close();
1132
1133	{
1134	var man = cache.obtain();
1135	defer man.deinit();
1136
1137	man.hash.addBytes("1234");
1138
1139	// There should be nothing in the cache
1140	try testing.expectEqual(false, try man.hit());
1141
1142	digest1 = man.final();
1143
1144	try man.writeManifest();
1145	}
1146	{
1147	var man = cache.obtain();
1148	defer man.deinit();
1149
1150	man.hash.addBytes("1234");
1151
1152	try testing.expect(try man.hit());
1153	digest2 = man.final();
1154	try testing.expectEqual(false, man.have_exclusive_lock);
1155	}
1156
1157	try testing.expectEqual(digest1, digest2);
1158	}
1159
1160	test "Manifest with files added after initial hash work" {
1161	if (builtin.os.tag == .wasi) {
1162	// https://github.com/ziglang/zig/issues/5437
1163	return error.SkipZigTest;
1164	}
1165	const cwd = fs.cwd();
1166
1167	const temp_file1 = "cache_hash_post_file_test1.txt";
1168	const temp_file2 = "cache_hash_post_file_test2.txt";
1169	const temp_manifest_dir = "cache_hash_post_file_manifest_dir";
1170
1171	try cwd.writeFile(temp_file1, "Hello, world!\n");
1172	try cwd.writeFile(temp_file2, "Hello world the second!\n");
1173
1174	// Wait for file timestamps to tick
1175	const initial_time = try testGetCurrentFileTimestamp();
1176	while ((try testGetCurrentFileTimestamp()) == initial_time) {
1177	std.time.sleep(1);
1178	}
1179
1180	var digest1: [hex_digest_len]u8 = undefined;
1181	var digest2: [hex_digest_len]u8 = undefined;
1182	var digest3: [hex_digest_len]u8 = undefined;
1183
1184	{
1185	var cache = Cache{
1186	.gpa = testing.allocator,
1187	.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
1188	};
1189	cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
1190	defer cache.manifest_dir.close();
1191
1192	{
1193	var ch = cache.obtain();
1194	defer ch.deinit();
1195
1196	ch.hash.addBytes("1234");
1197	_ = try ch.addFile(temp_file1, null);
1198
1199	// There should be nothing in the cache
1200	try testing.expectEqual(false, try ch.hit());
1201
1202	_ = try ch.addFilePost(temp_file2);
1203
1204	digest1 = ch.final();
1205	try ch.writeManifest();
1206	}
1207	{
1208	var ch = cache.obtain();
1209	defer ch.deinit();
1210
1211	ch.hash.addBytes("1234");
1212	_ = try ch.addFile(temp_file1, null);
1213
1214	try testing.expect(try ch.hit());
1215	digest2 = ch.final();
1216
1217	try testing.expectEqual(false, ch.have_exclusive_lock);
1218	}
1219	try testing.expect(mem.eql(u8, &digest1, &digest2));
1220
1221	// Modify the file added after initial hash
1222	try cwd.writeFile(temp_file2, "Hello world the second, updated\n");
1223
1224	// Wait for file timestamps to tick
1225	const initial_time2 = try testGetCurrentFileTimestamp();
1226	while ((try testGetCurrentFileTimestamp()) == initial_time2) {
1227	std.time.sleep(1);
1228	}
1229
1230	{
1231	var ch = cache.obtain();
1232	defer ch.deinit();
1233
1234	ch.hash.addBytes("1234");
1235	_ = try ch.addFile(temp_file1, null);
1236
1237	// A file that we depend on has been updated, so the cache should not contain an entry for it
1238	try testing.expectEqual(false, try ch.hit());
1239
1240	_ = try ch.addFilePost(temp_file2);
1241
1242	digest3 = ch.final();
1243
1244	try ch.writeManifest();
1245	}
1246
1247	try testing.expect(!mem.eql(u8, &digest1, &digest3));
1248	}
1249
1250	try cwd.deleteTree(temp_manifest_dir);
1251	try cwd.deleteFile(temp_file1);
1252	try cwd.deleteFile(temp_file2);
1253	}

lib/std/Build/Cache/DepTokenizer.zig created+1069

...	@@ -0,0 +1,1069 @@
1	const Tokenizer = @This();
2
3	index: usize = 0,
4	bytes: []const u8,
5	state: State = .lhs,
6
7	const std = @import("std");
8	const testing = std.testing;
9	const assert = std.debug.assert;
10
11	pub fn next(self: *Tokenizer) ?Token {
12	var start = self.index;
13	var must_resolve = false;
14	while (self.index < self.bytes.len) {
15	const char = self.bytes[self.index];
16	switch (self.state) {
17	.lhs => switch (char) {
18	'\t', '\n', '\r', ' ' => {
19	// silently ignore whitespace
20	self.index += 1;
21	},
22	else => {
23	start = self.index;
24	self.state = .target;
25	},
26	},
27	.target => switch (char) {
28	'\t', '\n', '\r', ' ' => {
29	return errorIllegalChar(.invalid_target, self.index, char);
30	},
31	'$' => {
32	self.state = .target_dollar_sign;
33	self.index += 1;
34	},
35	'\\' => {
36	self.state = .target_reverse_solidus;
37	self.index += 1;
38	},
39	':' => {
40	self.state = .target_colon;
41	self.index += 1;
42	},
43	else => {
44	self.index += 1;
45	},
46	},
47	.target_reverse_solidus => switch (char) {
48	'\t', '\n', '\r' => {
49	return errorIllegalChar(.bad_target_escape, self.index, char);
50	},
51	' ', '#', '\\' => {
52	must_resolve = true;
53	self.state = .target;
54	self.index += 1;
55	},
56	'$' => {
57	self.state = .target_dollar_sign;
58	self.index += 1;
59	},
60	else => {
61	self.state = .target;
62	self.index += 1;
63	},
64	},
65	.target_dollar_sign => switch (char) {
66	'$' => {
67	must_resolve = true;
68	self.state = .target;
69	self.index += 1;
70	},
71	else => {
72	return errorIllegalChar(.expected_dollar_sign, self.index, char);
73	},
74	},
75	.target_colon => switch (char) {
76	'\n', '\r' => {
77	const bytes = self.bytes[start .. self.index - 1];
78	if (bytes.len != 0) {
79	self.state = .lhs;
80	return finishTarget(must_resolve, bytes);
81	}
82	// silently ignore null target
83	self.state = .lhs;
84	},
85	'/', '\\' => {
86	self.state = .target_colon_reverse_solidus;
87	self.index += 1;
88	},
89	else => {
90	const bytes = self.bytes[start .. self.index - 1];
91	if (bytes.len != 0) {
92	self.state = .rhs;
93	return finishTarget(must_resolve, bytes);
94	}
95	// silently ignore null target
96	self.state = .lhs;
97	},
98	},
99	.target_colon_reverse_solidus => switch (char) {
100	'\n', '\r' => {
101	const bytes = self.bytes[start .. self.index - 2];
102	if (bytes.len != 0) {
103	self.state = .lhs;
104	return finishTarget(must_resolve, bytes);
105	}
106	// silently ignore null target
107	self.state = .lhs;
108	},
109	else => {
110	self.state = .target;
111	},
112	},
113	.rhs => switch (char) {
114	'\t', ' ' => {
115	// silently ignore horizontal whitespace
116	self.index += 1;
117	},
118	'\n', '\r' => {
119	self.state = .lhs;
120	},
121	'\\' => {
122	self.state = .rhs_continuation;
123	self.index += 1;
124	},
125	'"' => {
126	self.state = .prereq_quote;
127	self.index += 1;
128	start = self.index;
129	},
130	else => {
131	start = self.index;
132	self.state = .prereq;
133	},
134	},
135	.rhs_continuation => switch (char) {
136	'\n' => {
137	self.state = .rhs;
138	self.index += 1;
139	},
140	'\r' => {
141	self.state = .rhs_continuation_linefeed;
142	self.index += 1;
143	},
144	else => {
145	return errorIllegalChar(.continuation_eol, self.index, char);
146	},
147	},
148	.rhs_continuation_linefeed => switch (char) {
149	'\n' => {
150	self.state = .rhs;
151	self.index += 1;
152	},
153	else => {
154	return errorIllegalChar(.continuation_eol, self.index, char);
155	},
156	},
157	.prereq_quote => switch (char) {
158	'"' => {
159	self.index += 1;
160	self.state = .rhs;
161	return Token{ .prereq = self.bytes[start .. self.index - 1] };
162	},
163	else => {
164	self.index += 1;
165	},
166	},
167	.prereq => switch (char) {
168	'\t', ' ' => {
169	self.state = .rhs;
170	return Token{ .prereq = self.bytes[start..self.index] };
171	},
172	'\n', '\r' => {
173	self.state = .lhs;
174	return Token{ .prereq = self.bytes[start..self.index] };
175	},
176	'\\' => {
177	self.state = .prereq_continuation;
178	self.index += 1;
179	},
180	else => {
181	self.index += 1;
182	},
183	},
184	.prereq_continuation => switch (char) {
185	'\n' => {
186	self.index += 1;
187	self.state = .rhs;
188	return Token{ .prereq = self.bytes[start .. self.index - 2] };
189	},
190	'\r' => {
191	self.state = .prereq_continuation_linefeed;
192	self.index += 1;
193	},
194	else => {
195	// not continuation
196	self.state = .prereq;
197	self.index += 1;
198	},
199	},
200	.prereq_continuation_linefeed => switch (char) {
201	'\n' => {
202	self.index += 1;
203	self.state = .rhs;
204	return Token{ .prereq = self.bytes[start .. self.index - 1] };
205	},
206	else => {
207	return errorIllegalChar(.continuation_eol, self.index, char);
208	},
209	},
210	}
211	} else {
212	switch (self.state) {
213	.lhs,
214	.rhs,
215	.rhs_continuation,
216	.rhs_continuation_linefeed,
217	=> return null,
218	.target => {
219	return errorPosition(.incomplete_target, start, self.bytes[start..]);
220	},
221	.target_reverse_solidus,
222	.target_dollar_sign,
223	=> {
224	const idx = self.index - 1;
225	return errorIllegalChar(.incomplete_escape, idx, self.bytes[idx]);
226	},
227	.target_colon => {
228	const bytes = self.bytes[start .. self.index - 1];
229	if (bytes.len != 0) {
230	self.index += 1;
231	self.state = .rhs;
232	return finishTarget(must_resolve, bytes);
233	}
234	// silently ignore null target
235	self.state = .lhs;
236	return null;
237	},
238	.target_colon_reverse_solidus => {
239	const bytes = self.bytes[start .. self.index - 2];
240	if (bytes.len != 0) {
241	self.index += 1;
242	self.state = .rhs;
243	return finishTarget(must_resolve, bytes);
244	}
245	// silently ignore null target
246	self.state = .lhs;
247	return null;
248	},
249	.prereq_quote => {
250	return errorPosition(.incomplete_quoted_prerequisite, start, self.bytes[start..]);
251	},
252	.prereq => {
253	self.state = .lhs;
254	return Token{ .prereq = self.bytes[start..] };
255	},
256	.prereq_continuation => {
257	self.state = .lhs;
258	return Token{ .prereq = self.bytes[start .. self.index - 1] };
259	},
260	.prereq_continuation_linefeed => {
261	self.state = .lhs;
262	return Token{ .prereq = self.bytes[start .. self.index - 2] };
263	},
264	}
265	}
266	unreachable;
267	}
268
269	fn errorPosition(comptime id: std.meta.Tag(Token), index: usize, bytes: []const u8) Token {
270	return @unionInit(Token, @tagName(id), .{ .index = index, .bytes = bytes });
271	}
272
273	fn errorIllegalChar(comptime id: std.meta.Tag(Token), index: usize, char: u8) Token {
274	return @unionInit(Token, @tagName(id), .{ .index = index, .char = char });
275	}
276
277	fn finishTarget(must_resolve: bool, bytes: []const u8) Token {
278	return if (must_resolve) .{ .target_must_resolve = bytes } else .{ .target = bytes };
279	}
280
281	const State = enum {
282	lhs,
283	target,
284	target_reverse_solidus,
285	target_dollar_sign,
286	target_colon,
287	target_colon_reverse_solidus,
288	rhs,
289	rhs_continuation,
290	rhs_continuation_linefeed,
291	prereq_quote,
292	prereq,
293	prereq_continuation,
294	prereq_continuation_linefeed,
295	};
296
297	pub const Token = union(enum) {
298	target: []const u8,
299	target_must_resolve: []const u8,
300	prereq: []const u8,
301
302	incomplete_quoted_prerequisite: IndexAndBytes,
303	incomplete_target: IndexAndBytes,
304
305	invalid_target: IndexAndChar,
306	bad_target_escape: IndexAndChar,
307	expected_dollar_sign: IndexAndChar,
308	continuation_eol: IndexAndChar,
309	incomplete_escape: IndexAndChar,
310
311	pub const IndexAndChar = struct {
312	index: usize,
313	char: u8,
314	};
315
316	pub const IndexAndBytes = struct {
317	index: usize,
318	bytes: []const u8,
319	};
320
321	/// Resolve escapes in target. Only valid with .target_must_resolve.
322	pub fn resolve(self: Token, writer: anytype) @TypeOf(writer).Error!void {
323	const bytes = self.target_must_resolve; // resolve called on incorrect token
324
325	var state: enum { start, escape, dollar } = .start;
326	for (bytes) \|c\| {
327	switch (state) {
328	.start => {
329	switch (c) {
330	'\\' => state = .escape,
331	'$' => state = .dollar,
332	else => try writer.writeByte(c),
333	}
334	},
335	.escape => {
336	switch (c) {
337	' ', '#', '\\' => {},
338	'$' => {
339	try writer.writeByte('\\');
340	state = .dollar;
341	continue;
342	},
343	else => try writer.writeByte('\\'),
344	}
345	try writer.writeByte(c);
346	state = .start;
347	},
348	.dollar => {
349	try writer.writeByte('$');
350	switch (c) {
351	'$' => {},
352	else => try writer.writeByte(c),
353	}
354	state = .start;
355	},
356	}
357	}
358	}
359
360	pub fn printError(self: Token, writer: anytype) @TypeOf(writer).Error!void {
361	switch (self) {
362	.target, .target_must_resolve, .prereq => unreachable, // not an error
363	.incomplete_quoted_prerequisite,
364	.incomplete_target,
365	=> \|index_and_bytes\| {
366	try writer.print("{s} '", .{self.errStr()});
367	if (self == .incomplete_target) {
368	const tmp = Token{ .target_must_resolve = index_and_bytes.bytes };
369	try tmp.resolve(writer);
370	} else {
371	try printCharValues(writer, index_and_bytes.bytes);
372	}
373	try writer.print("' at position {d}", .{index_and_bytes.index});
374	},
375	.invalid_target,
376	.bad_target_escape,
377	.expected_dollar_sign,
378	.continuation_eol,
379	.incomplete_escape,
380	=> \|index_and_char\| {
381	try writer.writeAll("illegal char ");
382	try printUnderstandableChar(writer, index_and_char.char);
383	try writer.print(" at position {d}: {s}", .{ index_and_char.index, self.errStr() });
384	},
385	}
386	}
387
388	fn errStr(self: Token) []const u8 {
389	return switch (self) {
390	.target, .target_must_resolve, .prereq => unreachable, // not an error
391	.incomplete_quoted_prerequisite => "incomplete quoted prerequisite",
392	.incomplete_target => "incomplete target",
393	.invalid_target => "invalid target",
394	.bad_target_escape => "bad target escape",
395	.expected_dollar_sign => "expecting '$'",
396	.continuation_eol => "continuation expecting end-of-line",
397	.incomplete_escape => "incomplete escape",
398	};
399	}
400	};
401
402	test "empty file" {
403	try depTokenizer("", "");
404	}
405
406	test "empty whitespace" {
407	try depTokenizer("\n", "");
408	try depTokenizer("\r", "");
409	try depTokenizer("\r\n", "");
410	try depTokenizer(" ", "");
411	}
412
413	test "empty colon" {
414	try depTokenizer(":", "");
415	try depTokenizer("\n:", "");
416	try depTokenizer("\r:", "");
417	try depTokenizer("\r\n:", "");
418	try depTokenizer(" :", "");
419	}
420
421	test "empty target" {
422	try depTokenizer("foo.o:", "target = {foo.o}");
423	try depTokenizer(
424	\\foo.o:
425	\\bar.o:
426	\\abcd.o:
427	,
428	\\target = {foo.o}
429	\\target = {bar.o}
430	\\target = {abcd.o}
431	);
432	}
433
434	test "whitespace empty target" {
435	try depTokenizer("\nfoo.o:", "target = {foo.o}");
436	try depTokenizer("\rfoo.o:", "target = {foo.o}");
437	try depTokenizer("\r\nfoo.o:", "target = {foo.o}");
438	try depTokenizer(" foo.o:", "target = {foo.o}");
439	}
440
441	test "escape empty target" {
442	try depTokenizer("\\ foo.o:", "target = { foo.o}");
443	try depTokenizer("\\#foo.o:", "target = {#foo.o}");
444	try depTokenizer("\\\\foo.o:", "target = {\\foo.o}");
445	try depTokenizer("$$foo.o:", "target = {$foo.o}");
446	}
447
448	test "empty target linefeeds" {
449	try depTokenizer("\n", "");
450	try depTokenizer("\r\n", "");
451
452	const expect = "target = {foo.o}";
453	try depTokenizer(
454	\\foo.o:
455	, expect);
456	try depTokenizer(
457	\\foo.o:
458	\\
459	, expect);
460	try depTokenizer(
461	\\foo.o:
462	, expect);
463	try depTokenizer(
464	\\foo.o:
465	\\
466	, expect);
467	}
468
469	test "empty target linefeeds + continuations" {
470	const expect = "target = {foo.o}";
471	try depTokenizer(
472	\\foo.o:\
473	, expect);
474	try depTokenizer(
475	\\foo.o:\
476	\\
477	, expect);
478	try depTokenizer(
479	\\foo.o:\
480	, expect);
481	try depTokenizer(
482	\\foo.o:\
483	\\
484	, expect);
485	}
486
487	test "empty target linefeeds + hspace + continuations" {
488	const expect = "target = {foo.o}";
489	try depTokenizer(
490	\\foo.o: \
491	, expect);
492	try depTokenizer(
493	\\foo.o: \
494	\\
495	, expect);
496	try depTokenizer(
497	\\foo.o: \
498	, expect);
499	try depTokenizer(
500	\\foo.o: \
501	\\
502	, expect);
503	}
504
505	test "prereq" {
506	const expect =
507	\\target = {foo.o}
508	\\prereq = {foo.c}
509	;
510	try depTokenizer("foo.o: foo.c", expect);
511	try depTokenizer(
512	\\foo.o: \
513	\\foo.c
514	, expect);
515	try depTokenizer(
516	\\foo.o: \
517	\\ foo.c
518	, expect);
519	try depTokenizer(
520	\\foo.o: \
521	\\ foo.c
522	, expect);
523	}
524
525	test "prereq continuation" {
526	const expect =
527	\\target = {foo.o}
528	\\prereq = {foo.h}
529	\\prereq = {bar.h}
530	;
531	try depTokenizer(
532	\\foo.o: foo.h\
533	\\bar.h
534	, expect);
535	try depTokenizer(
536	\\foo.o: foo.h\
537	\\bar.h
538	, expect);
539	}
540
541	test "multiple prereqs" {
542	const expect =
543	\\target = {foo.o}
544	\\prereq = {foo.c}
545	\\prereq = {foo.h}
546	\\prereq = {bar.h}
547	;
548	try depTokenizer("foo.o: foo.c foo.h bar.h", expect);
549	try depTokenizer(
550	\\foo.o: \
551	\\foo.c foo.h bar.h
552	, expect);
553	try depTokenizer(
554	\\foo.o: foo.c foo.h bar.h\
555	, expect);
556	try depTokenizer(
557	\\foo.o: foo.c foo.h bar.h\
558	\\
559	, expect);
560	try depTokenizer(
561	\\foo.o: \
562	\\foo.c \
563	\\ foo.h\
564	\\bar.h
565	\\
566	, expect);
567	try depTokenizer(
568	\\foo.o: \
569	\\foo.c \
570	\\ foo.h\
571	\\bar.h\
572	\\
573	, expect);
574	try depTokenizer(
575	\\foo.o: \
576	\\foo.c \
577	\\ foo.h\
578	\\bar.h\
579	, expect);
580	}
581
582	test "multiple targets and prereqs" {
583	try depTokenizer(
584	\\foo.o: foo.c
585	\\bar.o: bar.c a.h b.h c.h
586	\\abc.o: abc.c \
587	\\ one.h two.h \
588	\\ three.h four.h
589	,
590	\\target = {foo.o}
591	\\prereq = {foo.c}
592	\\target = {bar.o}
593	\\prereq = {bar.c}
594	\\prereq = {a.h}
595	\\prereq = {b.h}
596	\\prereq = {c.h}
597	\\target = {abc.o}
598	\\prereq = {abc.c}
599	\\prereq = {one.h}
600	\\prereq = {two.h}
601	\\prereq = {three.h}
602	\\prereq = {four.h}
603	);
604	try depTokenizer(
605	\\ascii.o: ascii.c
606	\\base64.o: base64.c stdio.h
607	\\elf.o: elf.c a.h b.h c.h
608	\\macho.o: \
609	\\ macho.c\
610	\\ a.h b.h c.h
611	,
612	\\target = {ascii.o}
613	\\prereq = {ascii.c}
614	\\target = {base64.o}
615	\\prereq = {base64.c}
616	\\prereq = {stdio.h}
617	\\target = {elf.o}
618	\\prereq = {elf.c}
619	\\prereq = {a.h}
620	\\prereq = {b.h}
621	\\prereq = {c.h}
622	\\target = {macho.o}
623	\\prereq = {macho.c}
624	\\prereq = {a.h}
625	\\prereq = {b.h}
626	\\prereq = {c.h}
627	);
628	try depTokenizer(
629	\\a$$scii.o: ascii.c
630	\\\\base64.o: "\base64.c" "s t#dio.h"
631	\\e\\lf.o: "e\lf.c" "a.h$$" "$$b.h c.h$$"
632	\\macho.o: \
633	\\ "macho!.c" \
634	\\ a.h b.h c.h
635	,
636	\\target = {a$scii.o}
637	\\prereq = {ascii.c}
638	\\target = {\base64.o}
639	\\prereq = {\base64.c}
640	\\prereq = {s t#dio.h}
641	\\target = {e\lf.o}
642	\\prereq = {e\lf.c}
643	\\prereq = {a.h$$}
644	\\prereq = {$$b.h c.h$$}
645	\\target = {macho.o}
646	\\prereq = {macho!.c}
647	\\prereq = {a.h}
648	\\prereq = {b.h}
649	\\prereq = {c.h}
650	);
651	}
652
653	test "windows quoted prereqs" {
654	try depTokenizer(
655	\\c:\foo.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo.c"
656	\\c:\foo2.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo2.c" \
657	\\ "C:\Program Files (x86)\Microsoft Visual Studio\foo1.h" \
658	\\ "C:\Program Files (x86)\Microsoft Visual Studio\foo2.h"
659	,
660	\\target = {c:\foo.o}
661	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo.c}
662	\\target = {c:\foo2.o}
663	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.c}
664	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo1.h}
665	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.h}
666	);
667	}
668
669	test "windows mixed prereqs" {
670	try depTokenizer(
671	\\cimport.o: \
672	\\ C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h \
673	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h" \
674	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h" \
675	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h" \
676	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h" \
677	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h" \
678	\\ C:\msys64\opt\zig\lib\zig\include\vadefs.h \
679	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h" \
680	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h" \
681	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h" \
682	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h" \
683	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h" \
684	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h" \
685	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h" \
686	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h" \
687	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h"
688	,
689	\\target = {cimport.o}
690	\\prereq = {C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h}
691	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h}
692	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h}
693	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h}
694	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h}
695	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h}
696	\\prereq = {C:\msys64\opt\zig\lib\zig\include\vadefs.h}
697	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h}
698	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h}
699	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h}
700	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h}
701	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h}
702	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h}
703	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h}
704	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h}
705	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h}
706	);
707	}
708
709	test "windows funky targets" {
710	try depTokenizer(
711	\\C:\Users\anon\foo.o:
712	\\C:\Users\anon\foo\ .o:
713	\\C:\Users\anon\foo\#.o:
714	\\C:\Users\anon\foo$$.o:
715	\\C:\Users\anon\\\ foo.o:
716	\\C:\Users\anon\\#foo.o:
717	\\C:\Users\anon\$$foo.o:
718	\\C:\Users\anon\\\ \ \ \ \ foo.o:
719	,
720	\\target = {C:\Users\anon\foo.o}
721	\\target = {C:\Users\anon\foo .o}
722	\\target = {C:\Users\anon\foo#.o}
723	\\target = {C:\Users\anon\foo$.o}
724	\\target = {C:\Users\anon\ foo.o}
725	\\target = {C:\Users\anon\#foo.o}
726	\\target = {C:\Users\anon\$foo.o}
727	\\target = {C:\Users\anon\ foo.o}
728	);
729	}
730
731	test "windows drive and forward slashes" {
732	try depTokenizer(
733	\\C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj: \
734	\\ C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c
735	,
736	\\target = {C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj}
737	\\prereq = {C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c}
738	);
739	}
740
741	test "error incomplete escape - reverse_solidus" {
742	try depTokenizer("\\",
743	\\ERROR: illegal char '\' at position 0: incomplete escape
744	);
745	try depTokenizer("\t\\",
746	\\ERROR: illegal char '\' at position 1: incomplete escape
747	);
748	try depTokenizer("\n\\",
749	\\ERROR: illegal char '\' at position 1: incomplete escape
750	);
751	try depTokenizer("\r\\",
752	\\ERROR: illegal char '\' at position 1: incomplete escape
753	);
754	try depTokenizer("\r\n\\",
755	\\ERROR: illegal char '\' at position 2: incomplete escape
756	);
757	try depTokenizer(" \\",
758	\\ERROR: illegal char '\' at position 1: incomplete escape
759	);
760	}
761
762	test "error incomplete escape - dollar_sign" {
763	try depTokenizer("$",
764	\\ERROR: illegal char '$' at position 0: incomplete escape
765	);
766	try depTokenizer("\t$",
767	\\ERROR: illegal char '$' at position 1: incomplete escape
768	);
769	try depTokenizer("\n$",
770	\\ERROR: illegal char '$' at position 1: incomplete escape
771	);
772	try depTokenizer("\r$",
773	\\ERROR: illegal char '$' at position 1: incomplete escape
774	);
775	try depTokenizer("\r\n$",
776	\\ERROR: illegal char '$' at position 2: incomplete escape
777	);
778	try depTokenizer(" $",
779	\\ERROR: illegal char '$' at position 1: incomplete escape
780	);
781	}
782
783	test "error incomplete target" {
784	try depTokenizer("foo.o",
785	\\ERROR: incomplete target 'foo.o' at position 0
786	);
787	try depTokenizer("\tfoo.o",
788	\\ERROR: incomplete target 'foo.o' at position 1
789	);
790	try depTokenizer("\nfoo.o",
791	\\ERROR: incomplete target 'foo.o' at position 1
792	);
793	try depTokenizer("\rfoo.o",
794	\\ERROR: incomplete target 'foo.o' at position 1
795	);
796	try depTokenizer("\r\nfoo.o",
797	\\ERROR: incomplete target 'foo.o' at position 2
798	);
799	try depTokenizer(" foo.o",
800	\\ERROR: incomplete target 'foo.o' at position 1
801	);
802
803	try depTokenizer("\\ foo.o",
804	\\ERROR: incomplete target ' foo.o' at position 0
805	);
806	try depTokenizer("\\#foo.o",
807	\\ERROR: incomplete target '#foo.o' at position 0
808	);
809	try depTokenizer("\\\\foo.o",
810	\\ERROR: incomplete target '\foo.o' at position 0
811	);
812	try depTokenizer("$$foo.o",
813	\\ERROR: incomplete target '$foo.o' at position 0
814	);
815	}
816
817	test "error illegal char at position - bad target escape" {
818	try depTokenizer("\\\t",
819	\\ERROR: illegal char \x09 at position 1: bad target escape
820	);
821	try depTokenizer("\\\n",
822	\\ERROR: illegal char \x0A at position 1: bad target escape
823	);
824	try depTokenizer("\\\r",
825	\\ERROR: illegal char \x0D at position 1: bad target escape
826	);
827	try depTokenizer("\\\r\n",
828	\\ERROR: illegal char \x0D at position 1: bad target escape
829	);
830	}
831
832	test "error illegal char at position - execting dollar_sign" {
833	try depTokenizer("$\t",
834	\\ERROR: illegal char \x09 at position 1: expecting '$'
835	);
836	try depTokenizer("$\n",
837	\\ERROR: illegal char \x0A at position 1: expecting '$'
838	);
839	try depTokenizer("$\r",
840	\\ERROR: illegal char \x0D at position 1: expecting '$'
841	);
842	try depTokenizer("$\r\n",
843	\\ERROR: illegal char \x0D at position 1: expecting '$'
844	);
845	}
846
847	test "error illegal char at position - invalid target" {
848	try depTokenizer("foo\t.o",
849	\\ERROR: illegal char \x09 at position 3: invalid target
850	);
851	try depTokenizer("foo\n.o",
852	\\ERROR: illegal char \x0A at position 3: invalid target
853	);
854	try depTokenizer("foo\r.o",
855	\\ERROR: illegal char \x0D at position 3: invalid target
856	);
857	try depTokenizer("foo\r\n.o",
858	\\ERROR: illegal char \x0D at position 3: invalid target
859	);
860	}
861
862	test "error target - continuation expecting end-of-line" {
863	try depTokenizer("foo.o: \\\t",
864	\\target = {foo.o}
865	\\ERROR: illegal char \x09 at position 8: continuation expecting end-of-line
866	);
867	try depTokenizer("foo.o: \\ ",
868	\\target = {foo.o}
869	\\ERROR: illegal char ' ' at position 8: continuation expecting end-of-line
870	);
871	try depTokenizer("foo.o: \\x",
872	\\target = {foo.o}
873	\\ERROR: illegal char 'x' at position 8: continuation expecting end-of-line
874	);
875	try depTokenizer("foo.o: \\\x0dx",
876	\\target = {foo.o}
877	\\ERROR: illegal char 'x' at position 9: continuation expecting end-of-line
878	);
879	}
880
881	test "error prereq - continuation expecting end-of-line" {
882	try depTokenizer("foo.o: foo.h\\\x0dx",
883	\\target = {foo.o}
884	\\ERROR: illegal char 'x' at position 14: continuation expecting end-of-line
885	);
886	}
887
888	// - tokenize input, emit textual representation, and compare to expect
889	fn depTokenizer(input: []const u8, expect: []const u8) !void {
890	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
891	const arena = arena_allocator.allocator();
892	defer arena_allocator.deinit();
893
894	var it: Tokenizer = .{ .bytes = input };
895	var buffer = std.ArrayList(u8).init(arena);
896	var resolve_buf = std.ArrayList(u8).init(arena);
897	var i: usize = 0;
898	while (it.next()) \|token\| {
899	if (i != 0) try buffer.appendSlice("\n");
900	switch (token) {
901	.target, .prereq => \|bytes\| {
902	try buffer.appendSlice(@tagName(token));
903	try buffer.appendSlice(" = {");
904	for (bytes) \|b\| {
905	try buffer.append(printable_char_tab[b]);
906	}
907	try buffer.appendSlice("}");
908	},
909	.target_must_resolve => {
910	try buffer.appendSlice("target = {");
911	try token.resolve(resolve_buf.writer());
912	for (resolve_buf.items) \|b\| {
913	try buffer.append(printable_char_tab[b]);
914	}
915	resolve_buf.items.len = 0;
916	try buffer.appendSlice("}");
917	},
918	else => {
919	try buffer.appendSlice("ERROR: ");
920	try token.printError(buffer.writer());
921	break;
922	},
923	}
924	i += 1;
925	}
926
927	if (std.mem.eql(u8, expect, buffer.items)) {
928	try testing.expect(true);
929	return;
930	}
931
932	const out = std.io.getStdErr().writer();
933
934	try out.writeAll("\n");
935	try printSection(out, "<<<< input", input);
936	try printSection(out, "==== expect", expect);
937	try printSection(out, ">>>> got", buffer.items);
938	try printRuler(out);
939
940	try testing.expect(false);
941	}
942
943	fn printSection(out: anytype, label: []const u8, bytes: []const u8) !void {
944	try printLabel(out, label, bytes);
945	try hexDump(out, bytes);
946	try printRuler(out);
947	try out.writeAll(bytes);
948	try out.writeAll("\n");
949	}
950
951	fn printLabel(out: anytype, label: []const u8, bytes: []const u8) !void {
952	var buf: [80]u8 = undefined;
953	var text = try std.fmt.bufPrint(buf[0..], "{s} {d} bytes ", .{ label, bytes.len });
954	try out.writeAll(text);
955	var i: usize = text.len;
956	const end = 79;
957	while (i < end) : (i += 1) {
958	try out.writeAll(&[_]u8{label[0]});
959	}
960	try out.writeAll("\n");
961	}
962
963	fn printRuler(out: anytype) !void {
964	var i: usize = 0;
965	const end = 79;
966	while (i < end) : (i += 1) {
967	try out.writeAll("-");
968	}
969	try out.writeAll("\n");
970	}
971
972	fn hexDump(out: anytype, bytes: []const u8) !void {
973	const n16 = bytes.len >> 4;
974	var line: usize = 0;
975	var offset: usize = 0;
976	while (line < n16) : (line += 1) {
977	try hexDump16(out, offset, bytes[offset .. offset + 16]);
978	offset += 16;
979	}
980
981	const n = bytes.len & 0x0f;
982	if (n > 0) {
983	try printDecValue(out, offset, 8);
984	try out.writeAll(":");
985	try out.writeAll(" ");
986	var end1 = std.math.min(offset + n, offset + 8);
987	for (bytes[offset..end1]) \|b\| {
988	try out.writeAll(" ");
989	try printHexValue(out, b, 2);
990	}
991	var end2 = offset + n;
992	if (end2 > end1) {
993	try out.writeAll(" ");
994	for (bytes[end1..end2]) \|b\| {
995	try out.writeAll(" ");
996	try printHexValue(out, b, 2);
997	}
998	}
999	const short = 16 - n;
1000	var i: usize = 0;
1001	while (i < short) : (i += 1) {
1002	try out.writeAll(" ");
1003	}
1004	if (end2 > end1) {
1005	try out.writeAll(" \|");
1006	} else {
1007	try out.writeAll(" \|");
1008	}
1009	try printCharValues(out, bytes[offset..end2]);
1010	try out.writeAll("\|\n");
1011	offset += n;
1012	}
1013
1014	try printDecValue(out, offset, 8);
1015	try out.writeAll(":");
1016	try out.writeAll("\n");
1017	}
1018
1019	fn hexDump16(out: anytype, offset: usize, bytes: []const u8) !void {
1020	try printDecValue(out, offset, 8);
1021	try out.writeAll(":");
1022	try out.writeAll(" ");
1023	for (bytes[0..8]) \|b\| {
1024	try out.writeAll(" ");
1025	try printHexValue(out, b, 2);
1026	}
1027	try out.writeAll(" ");
1028	for (bytes[8..16]) \|b\| {
1029	try out.writeAll(" ");
1030	try printHexValue(out, b, 2);
1031	}
1032	try out.writeAll(" \|");
1033	try printCharValues(out, bytes);
1034	try out.writeAll("\|\n");
1035	}
1036
1037	fn printDecValue(out: anytype, value: u64, width: u8) !void {
1038	var buffer: [20]u8 = undefined;
1039	const len = std.fmt.formatIntBuf(buffer[0..], value, 10, .lower, .{ .width = width, .fill = '0' });
1040	try out.writeAll(buffer[0..len]);
1041	}
1042
1043	fn printHexValue(out: anytype, value: u64, width: u8) !void {
1044	var buffer: [16]u8 = undefined;
1045	const len = std.fmt.formatIntBuf(buffer[0..], value, 16, .lower, .{ .width = width, .fill = '0' });
1046	try out.writeAll(buffer[0..len]);
1047	}
1048
1049	fn printCharValues(out: anytype, bytes: []const u8) !void {
1050	for (bytes) \|b\| {
1051	try out.writeAll(&[_]u8{printable_char_tab[b]});
1052	}
1053	}
1054
1055	fn printUnderstandableChar(out: anytype, char: u8) !void {
1056	if (std.ascii.isPrint(char)) {
1057	try out.print("'{c}'", .{char});
1058	} else {
1059	try out.print("\\x{X:0>2}", .{char});
1060	}
1061	}
1062
1063	// zig fmt: off
1064	const printable_char_tab: [256]u8 = (
1065	"................................ !\"#$%&'()*+,-./0123456789:;<=>?" ++
1066	"@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{\|}~." ++
1067	"................................................................" ++
1068	"................................................................"
1069	).*;

lib/std/Build/CompileStep.zig+23-41

 @@ -83,7 +83,7 @@ max_memory: ?u64 = null,
 shared_memory: bool = false,
 global_base: ?u64 = null,
 c_std: std.Build.CStd,
-override_lib_dir: ?[]const u8,
+zig_lib_dir: ?[]const u8,
 main_pkg_path: ?[]const u8,
 exec_cmd_args: ?[]const ?[]const u8,
 name_prefix: []const u8,
 @@ -344,7 +344,7 @@ pub fn create(builder: *std.Build, options: Options) *CompileStep {
         .installed_headers = ArrayList(*Step).init(builder.allocator),
         .object_src = undefined,
         .c_std = std.Build.CStd.C99,
-        .override_lib_dir = null,
+        .zig_lib_dir = null,
         .main_pkg_path = null,
         .exec_cmd_args = null,
         .name_prefix = "",
 @@ -506,26 +506,11 @@ pub fn installLibraryHeaders(a: *CompileStep, l: *CompileStep) void {
     a.installed_headers.appendSlice(l.installed_headers.items) catch @panic("OOM");
 }
-/// Creates a `RunStep` with an executable built with `addExecutable`.
+/// Deprecated: use `std.Build.addRunArtifact`
-/// Add command line arguments with `addArg`.
+/// This function will run in the context of the package that created the executable,
+/// which is undesirable when running an executable provided by a dependency package.
 pub fn run(exe: *CompileStep) *RunStep {
-    assert(exe.kind == .exe or exe.kind == .test_exe);
+    return exe.builder.addRunArtifact(exe);
-    // It doesn't have to be native. We catch that if you actually try to run it.
-    // Consider that this is declarative; the run step may not be run unless a user
-    // option is supplied.
-    const run_step = RunStep.create(exe.builder, exe.builder.fmt("run {s}", .{exe.step.name}));
-    run_step.addArtifactArg(exe);
-    if (exe.kind == .test_exe) {
-        run_step.addArg(exe.builder.zig_exe);
-    }
-    if (exe.vcpkg_bin_path) |path| {
-        run_step.addPathDir(path);
-    }
-    return run_step;
 }
 /// Creates an `EmulatableRunStep` with an executable built with `addExecutable`.
 @@ -872,7 +857,7 @@ pub fn setVerboseCC(self: *CompileStep, value: bool) void {
 }
 pub fn overrideZigLibDir(self: *CompileStep, dir_path: []const u8) void {
-    self.override_lib_dir = self.builder.dupePath(dir_path);
+    self.zig_lib_dir = self.builder.dupePath(dir_path);
 }
 pub fn setMainPkgPath(self: *CompileStep, dir_path: []const u8) void {
 @@ -1365,10 +1350,10 @@ fn make(step: *Step) !void {
     }
     try zig_args.append("--cache-dir");
-    try zig_args.append(builder.pathFromRoot(builder.cache_root));
+    try zig_args.append(builder.cache_root.path orelse ".");
     try zig_args.append("--global-cache-dir");
-    try zig_args.append(builder.pathFromRoot(builder.global_cache_root));
+    try zig_args.append(builder.global_cache_root.path orelse ".");
     try zig_args.append("--name");
     try zig_args.append(self.name);
 @@ -1718,12 +1703,12 @@ fn make(step: *Step) !void {
     try addFlag(&zig_args, "each-lib-rpath", self.each_lib_rpath);
     try addFlag(&zig_args, "build-id", self.build_id);
-    if (self.override_lib_dir) |dir| {
+    if (self.zig_lib_dir) |dir| {
         try zig_args.append("--zig-lib-dir");
         try zig_args.append(builder.pathFromRoot(dir));
-    } else if (builder.override_lib_dir) |dir| {
+    } else if (builder.zig_lib_dir) |dir| {
         try zig_args.append("--zig-lib-dir");
-        try zig_args.append(builder.pathFromRoot(dir));
+        try zig_args.append(dir);
     }
     if (self.main_pkg_path) |dir| {
 @@ -1760,23 +1745,15 @@ fn make(step: *Step) !void {
         args_length += arg.len + 1; // +1 to account for null terminator
     }
     if (args_length >= 30 * 1024) {
-        const args_dir = try fs.path.join(
+        try builder.cache_root.handle.makePath("args");
-            builder.allocator,
-            &[_][]const u8{ builder.pathFromRoot("zig-cache"), "args" },
-        );
-        try std.fs.cwd().makePath(args_dir);
-        var args_arena = std.heap.ArenaAllocator.init(builder.allocator);
-        defer args_arena.deinit();
         const args_to_escape = zig_args.items[2..];
-        var escaped_args = try ArrayList([]const u8).initCapacity(args_arena.allocator(), args_to_escape.len);
+        var escaped_args = try ArrayList([]const u8).initCapacity(builder.allocator, args_to_escape.len);
         arg_blk: for (args_to_escape) |arg| {
             for (arg) |c, arg_idx| {
                 if (c == '\\' or c == '"') {
                     // Slow path for arguments that need to be escaped. We'll need to allocate and copy
-                    var escaped = try ArrayList(u8).initCapacity(args_arena.allocator(), arg.len + 1);
+                    var escaped = try ArrayList(u8).initCapacity(builder.allocator, arg.len + 1);
                     const writer = escaped.writer();
                     try writer.writeAll(arg[0..arg_idx]);
                     for (arg[arg_idx..]) |to_escape| {
 @@ -1804,11 +1781,16 @@ fn make(step: *Step) !void {
             .{std.fmt.fmtSliceHexLower(&args_hash)},
         );
-        const args_file = try fs.path.join(builder.allocator, &[_][]const u8{ args_dir, args_hex_hash[0..] });
+        const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash;
-        try std.fs.cwd().writeFile(args_file, args);
+        try builder.cache_root.handle.writeFile(args_file, args);
+        const resolved_args_file = try mem.concat(builder.allocator, u8, &.{
+            "@",
+            try builder.cache_root.join(builder.allocator, &.{args_file}),
+        });
         zig_args.shrinkRetainingCapacity(2);
-        try zig_args.append(try std.mem.concat(builder.allocator, u8, &[_][]const u8{ "@", args_file }));
+        try zig_args.append(resolved_args_file);
     }
     const output_dir_nl = try builder.execFromStep(zig_args.items, &self.step);

lib/std/Build/ConfigHeaderStep.zig+65-9

 @@ -13,11 +13,13 @@ pub const Style = union(enum) {
     cmake: std.Build.FileSource,
     /// Instead of starting with an input file, start with nothing.
     blank,
+    /// Start with nothing, like blank, and output a nasm .asm file.
+    nasm,
     pub fn getFileSource(style: Style) ?std.Build.FileSource {
         switch (style) {
             .autoconf, .cmake => |s| return s,
-            .blank => return null,
+            .blank, .nasm => return null,
         }
     }
 };
 @@ -84,6 +86,10 @@ pub fn addValues(self: *ConfigHeaderStep, values: anytype) void {
     return addValuesInner(self, values) catch @panic("OOM");
 }
+pub fn getFileSource(self: *ConfigHeaderStep) std.Build.FileSource {
+    return .{ .generated = &self.output_file };
+}
 fn addValuesInner(self: *ConfigHeaderStep, values: anytype) !void {
     inline for (@typeInfo(@TypeOf(values)).Struct.fields) |field| {
         try putValue(self, field.name, field.type, @field(values, field.name));
 @@ -125,6 +131,12 @@ fn putValue(self: *ConfigHeaderStep, field_name: []const u8, comptime T: type, v
                         return;
                     }
                 },
+                .Int => {
+                    if (ptr.size == .Slice and ptr.child == u8) {
+                        try self.values.put(field_name, .{ .string = v });
+                        return;
+                    }
+                },
                 else => {},
             }
 @@ -158,22 +170,31 @@ fn make(step: *Step) !void {
     var output = std.ArrayList(u8).init(gpa);
     defer output.deinit();
-    try output.appendSlice("/* This file was generated by ConfigHeaderStep using the Zig Build System. */\n");
+    const header_text = "This file was generated by ConfigHeaderStep using the Zig Build System.";
+    const c_generated_line = "/* " ++ header_text ++ " */\n";
+    const asm_generated_line = "; " ++ header_text ++ "\n";
     switch (self.style) {
         .autoconf => |file_source| {
+            try output.appendSlice(c_generated_line);
             const src_path = file_source.getPath(self.builder);
             const contents = try std.fs.cwd().readFileAlloc(gpa, src_path, self.max_bytes);
             try render_autoconf(contents, &output, self.values, src_path);
         },
         .cmake => |file_source| {
+            try output.appendSlice(c_generated_line);
             const src_path = file_source.getPath(self.builder);
             const contents = try std.fs.cwd().readFileAlloc(gpa, src_path, self.max_bytes);
             try render_cmake(contents, &output, self.values, src_path);
         },
         .blank => {
+            try output.appendSlice(c_generated_line);
             try render_blank(&output, self.values, self.include_path);
         },
+        .nasm => {
+            try output.appendSlice(asm_generated_line);
+            try render_nasm(&output, self.values);
+        },
     }
     hash.update(output.items);
 @@ -187,9 +208,7 @@ fn make(step: *Step) !void {
         .{std.fmt.fmtSliceHexLower(&digest)},
     ) catch unreachable;
-    const output_dir = try std.fs.path.join(gpa, &[_][]const u8{
+    const output_dir = try self.builder.cache_root.join(gpa, &.{ "o", &hash_basename });
-        self.builder.cache_root, "o", &hash_basename,
-    });
     // If output_path has directory parts, deal with them.  Example:
     // output_dir is zig-cache/o/HASH
 @@ -247,7 +266,7 @@ fn render_autoconf(
             any_errors = true;
             continue;
         };
-        try renderValue(output, name, kv.value);
+        try renderValueC(output, name, kv.value);
     }
     for (values_copy.keys()) |name| {
 @@ -298,7 +317,7 @@ fn render_cmake(
             any_errors = true;
             continue;
         };
-        try renderValue(output, name, kv.value);
+        try renderValueC(output, name, kv.value);
     }
     for (values_copy.keys()) |name| {
 @@ -332,7 +351,7 @@ fn render_blank(
     const values = defines.values();
     for (defines.keys()) |name, i| {
-        try renderValue(output, name, values[i]);
+        try renderValueC(output, name, values[i]);
     }
     try output.appendSlice("#endif /* ");
 @@ -340,7 +359,14 @@ fn render_blank(
     try output.appendSlice(" */\n");
 }
-fn renderValue(output: *std.ArrayList(u8), name: []const u8, value: Value) !void {
+fn render_nasm(output: *std.ArrayList(u8), defines: std.StringArrayHashMap(Value)) !void {
+    const values = defines.values();
+    for (defines.keys()) |name, i| {
+        try renderValueNasm(output, name, values[i]);
+    }
+}
+fn renderValueC(output: *std.ArrayList(u8), name: []const u8, value: Value) !void {
     switch (value) {
         .undef => {
             try output.appendSlice("/* #undef ");
 @@ -370,3 +396,33 @@ fn renderValue(output: *std.ArrayList(u8), name: []const u8, value: Value) !void
         },
     }
 }
+fn renderValueNasm(output: *std.ArrayList(u8), name: []const u8, value: Value) !void {
+    switch (value) {
+        .undef => {
+            try output.appendSlice("; %undef ");
+            try output.appendSlice(name);
+            try output.appendSlice("\n");
+        },
+        .defined => {
+            try output.appendSlice("%define ");
+            try output.appendSlice(name);
+            try output.appendSlice("\n");
+        },
+        .boolean => |b| {
+            try output.appendSlice("%define ");
+            try output.appendSlice(name);
+            try output.appendSlice(if (b) " 1\n" else " 0\n");
+        },
+        .int => |i| {
+            try output.writer().print("%define {s} {d}\n", .{ name, i });
+        },
+        .ident => |ident| {
+            try output.writer().print("%define {s} {s}\n", .{ name, ident });
+        },
+        .string => |string| {
+            // TODO: use nasm-specific escaping instead of zig string literals
+            try output.writer().print("%define {s} \"{}\"\n", .{ name, std.zig.fmtEscapes(string) });
+        },
+    }
+}

lib/std/Build/OptionsStep.zig+17-17

 @@ -234,26 +234,20 @@ fn make(step: *Step) !void {
         );
     }
-    const options_directory = self.builder.pathFromRoot(
+    var options_dir = try self.builder.cache_root.handle.makeOpenPath("options", .{});
-        try fs.path.join(
+    defer options_dir.close();
-            self.builder.allocator,
-            &[_][]const u8{ self.builder.cache_root, "options" },
-        ),
-    );
-    try fs.cwd().makePath(options_directory);
-    const options_file = try fs.path.join(
+    const basename = self.hashContentsToFileName();
-        self.builder.allocator,
-        &[_][]const u8{ options_directory, &self.hashContentsToFileName() },
-    );
-    try fs.cwd().writeFile(options_file, self.contents.items);
+    try options_dir.writeFile(&basename, self.contents.items);
-    self.generated_file.path = options_file;
+    self.generated_file.path = try self.builder.cache_root.join(self.builder.allocator, &.{
+        "options", &basename,
+    });
 }
 fn hashContentsToFileName(self: *OptionsStep) [64]u8 {
+    // TODO update to use the cache system instead of this
     // This implementation is copied from `WriteFileStep.make`
     var hash = std.crypto.hash.blake2.Blake2b384.init(.{});
 @@ -289,13 +283,19 @@ test "OptionsStep" {
     const host = try std.zig.system.NativeTargetInfo.detect(.{});
+    var cache: std.Build.Cache = .{
+        .gpa = arena.allocator(),
+        .manifest_dir = std.fs.cwd(),
+    };
     var builder = try std.Build.create(
         arena.allocator(),
         "test",
-        "test",
+        .{ .path = "test", .handle = std.fs.cwd() },
-        "test",
+        .{ .path = "test", .handle = std.fs.cwd() },
-        "test",
+        .{ .path = "test", .handle = std.fs.cwd() },
         host,
+        &cache,
     );
     defer builder.destroy();

lib/std/Build/RunStep.zig+133-6

...	@@ -39,6 +39,14 @@ expected_exit_code: ?u8 = 0,	...	@@ -39,6 +39,14 @@ expected_exit_code: ?u8 = 0,
39		39
40	/// Print the command before running it	40	/// Print the command before running it
41	print: bool,	41	print: bool,
		42	/// Controls whether execution is skipped if the output file is up-to-date.
		43	/// The default is to always run if there is no output file, and to skip
		44	/// running if all output files are up-to-date.
		45	condition: enum { output_outdated, always } = .output_outdated,
		46
		47	/// Additional file paths relative to build.zig that, when modified, indicate
		48	/// that the RunStep should be re-executed.
		49	extra_file_dependencies: []const []const u8 = &.{},
42		50
43	pub const StdIoAction = union(enum) {	51	pub const StdIoAction = union(enum) {
44	inherit,	52	inherit,
...	@@ -51,6 +59,12 @@ pub const Arg = union(enum) {	...	@@ -51,6 +59,12 @@ pub const Arg = union(enum) {
51	artifact: *CompileStep,	59	artifact: *CompileStep,
52	file_source: std.Build.FileSource,	60	file_source: std.Build.FileSource,
53	bytes: []u8,	61	bytes: []u8,
		62	output: Output,
		63
		64	pub const Output = struct {
		65	generated_file: *std.Build.GeneratedFile,
		66	basename: []const u8,
		67	};
54	};	68	};
55		69
56	pub fn create(builder: std.Build, name: []const u8) RunStep {	70	pub fn create(builder: std.Build, name: []const u8) RunStep {
...	@@ -71,6 +85,20 @@ pub fn addArtifactArg(self: RunStep, artifact: CompileStep) void {	...	@@ -71,6 +85,20 @@ pub fn addArtifactArg(self: RunStep, artifact: CompileStep) void {
71	self.step.dependOn(&artifact.step);	85	self.step.dependOn(&artifact.step);
72	}	86	}
73		87
		88	/// This provides file path as a command line argument to the command being
		89	/// run, and returns a FileSource which can be used as inputs to other APIs
		90	/// throughout the build system.
		91	pub fn addOutputFileArg(rs: *RunStep, basename: []const u8) std.Build.FileSource {
		92	const generated_file = rs.builder.allocator.create(std.Build.GeneratedFile) catch @panic("OOM");
		93	generated_file.* = .{ .step = &rs.step };
		94	rs.argv.append(.{ .output = .{
		95	.generated_file = generated_file,
		96	.basename = rs.builder.dupe(basename),
		97	} }) catch @panic("OOM");
		98
		99	return .{ .generated = generated_file };
		100	}
		101
74	pub fn addFileSourceArg(self: *RunStep, file_source: std.Build.FileSource) void {	102	pub fn addFileSourceArg(self: *RunStep, file_source: std.Build.FileSource) void {
75	self.argv.append(Arg{	103	self.argv.append(Arg{
76	.file_source = file_source.dupe(self.builder),	104	.file_source = file_source.dupe(self.builder),
...	@@ -159,25 +187,105 @@ fn stdIoActionToBehavior(action: StdIoAction) std.ChildProcess.StdIo {	...	@@ -159,25 +187,105 @@ fn stdIoActionToBehavior(action: StdIoAction) std.ChildProcess.StdIo {
159	};	187	};
160	}	188	}
161		189
		190	fn needOutputCheck(self: RunStep) bool {
		191	if (self.extra_file_dependencies.len > 0) return true;
		192
		193	for (self.argv.items) \|arg\| switch (arg) {
		194	.output => return true,
		195	else => continue,
		196	};
		197
		198	return switch (self.condition) {
		199	.always => false,
		200	.output_outdated => true,
		201	};
		202	}
		203
162	fn make(step: *Step) !void {	204	fn make(step: *Step) !void {
163	const self = @fieldParentPtr(RunStep, "step", step);	205	const self = @fieldParentPtr(RunStep, "step", step);
		206	const need_output_check = self.needOutputCheck();
164		207
165	var argv_list = ArrayList([]const u8).init(self.builder.allocator);	208	var argv_list = ArrayList([]const u8).init(self.builder.allocator);
		209	var output_placeholders = ArrayList(struct {
		210	index: usize,
		211	output: Arg.Output,
		212	}).init(self.builder.allocator);
		213
		214	var man = self.builder.cache.obtain();
		215	defer man.deinit();
		216
166	for (self.argv.items) \|arg\| {	217	for (self.argv.items) \|arg\| {
167	switch (arg) {	218	switch (arg) {
168	.bytes => \|bytes\| try argv_list.append(bytes),	219	.bytes => \|bytes\| {
169	.file_source => \|file\| try argv_list.append(file.getPath(self.builder)),	220	try argv_list.append(bytes);
		221	man.hash.addBytes(bytes);
		222	},
		223	.file_source => \|file\| {
		224	const file_path = file.getPath(self.builder);
		225	try argv_list.append(file_path);
		226	_ = try man.addFile(file_path, null);
		227	},
170	.artifact => \|artifact\| {	228	.artifact => \|artifact\| {
171	if (artifact.target.isWindows()) {	229	if (artifact.target.isWindows()) {
172	// On Windows we don't have rpaths so we have to add .dll search paths to PATH	230	// On Windows we don't have rpaths so we have to add .dll search paths to PATH
173	self.addPathForDynLibs(artifact);	231	self.addPathForDynLibs(artifact);
174	}	232	}
175	const executable_path = artifact.installed_path orelse artifact.getOutputSource().getPath(self.builder);	233	const file_path = artifact.installed_path orelse
176	try argv_list.append(executable_path);	234	artifact.getOutputSource().getPath(self.builder);
		235
		236	try argv_list.append(file_path);
		237
		238	_ = try man.addFile(file_path, null);
		239	},
		240	.output => \|output\| {
		241	man.hash.addBytes(output.basename);
		242	// Add a placeholder into the argument list because we need the
		243	// manifest hash to be updated with all arguments before the
		244	// object directory is computed.
		245	try argv_list.append("");
		246	try output_placeholders.append(.{
		247	.index = argv_list.items.len - 1,
		248	.output = output,
		249	});
177	},	250	},
178	}	251	}
179	}	252	}
180		253
		254	if (need_output_check) {
		255	for (self.extra_file_dependencies) \|file_path\| {
		256	_ = try man.addFile(self.builder.pathFromRoot(file_path), null);
		257	}
		258
		259	if (man.hit() catch \|err\| failWithCacheError(man, err)) {
		260	// cache hit, skip running command
		261	const digest = man.final();
		262	for (output_placeholders.items) \|placeholder\| {
		263	placeholder.output.generated_file.path = try self.builder.cache_root.join(
		264	self.builder.allocator,
		265	&.{ "o", &digest, placeholder.output.basename },
		266	);
		267	}
		268	return;
		269	}
		270
		271	const digest = man.final();
		272
		273	for (output_placeholders.items) \|placeholder\| {
		274	const output_path = try self.builder.cache_root.join(
		275	self.builder.allocator,
		276	&.{ "o", &digest, placeholder.output.basename },
		277	);
		278	const output_dir = fs.path.dirname(output_path).?;
		279	fs.cwd().makePath(output_dir) catch \|err\| {
		280	std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });
		281	return err;
		282	};
		283
		284	placeholder.output.generated_file.path = output_path;
		285	argv_list.items[placeholder.index] = output_path;
		286	}
		287	}
		288
181	try runCommand(	289	try runCommand(
182	argv_list.items,	290	argv_list.items,
183	self.builder,	291	self.builder,
...	@@ -189,6 +297,10 @@ fn make(step: *Step) !void {	...	@@ -189,6 +297,10 @@ fn make(step: *Step) !void {
189	self.cwd,	297	self.cwd,
190	self.print,	298	self.print,
191	);	299	);
		300
		301	if (need_output_check) {
		302	try man.writeManifest();
		303	}
192	}	304	}
193		305
194	pub fn runCommand(	306	pub fn runCommand(
...	@@ -202,11 +314,13 @@ pub fn runCommand(	...	@@ -202,11 +314,13 @@ pub fn runCommand(
202	maybe_cwd: ?[]const u8,	314	maybe_cwd: ?[]const u8,
203	print: bool,	315	print: bool,
204	) !void {	316	) !void {
205	const cwd = if (maybe_cwd) \|cwd\| builder.pathFromRoot(cwd) else builder.build_root;	317	const cwd = if (maybe_cwd) \|cwd\| builder.pathFromRoot(cwd) else builder.build_root.path;
206		318
207	if (!std.process.can_spawn) {	319	if (!std.process.can_spawn) {
208	const cmd = try std.mem.join(builder.allocator, " ", argv);	320	const cmd = try std.mem.join(builder.allocator, " ", argv);
209	std.debug.print("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{ @tagName(builtin.os.tag), cmd });	321	std.debug.print("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{
		322	@tagName(builtin.os.tag), cmd,
		323	});
210	builder.allocator.free(cmd);	324	builder.allocator.free(cmd);
211	return ExecError.ExecNotSupported;	325	return ExecError.ExecNotSupported;
212	}	326	}
...	@@ -347,6 +461,19 @@ pub fn runCommand(	...	@@ -347,6 +461,19 @@ pub fn runCommand(
347	}	461	}
348	}	462	}
349		463
		464	fn failWithCacheError(man: std.Build.Cache.Manifest, err: anyerror) noreturn {
		465	const i = man.failed_file_index orelse failWithSimpleError(err);
		466	const pp = man.files.items[i].prefixed_path orelse failWithSimpleError(err);
		467	const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
		468	std.debug.print("{s}: {s}/{s}\n", .{ @errorName(err), prefix, pp.sub_path });
		469	std.process.exit(1);
		470	}
		471
		472	fn failWithSimpleError(err: anyerror) noreturn {
		473	std.debug.print("{s}\n", .{@errorName(err)});
		474	std.process.exit(1);
		475	}
		476
350	fn printCmd(cwd: ?[]const u8, argv: []const []const u8) void {	477	fn printCmd(cwd: ?[]const u8, argv: []const []const u8) void {
351	if (cwd) \|yes_cwd\| std.debug.print("cd {s} && ", .{yes_cwd});	478	if (cwd) \|yes_cwd\| std.debug.print("cd {s} && ", .{yes_cwd});
352	for (argv) \|arg\| {	479	for (argv) \|arg\| {

lib/std/Build/WriteFileStep.zig+2-2

...	@@ -85,8 +85,8 @@ fn make(step: *Step) !void {	...	@@ -85,8 +85,8 @@ fn make(step: *Step) !void {
85	.{std.fmt.fmtSliceHexLower(&digest)},	85	.{std.fmt.fmtSliceHexLower(&digest)},
86	) catch unreachable;	86	) catch unreachable;
87		87
88	const output_dir = try fs.path.join(self.builder.allocator, &[_][]const u8{	88	const output_dir = try self.builder.cache_root.join(self.builder.allocator, &.{
89	self.builder.cache_root, "o", &hash_basename,	89	"o", &hash_basename,
90	});	90	});
91	var dir = fs.cwd().makeOpenPath(output_dir, .{}) catch \|err\| {	91	var dir = fs.cwd().makeOpenPath(output_dir, .{}) catch \|err\| {
92	std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });	92	std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });

src/Cache.zig deleted-1265

...	@@ -1,1265 +0,0 @@
1	//! Manages `zig-cache` directories.
2	//! This is not a general-purpose cache. It is designed to be fast and simple,
3	//! not to withstand attacks using specially-crafted input.
4
5	gpa: Allocator,
6	manifest_dir: fs.Dir,
7	hash: HashHelper = .{},
8	/// This value is accessed from multiple threads, protected by mutex.
9	recent_problematic_timestamp: i128 = 0,
10	mutex: std.Thread.Mutex = .{},
11
12	/// A set of strings such as the zig library directory or project source root, which
13	/// are stripped from the file paths before putting into the cache. They
14	/// are replaced with single-character indicators. This is not to save
15	/// space but to eliminate absolute file paths. This improves portability
16	/// and usefulness of the cache for advanced use cases.
17	prefixes_buffer: [3]Compilation.Directory = undefined,
18	prefixes_len: usize = 0,
19
20	const Cache = @This();
21	const std = @import("std");
22	const builtin = @import("builtin");
23	const crypto = std.crypto;
24	const fs = std.fs;
25	const assert = std.debug.assert;
26	const testing = std.testing;
27	const mem = std.mem;
28	const fmt = std.fmt;
29	const Allocator = std.mem.Allocator;
30	const Compilation = @import("Compilation.zig");
31	const log = std.log.scoped(.cache);
32
33	pub fn addPrefix(cache: *Cache, directory: Compilation.Directory) void {
34	if (directory.path) \|p\| {
35	log.debug("Cache.addPrefix {d} {s}", .{ cache.prefixes_len, p });
36	}
37	cache.prefixes_buffer[cache.prefixes_len] = directory;
38	cache.prefixes_len += 1;
39	}
40
41	/// Be sure to call `Manifest.deinit` after successful initialization.
42	pub fn obtain(cache: *Cache) Manifest {
43	return Manifest{
44	.cache = cache,
45	.hash = cache.hash,
46	.manifest_file = null,
47	.manifest_dirty = false,
48	.hex_digest = undefined,
49	};
50	}
51
52	pub fn prefixes(cache: *const Cache) []const Compilation.Directory {
53	return cache.prefixes_buffer[0..cache.prefixes_len];
54	}
55
56	const PrefixedPath = struct {
57	prefix: u8,
58	sub_path: []u8,
59	};
60
61	fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath {
62	const gpa = cache.gpa;
63	const resolved_path = try fs.path.resolve(gpa, &[_][]const u8{file_path});
64	errdefer gpa.free(resolved_path);
65	return findPrefixResolved(cache, resolved_path);
66	}
67
68	/// Takes ownership of `resolved_path` on success.
69	fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath {
70	const gpa = cache.gpa;
71	const prefixes_slice = cache.prefixes();
72	var i: u8 = 1; // Start at 1 to skip over checking the null prefix.
73	while (i < prefixes_slice.len) : (i += 1) {
74	const p = prefixes_slice[i].path.?;
75	if (mem.startsWith(u8, resolved_path, p)) {
76	// +1 to skip over the path separator here
77	const sub_path = try gpa.dupe(u8, resolved_path[p.len + 1 ..]);
78	gpa.free(resolved_path);
79	return PrefixedPath{
80	.prefix = @intCast(u8, i),
81	.sub_path = sub_path,
82	};
83	} else {
84	log.debug("'{s}' does not start with '{s}'", .{ resolved_path, p });
85	}
86	}
87
88	return PrefixedPath{
89	.prefix = 0,
90	.sub_path = resolved_path,
91	};
92	}
93
94	/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
95	pub const bin_digest_len = 16;
96	pub const hex_digest_len = bin_digest_len * 2;
97	pub const BinDigest = [bin_digest_len]u8;
98
99	const manifest_file_size_max = 50 * 1024 * 1024;
100
101	/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it
102	/// provides enough collision resistance for the Manifest use cases, while being one of our
103	/// fastest options right now.
104	pub const Hasher = crypto.auth.siphash.SipHash128(1, 3);
105
106	/// Initial state, that can be copied.
107	pub const hasher_init: Hasher = Hasher.init(&[_]u8{0} ** Hasher.key_length);
108
109	pub const File = struct {
110	prefixed_path: ?PrefixedPath,
111	max_file_size: ?usize,
112	stat: Stat,
113	bin_digest: BinDigest,
114	contents: ?[]const u8,
115
116	pub const Stat = struct {
117	inode: fs.File.INode,
118	size: u64,
119	mtime: i128,
120	};
121
122	pub fn deinit(self: *File, gpa: Allocator) void {
123	if (self.prefixed_path) \|pp\| {
124	gpa.free(pp.sub_path);
125	self.prefixed_path = null;
126	}
127	if (self.contents) \|contents\| {
128	gpa.free(contents);
129	self.contents = null;
130	}
131	self.* = undefined;
132	}
133	};
134
135	pub const HashHelper = struct {
136	hasher: Hasher = hasher_init,
137
138	const EmitLoc = Compilation.EmitLoc;
139
140	/// Record a slice of bytes as an dependency of the process being cached
141	pub fn addBytes(hh: *HashHelper, bytes: []const u8) void {
142	hh.hasher.update(mem.asBytes(&bytes.len));
143	hh.hasher.update(bytes);
144	}
145
146	pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void {
147	hh.add(optional_bytes != null);
148	hh.addBytes(optional_bytes orelse return);
149	}
150
151	pub fn addEmitLoc(hh: *HashHelper, emit_loc: EmitLoc) void {
152	hh.addBytes(emit_loc.basename);
153	}
154
155	pub fn addOptionalEmitLoc(hh: *HashHelper, optional_emit_loc: ?EmitLoc) void {
156	hh.add(optional_emit_loc != null);
157	hh.addEmitLoc(optional_emit_loc orelse return);
158	}
159
160	pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void {
161	hh.add(list_of_bytes.len);
162	for (list_of_bytes) \|bytes\| hh.addBytes(bytes);
163	}
164
165	/// Convert the input value into bytes and record it as a dependency of the process being cached.
166	pub fn add(hh: *HashHelper, x: anytype) void {
167	switch (@TypeOf(x)) {
168	std.builtin.Version => {
169	hh.add(x.major);
170	hh.add(x.minor);
171	hh.add(x.patch);
172	},
173	std.Target.Os.TaggedVersionRange => {
174	switch (x) {
175	.linux => \|linux\| {
176	hh.add(linux.range.min);
177	hh.add(linux.range.max);
178	hh.add(linux.glibc);
179	},
180	.windows => \|windows\| {
181	hh.add(windows.min);
182	hh.add(windows.max);
183	},
184	.semver => \|semver\| {
185	hh.add(semver.min);
186	hh.add(semver.max);
187	},
188	.none => {},
189	}
190	},
191	else => switch (@typeInfo(@TypeOf(x))) {
192	.Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)),
193	else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))),
194	},
195	}
196	}
197
198	pub fn addOptional(hh: *HashHelper, optional: anytype) void {
199	hh.add(optional != null);
200	hh.add(optional orelse return);
201	}
202
203	/// Returns a hex encoded hash of the inputs, without modifying state.
204	pub fn peek(hh: HashHelper) [hex_digest_len]u8 {
205	var copy = hh;
206	return copy.final();
207	}
208
209	pub fn peekBin(hh: HashHelper) BinDigest {
210	var copy = hh;
211	var bin_digest: BinDigest = undefined;
212	copy.hasher.final(&bin_digest);
213	return bin_digest;
214	}
215
216	/// Returns a hex encoded hash of the inputs, mutating the state of the hasher.
217	pub fn final(hh: *HashHelper) [hex_digest_len]u8 {
218	var bin_digest: BinDigest = undefined;
219	hh.hasher.final(&bin_digest);
220
221	var out_digest: [hex_digest_len]u8 = undefined;
222	_ = std.fmt.bufPrint(
223	&out_digest,
224	"{s}",
225	.{std.fmt.fmtSliceHexLower(&bin_digest)},
226	) catch unreachable;
227	return out_digest;
228	}
229	};
230
231	pub const Lock = struct {
232	manifest_file: fs.File,
233
234	pub fn release(lock: *Lock) void {
235	if (builtin.os.tag == .windows) {
236	// Windows does not guarantee that locks are immediately unlocked when
237	// the file handle is closed. See LockFileEx documentation.
238	lock.manifest_file.unlock();
239	}
240
241	lock.manifest_file.close();
242	lock.* = undefined;
243	}
244	};
245
246	pub const Manifest = struct {
247	cache: *Cache,
248	/// Current state for incremental hashing.
249	hash: HashHelper,
250	manifest_file: ?fs.File,
251	manifest_dirty: bool,
252	/// Set this flag to true before calling hit() in order to indicate that
253	/// upon a cache hit, the code using the cache will not modify the files
254	/// within the cache directory. This allows multiple processes to utilize
255	/// the same cache directory at the same time.
256	want_shared_lock: bool = true,
257	have_exclusive_lock: bool = false,
258	// Indicate that we want isProblematicTimestamp to perform a filesystem write in
259	// order to obtain a problematic timestamp for the next call. Calls after that
260	// will then use the same timestamp, to avoid unnecessary filesystem writes.
261	want_refresh_timestamp: bool = true,
262	files: std.ArrayListUnmanaged(File) = .{},
263	hex_digest: [hex_digest_len]u8,
264	/// Populated when hit() returns an error because of one
265	/// of the files listed in the manifest.
266	failed_file_index: ?usize = null,
267	/// Keeps track of the last time we performed a file system write to observe
268	/// what time the file system thinks it is, according to its own granularity.
269	recent_problematic_timestamp: i128 = 0,
270
271	/// Add a file as a dependency of process being cached. When `hit` is
272	/// called, the file's contents will be checked to ensure that it matches
273	/// the contents from previous times.
274	///
275	/// Max file size will be used to determine the amount of space the file contents
276	/// are allowed to take up in memory. If max_file_size is null, then the contents
277	/// will not be loaded into memory.
278	///
279	/// Returns the index of the entry in the `files` array list. You can use it
280	/// to access the contents of the file after calling `hit()` like so:
281	///
282	/// ```
283	/// var file_contents = cache_hash.files.items[file_index].contents.?;
284	/// ```
285	pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize {
286	assert(self.manifest_file == null);
287
288	const gpa = self.cache.gpa;
289	try self.files.ensureUnusedCapacity(gpa, 1);
290	const prefixed_path = try self.cache.findPrefix(file_path);
291	errdefer gpa.free(prefixed_path.sub_path);
292
293	log.debug("Manifest.addFile {s} -> {d} {s}", .{
294	file_path, prefixed_path.prefix, prefixed_path.sub_path,
295	});
296
297	self.files.addOneAssumeCapacity().* = .{
298	.prefixed_path = prefixed_path,
299	.contents = null,
300	.max_file_size = max_file_size,
301	.stat = undefined,
302	.bin_digest = undefined,
303	};
304
305	self.hash.add(prefixed_path.prefix);
306	self.hash.addBytes(prefixed_path.sub_path);
307
308	return self.files.items.len - 1;
309	}
310
311	pub fn hashCSource(self: *Manifest, c_source: Compilation.CSourceFile) !void {
312	_ = try self.addFile(c_source.src_path, null);
313	// Hash the extra flags, with special care to call addFile for file parameters.
314	// TODO this logic can likely be improved by utilizing clang_options_data.zig.
315	const file_args = [_][]const u8{"-include"};
316	var arg_i: usize = 0;
317	while (arg_i < c_source.extra_flags.len) : (arg_i += 1) {
318	const arg = c_source.extra_flags[arg_i];
319	self.hash.addBytes(arg);
320	for (file_args) \|file_arg\| {
321	if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_source.extra_flags.len) {
322	arg_i += 1;
323	_ = try self.addFile(c_source.extra_flags[arg_i], null);
324	}
325	}
326	}
327	}
328
329	pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void {
330	self.hash.add(optional_file_path != null);
331	const file_path = optional_file_path orelse return;
332	_ = try self.addFile(file_path, null);
333	}
334
335	pub fn addListOfFiles(self: *Manifest, list_of_files: []const []const u8) !void {
336	self.hash.add(list_of_files.len);
337	for (list_of_files) \|file_path\| {
338	_ = try self.addFile(file_path, null);
339	}
340	}
341
342	/// Check the cache to see if the input exists in it. If it exists, returns `true`.
343	/// A hex encoding of its hash is available by calling `final`.
344	///
345	/// This function will also acquire an exclusive lock to the manifest file. This means
346	/// that a process holding a Manifest will block any other process attempting to
347	/// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the
348	/// manifest file to be locked in shared mode, and a cache miss guarantees the manifest
349	/// file to be locked in exclusive mode.
350	///
351	/// The lock on the manifest file is released when `deinit` is called. As another
352	/// option, one may call `toOwnedLock` to obtain a smaller object which can represent
353	/// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
354	pub fn hit(self: *Manifest) !bool {
355	const gpa = self.cache.gpa;
356	assert(self.manifest_file == null);
357
358	self.failed_file_index = null;
359
360	const ext = ".txt";
361	var manifest_file_path: [self.hex_digest.len + ext.len]u8 = undefined;
362
363	var bin_digest: BinDigest = undefined;
364	self.hash.hasher.final(&bin_digest);
365
366	_ = std.fmt.bufPrint(
367	&self.hex_digest,
368	"{s}",
369	.{std.fmt.fmtSliceHexLower(&bin_digest)},
370	) catch unreachable;
371
372	self.hash.hasher = hasher_init;
373	self.hash.hasher.update(&bin_digest);
374
375	mem.copy(u8, &manifest_file_path, &self.hex_digest);
376	manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*;
377
378	if (self.files.items.len == 0) {
379	// If there are no file inputs, we check if the manifest file exists instead of
380	// comparing the hashes on the files used for the cached item
381	while (true) {
382	if (self.cache.manifest_dir.openFile(&manifest_file_path, .{
383	.mode = .read_write,
384	.lock = .Exclusive,
385	.lock_nonblocking = self.want_shared_lock,
386	})) \|manifest_file\| {
387	self.manifest_file = manifest_file;
388	self.have_exclusive_lock = true;
389	break;
390	} else \|open_err\| switch (open_err) {
391	error.WouldBlock => {
392	self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
393	.lock = .Shared,
394	});
395	break;
396	},
397	error.FileNotFound => {
398	if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
399	.read = true,
400	.truncate = false,
401	.lock = .Exclusive,
402	.lock_nonblocking = self.want_shared_lock,
403	})) \|manifest_file\| {
404	self.manifest_file = manifest_file;
405	self.manifest_dirty = true;
406	self.have_exclusive_lock = true;
407	return false; // cache miss; exclusive lock already held
408	} else \|err\| switch (err) {
409	error.WouldBlock => continue,
410	else => \|e\| return e,
411	}
412	},
413	else => \|e\| return e,
414	}
415	}
416	} else {
417	if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
418	.read = true,
419	.truncate = false,
420	.lock = .Exclusive,
421	.lock_nonblocking = self.want_shared_lock,
422	})) \|manifest_file\| {
423	self.manifest_file = manifest_file;
424	self.have_exclusive_lock = true;
425	} else \|err\| switch (err) {
426	error.WouldBlock => {
427	self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
428	.lock = .Shared,
429	});
430	},
431	else => \|e\| return e,
432	}
433	}
434
435	self.want_refresh_timestamp = true;
436
437	const file_contents = try self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max);
438	defer gpa.free(file_contents);
439
440	const input_file_count = self.files.items.len;
441	var any_file_changed = false;
442	var line_iter = mem.tokenize(u8, file_contents, "\n");
443	var idx: usize = 0;
444	while (line_iter.next()) \|line\| {
445	defer idx += 1;
446
447	const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: {
448	const new = try self.files.addOne(gpa);
449	new.* = .{
450	.prefixed_path = null,
451	.contents = null,
452	.max_file_size = null,
453	.stat = undefined,
454	.bin_digest = undefined,
455	};
456	break :blk new;
457	};
458
459	var iter = mem.tokenize(u8, line, " ");
460	const size = iter.next() orelse return error.InvalidFormat;
461	const inode = iter.next() orelse return error.InvalidFormat;
462	const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
463	const digest_str = iter.next() orelse return error.InvalidFormat;
464	const prefix_str = iter.next() orelse return error.InvalidFormat;
465	const file_path = iter.rest();
466
467	cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat;
468	cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat;
469	cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
470	_ = std.fmt.hexToBytes(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;
471	const prefix = fmt.parseInt(u8, prefix_str, 10) catch return error.InvalidFormat;
472	if (prefix >= self.cache.prefixes_len) return error.InvalidFormat;
473
474	if (file_path.len == 0) {
475	return error.InvalidFormat;
476	}
477	if (cache_hash_file.prefixed_path) \|pp\| {
478	if (pp.prefix != prefix or !mem.eql(u8, file_path, pp.sub_path)) {
479	return error.InvalidFormat;
480	}
481	}
482
483	if (cache_hash_file.prefixed_path == null) {
484	cache_hash_file.prefixed_path = .{
485	.prefix = prefix,
486	.sub_path = try gpa.dupe(u8, file_path),
487	};
488	}
489
490	const pp = cache_hash_file.prefixed_path.?;
491	const dir = self.cache.prefixes()[pp.prefix].handle;
492	const this_file = dir.openFile(pp.sub_path, .{ .mode = .read_only }) catch \|err\| switch (err) {
493	error.FileNotFound => {
494	try self.upgradeToExclusiveLock();
495	return false;
496	},
497	else => return error.CacheUnavailable,
498	};
499	defer this_file.close();
500
501	const actual_stat = this_file.stat() catch \|err\| {
502	self.failed_file_index = idx;
503	return err;
504	};
505	const size_match = actual_stat.size == cache_hash_file.stat.size;
506	const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime;
507	const inode_match = actual_stat.inode == cache_hash_file.stat.inode;
508
509	if (!size_match or !mtime_match or !inode_match) {
510	self.manifest_dirty = true;
511
512	cache_hash_file.stat = .{
513	.size = actual_stat.size,
514	.mtime = actual_stat.mtime,
515	.inode = actual_stat.inode,
516	};
517
518	if (self.isProblematicTimestamp(cache_hash_file.stat.mtime)) {
519	// The actual file has an unreliable timestamp, force it to be hashed
520	cache_hash_file.stat.mtime = 0;
521	cache_hash_file.stat.inode = 0;
522	}
523
524	var actual_digest: BinDigest = undefined;
525	hashFile(this_file, &actual_digest) catch \|err\| {
526	self.failed_file_index = idx;
527	return err;
528	};
529
530	if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
531	cache_hash_file.bin_digest = actual_digest;
532	// keep going until we have the input file digests
533	any_file_changed = true;
534	}
535	}
536
537	if (!any_file_changed) {
538	self.hash.hasher.update(&cache_hash_file.bin_digest);
539	}
540	}
541
542	if (any_file_changed) {
543	// cache miss
544	// keep the manifest file open
545	self.unhit(bin_digest, input_file_count);
546	try self.upgradeToExclusiveLock();
547	return false;
548	}
549
550	if (idx < input_file_count) {
551	self.manifest_dirty = true;
552	while (idx < input_file_count) : (idx += 1) {
553	const ch_file = &self.files.items[idx];
554	self.populateFileHash(ch_file) catch \|err\| {
555	self.failed_file_index = idx;
556	return err;
557	};
558	}
559	try self.upgradeToExclusiveLock();
560	return false;
561	}
562
563	if (self.want_shared_lock) {
564	try self.downgradeToSharedLock();
565	}
566
567	return true;
568	}
569
570	pub fn unhit(self: *Manifest, bin_digest: BinDigest, input_file_count: usize) void {
571	// Reset the hash.
572	self.hash.hasher = hasher_init;
573	self.hash.hasher.update(&bin_digest);
574
575	// Remove files not in the initial hash.
576	for (self.files.items[input_file_count..]) \|*file\| {
577	file.deinit(self.cache.gpa);
578	}
579	self.files.shrinkRetainingCapacity(input_file_count);
580
581	for (self.files.items) \|file\| {
582	self.hash.hasher.update(&file.bin_digest);
583	}
584	}
585
586	fn isProblematicTimestamp(man: *Manifest, file_time: i128) bool {
587	// If the file_time is prior to the most recent problematic timestamp
588	// then we don't need to access the filesystem.
589	if (file_time < man.recent_problematic_timestamp)
590	return false;
591
592	// Next we will check the globally shared Cache timestamp, which is accessed
593	// from multiple threads.
594	man.cache.mutex.lock();
595	defer man.cache.mutex.unlock();
596
597	// Save the global one to our local one to avoid locking next time.
598	man.recent_problematic_timestamp = man.cache.recent_problematic_timestamp;
599	if (file_time < man.recent_problematic_timestamp)
600	return false;
601
602	// This flag prevents multiple filesystem writes for the same hit() call.
603	if (man.want_refresh_timestamp) {
604	man.want_refresh_timestamp = false;
605
606	var file = man.cache.manifest_dir.createFile("timestamp", .{
607	.read = true,
608	.truncate = true,
609	}) catch return true;
610	defer file.close();
611
612	// Save locally and also save globally (we still hold the global lock).
613	man.recent_problematic_timestamp = (file.stat() catch return true).mtime;
614	man.cache.recent_problematic_timestamp = man.recent_problematic_timestamp;
615	}
616
617	return file_time >= man.recent_problematic_timestamp;
618	}
619
620	fn populateFileHash(self: Manifest, ch_file: File) !void {
621	const pp = ch_file.prefixed_path.?;
622	const dir = self.cache.prefixes()[pp.prefix].handle;
623	const file = try dir.openFile(pp.sub_path, .{});
624	defer file.close();
625
626	const actual_stat = try file.stat();
627	ch_file.stat = .{
628	.size = actual_stat.size,
629	.mtime = actual_stat.mtime,
630	.inode = actual_stat.inode,
631	};
632
633	if (self.isProblematicTimestamp(ch_file.stat.mtime)) {
634	// The actual file has an unreliable timestamp, force it to be hashed
635	ch_file.stat.mtime = 0;
636	ch_file.stat.inode = 0;
637	}
638
639	if (ch_file.max_file_size) \|max_file_size\| {
640	if (ch_file.stat.size > max_file_size) {
641	return error.FileTooBig;
642	}
643
644	const contents = try self.cache.gpa.alloc(u8, @intCast(usize, ch_file.stat.size));
645	errdefer self.cache.gpa.free(contents);
646
647	// Hash while reading from disk, to keep the contents in the cpu cache while
648	// doing hashing.
649	var hasher = hasher_init;
650	var off: usize = 0;
651	while (true) {
652	// give me everything you've got, captain
653	const bytes_read = try file.read(contents[off..]);
654	if (bytes_read == 0) break;
655	hasher.update(contents[off..][0..bytes_read]);
656	off += bytes_read;
657	}
658	hasher.final(&ch_file.bin_digest);
659
660	ch_file.contents = contents;
661	} else {
662	try hashFile(file, &ch_file.bin_digest);
663	}
664
665	self.hash.hasher.update(&ch_file.bin_digest);
666	}
667
668	/// Add a file as a dependency of process being cached, after the initial hash has been
669	/// calculated. This is useful for processes that don't know all the files that
670	/// are depended on ahead of time. For example, a source file that can import other files
671	/// will need to be recompiled if the imported file is changed.
672	pub fn addFilePostFetch(self: *Manifest, file_path: []const u8, max_file_size: usize) ![]const u8 {
673	assert(self.manifest_file != null);
674
675	const gpa = self.cache.gpa;
676	const prefixed_path = try self.cache.findPrefix(file_path);
677	errdefer gpa.free(prefixed_path.sub_path);
678
679	log.debug("Manifest.addFilePostFetch {s} -> {d} {s}", .{
680	file_path, prefixed_path.prefix, prefixed_path.sub_path,
681	});
682
683	const new_ch_file = try self.files.addOne(gpa);
684	new_ch_file.* = .{
685	.prefixed_path = prefixed_path,
686	.max_file_size = max_file_size,
687	.stat = undefined,
688	.bin_digest = undefined,
689	.contents = null,
690	};
691	errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
692
693	try self.populateFileHash(new_ch_file);
694
695	return new_ch_file.contents.?;
696	}
697
698	/// Add a file as a dependency of process being cached, after the initial hash has been
699	/// calculated. This is useful for processes that don't know the all the files that
700	/// are depended on ahead of time. For example, a source file that can import other files
701	/// will need to be recompiled if the imported file is changed.
702	pub fn addFilePost(self: *Manifest, file_path: []const u8) !void {
703	assert(self.manifest_file != null);
704
705	const gpa = self.cache.gpa;
706	const prefixed_path = try self.cache.findPrefix(file_path);
707	errdefer gpa.free(prefixed_path.sub_path);
708
709	log.debug("Manifest.addFilePost {s} -> {d} {s}", .{
710	file_path, prefixed_path.prefix, prefixed_path.sub_path,
711	});
712
713	const new_ch_file = try self.files.addOne(gpa);
714	new_ch_file.* = .{
715	.prefixed_path = prefixed_path,
716	.max_file_size = null,
717	.stat = undefined,
718	.bin_digest = undefined,
719	.contents = null,
720	};
721	errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
722
723	try self.populateFileHash(new_ch_file);
724	}
725
726	/// Like `addFilePost` but when the file contents have already been loaded from disk.
727	/// On success, cache takes ownership of `resolved_path`.
728	pub fn addFilePostContents(
729	self: *Manifest,
730	resolved_path: []u8,
731	bytes: []const u8,
732	stat: File.Stat,
733	) error{OutOfMemory}!void {
734	assert(self.manifest_file != null);
735	const gpa = self.cache.gpa;
736
737	const ch_file = try self.files.addOne(gpa);
738	errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
739
740	log.debug("Manifest.addFilePostContents resolved_path={s}", .{resolved_path});
741
742	const prefixed_path = try self.cache.findPrefixResolved(resolved_path);
743	errdefer gpa.free(prefixed_path.sub_path);
744
745	log.debug("Manifest.addFilePostContents -> {d} {s}", .{
746	prefixed_path.prefix, prefixed_path.sub_path,
747	});
748
749	ch_file.* = .{
750	.prefixed_path = prefixed_path,
751	.max_file_size = null,
752	.stat = stat,
753	.bin_digest = undefined,
754	.contents = null,
755	};
756
757	if (self.isProblematicTimestamp(ch_file.stat.mtime)) {
758	// The actual file has an unreliable timestamp, force it to be hashed
759	ch_file.stat.mtime = 0;
760	ch_file.stat.inode = 0;
761	}
762
763	{
764	var hasher = hasher_init;
765	hasher.update(bytes);
766	hasher.final(&ch_file.bin_digest);
767	}
768
769	self.hash.hasher.update(&ch_file.bin_digest);
770	}
771
772	pub fn addDepFilePost(self: *Manifest, dir: fs.Dir, dep_file_basename: []const u8) !void {
773	assert(self.manifest_file != null);
774
775	const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max);
776	defer self.cache.gpa.free(dep_file_contents);
777
778	var error_buf = std.ArrayList(u8).init(self.cache.gpa);
779	defer error_buf.deinit();
780
781	var it: @import("DepTokenizer.zig") = .{ .bytes = dep_file_contents };
782
783	// Skip first token: target.
784	switch (it.next() orelse return) { // Empty dep file OK.
785	.target, .target_must_resolve, .prereq => {},
786	else => \|err\| {
787	try err.printError(error_buf.writer());
788	log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
789	return error.InvalidDepFile;
790	},
791	}
792	// Process 0+ preqreqs.
793	// Clang is invoked in single-source mode so we never get more targets.
794	while (true) {
795	switch (it.next() orelse return) {
796	.target, .target_must_resolve => return,
797	.prereq => \|file_path\| try self.addFilePost(file_path),
798	else => \|err\| {
799	try err.printError(error_buf.writer());
800	log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
801	return error.InvalidDepFile;
802	},
803	}
804	}
805	}
806
807	/// Returns a hex encoded hash of the inputs.
808	pub fn final(self: *Manifest) [hex_digest_len]u8 {
809	assert(self.manifest_file != null);
810
811	// We don't close the manifest file yet, because we want to
812	// keep it locked until the API user is done using it.
813	// We also don't write out the manifest yet, because until
814	// cache_release is called we still might be working on creating
815	// the artifacts to cache.
816
817	var bin_digest: BinDigest = undefined;
818	self.hash.hasher.final(&bin_digest);
819
820	var out_digest: [hex_digest_len]u8 = undefined;
821	_ = std.fmt.bufPrint(
822	&out_digest,
823	"{s}",
824	.{std.fmt.fmtSliceHexLower(&bin_digest)},
825	) catch unreachable;
826
827	return out_digest;
828	}
829
830	/// If `want_shared_lock` is true, this function automatically downgrades the
831	/// lock from exclusive to shared.
832	pub fn writeManifest(self: *Manifest) !void {
833	assert(self.have_exclusive_lock);
834
835	const manifest_file = self.manifest_file.?;
836	if (self.manifest_dirty) {
837	self.manifest_dirty = false;
838
839	var contents = std.ArrayList(u8).init(self.cache.gpa);
840	defer contents.deinit();
841
842	const writer = contents.writer();
843	var encoded_digest: [hex_digest_len]u8 = undefined;
844
845	for (self.files.items) \|file\| {
846	_ = std.fmt.bufPrint(
847	&encoded_digest,
848	"{s}",
849	.{std.fmt.fmtSliceHexLower(&file.bin_digest)},
850	) catch unreachable;
851	try writer.print("{d} {d} {d} {s} {d} {s}\n", .{
852	file.stat.size,
853	file.stat.inode,
854	file.stat.mtime,
855	&encoded_digest,
856	file.prefixed_path.?.prefix,
857	file.prefixed_path.?.sub_path,
858	});
859	}
860
861	try manifest_file.setEndPos(contents.items.len);
862	try manifest_file.pwriteAll(contents.items, 0);
863	}
864
865	if (self.want_shared_lock) {
866	try self.downgradeToSharedLock();
867	}
868	}
869
870	fn downgradeToSharedLock(self: *Manifest) !void {
871	if (!self.have_exclusive_lock) return;
872
873	// WASI does not currently support flock, so we bypass it here.
874	// TODO: If/when flock is supported on WASI, this check should be removed.
875	// See https://github.com/WebAssembly/wasi-filesystem/issues/2
876	if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
877	const manifest_file = self.manifest_file.?;
878	try manifest_file.downgradeLock();
879	}
880
881	self.have_exclusive_lock = false;
882	}
883
884	fn upgradeToExclusiveLock(self: *Manifest) !void {
885	if (self.have_exclusive_lock) return;
886	assert(self.manifest_file != null);
887
888	// WASI does not currently support flock, so we bypass it here.
889	// TODO: If/when flock is supported on WASI, this check should be removed.
890	// See https://github.com/WebAssembly/wasi-filesystem/issues/2
891	if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
892	const manifest_file = self.manifest_file.?;
893	// Here we intentionally have a period where the lock is released, in case there are
894	// other processes holding a shared lock.
895	manifest_file.unlock();
896	try manifest_file.lock(.Exclusive);
897	}
898	self.have_exclusive_lock = true;
899	}
900
901	/// Obtain only the data needed to maintain a lock on the manifest file.
902	/// The `Manifest` remains safe to deinit.
903	/// Don't forget to call `writeManifest` before this!
904	pub fn toOwnedLock(self: *Manifest) Lock {
905	const lock: Lock = .{
906	.manifest_file = self.manifest_file.?,
907	};
908
909	self.manifest_file = null;
910	return lock;
911	}
912
913	/// Releases the manifest file and frees any memory the Manifest was using.
914	/// `Manifest.hit` must be called first.
915	/// Don't forget to call `writeManifest` before this!
916	pub fn deinit(self: *Manifest) void {
917	if (self.manifest_file) \|file\| {
918	if (builtin.os.tag == .windows) {
919	// See Lock.release for why this is required on Windows
920	file.unlock();
921	}
922
923	file.close();
924	}
925	for (self.files.items) \|*file\| {
926	file.deinit(self.cache.gpa);
927	}
928	self.files.deinit(self.cache.gpa);
929	}
930	};
931
932	/// On operating systems that support symlinks, does a readlink. On other operating systems,
933	/// uses the file contents. Windows supports symlinks but only with elevated privileges, so
934	/// it is treated as not supporting symlinks.
935	pub fn readSmallFile(dir: fs.Dir, sub_path: []const u8, buffer: []u8) ![]u8 {
936	if (builtin.os.tag == .windows) {
937	return dir.readFile(sub_path, buffer);
938	} else {
939	return dir.readLink(sub_path, buffer);
940	}
941	}
942
943	/// On operating systems that support symlinks, does a symlink. On other operating systems,
944	/// uses the file contents. Windows supports symlinks but only with elevated privileges, so
945	/// it is treated as not supporting symlinks.
946	/// `data` must be a valid UTF-8 encoded file path and 255 bytes or fewer.
947	pub fn writeSmallFile(dir: fs.Dir, sub_path: []const u8, data: []const u8) !void {
948	assert(data.len <= 255);
949	if (builtin.os.tag == .windows) {
950	return dir.writeFile(sub_path, data);
951	} else {
952	return dir.symLink(data, sub_path, .{});
953	}
954	}
955
956	fn hashFile(file: fs.File, bin_digest: *[Hasher.mac_length]u8) !void {
957	var buf: [1024]u8 = undefined;
958
959	var hasher = hasher_init;
960	while (true) {
961	const bytes_read = try file.read(&buf);
962	if (bytes_read == 0) break;
963	hasher.update(buf[0..bytes_read]);
964	}
965
966	hasher.final(bin_digest);
967	}
968
969	// Create/Write a file, close it, then grab its stat.mtime timestamp.
970	fn testGetCurrentFileTimestamp() !i128 {
971	var file = try fs.cwd().createFile("test-filetimestamp.tmp", .{
972	.read = true,
973	.truncate = true,
974	});
975	defer file.close();
976
977	return (try file.stat()).mtime;
978	}
979
980	test "cache file and then recall it" {
981	if (builtin.os.tag == .wasi) {
982	// https://github.com/ziglang/zig/issues/5437
983	return error.SkipZigTest;
984	}
985
986	const cwd = fs.cwd();
987
988	const temp_file = "test.txt";
989	const temp_manifest_dir = "temp_manifest_dir";
990
991	try cwd.writeFile(temp_file, "Hello, world!\n");
992
993	// Wait for file timestamps to tick
994	const initial_time = try testGetCurrentFileTimestamp();
995	while ((try testGetCurrentFileTimestamp()) == initial_time) {
996	std.time.sleep(1);
997	}
998
999	var digest1: [hex_digest_len]u8 = undefined;
1000	var digest2: [hex_digest_len]u8 = undefined;
1001
1002	{
1003	var cache = Cache{
1004	.gpa = testing.allocator,
1005	.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
1006	};
1007	cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
1008	defer cache.manifest_dir.close();
1009
1010	{
1011	var ch = cache.obtain();
1012	defer ch.deinit();
1013
1014	ch.hash.add(true);
1015	ch.hash.add(@as(u16, 1234));
1016	ch.hash.addBytes("1234");
1017	_ = try ch.addFile(temp_file, null);
1018
1019	// There should be nothing in the cache
1020	try testing.expectEqual(false, try ch.hit());
1021
1022	digest1 = ch.final();
1023	try ch.writeManifest();
1024	}
1025	{
1026	var ch = cache.obtain();
1027	defer ch.deinit();
1028
1029	ch.hash.add(true);
1030	ch.hash.add(@as(u16, 1234));
1031	ch.hash.addBytes("1234");
1032	_ = try ch.addFile(temp_file, null);
1033
1034	// Cache hit! We just "built" the same file
1035	try testing.expect(try ch.hit());
1036	digest2 = ch.final();
1037
1038	try testing.expectEqual(false, ch.have_exclusive_lock);
1039	}
1040
1041	try testing.expectEqual(digest1, digest2);
1042	}
1043
1044	try cwd.deleteTree(temp_manifest_dir);
1045	try cwd.deleteFile(temp_file);
1046	}
1047
1048	test "check that changing a file makes cache fail" {
1049	if (builtin.os.tag == .wasi) {
1050	// https://github.com/ziglang/zig/issues/5437
1051	return error.SkipZigTest;
1052	}
1053	const cwd = fs.cwd();
1054
1055	const temp_file = "cache_hash_change_file_test.txt";
1056	const temp_manifest_dir = "cache_hash_change_file_manifest_dir";
1057	const original_temp_file_contents = "Hello, world!\n";
1058	const updated_temp_file_contents = "Hello, world; but updated!\n";
1059
1060	try cwd.deleteTree(temp_manifest_dir);
1061	try cwd.deleteTree(temp_file);
1062
1063	try cwd.writeFile(temp_file, original_temp_file_contents);
1064
1065	// Wait for file timestamps to tick
1066	const initial_time = try testGetCurrentFileTimestamp();
1067	while ((try testGetCurrentFileTimestamp()) == initial_time) {
1068	std.time.sleep(1);
1069	}
1070
1071	var digest1: [hex_digest_len]u8 = undefined;
1072	var digest2: [hex_digest_len]u8 = undefined;
1073
1074	{
1075	var cache = Cache{
1076	.gpa = testing.allocator,
1077	.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
1078	};
1079	cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
1080	defer cache.manifest_dir.close();
1081
1082	{
1083	var ch = cache.obtain();
1084	defer ch.deinit();
1085
1086	ch.hash.addBytes("1234");
1087	const temp_file_idx = try ch.addFile(temp_file, 100);
1088
1089	// There should be nothing in the cache
1090	try testing.expectEqual(false, try ch.hit());
1091
1092	try testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
1093
1094	digest1 = ch.final();
1095
1096	try ch.writeManifest();
1097	}
1098
1099	try cwd.writeFile(temp_file, updated_temp_file_contents);
1100
1101	{
1102	var ch = cache.obtain();
1103	defer ch.deinit();
1104
1105	ch.hash.addBytes("1234");
1106	const temp_file_idx = try ch.addFile(temp_file, 100);
1107
1108	// A file that we depend on has been updated, so the cache should not contain an entry for it
1109	try testing.expectEqual(false, try ch.hit());
1110
1111	// The cache system does not keep the contents of re-hashed input files.
1112	try testing.expect(ch.files.items[temp_file_idx].contents == null);
1113
1114	digest2 = ch.final();
1115
1116	try ch.writeManifest();
1117	}
1118
1119	try testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));
1120	}
1121
1122	try cwd.deleteTree(temp_manifest_dir);
1123	try cwd.deleteTree(temp_file);
1124	}
1125
1126	test "no file inputs" {
1127	if (builtin.os.tag == .wasi) {
1128	// https://github.com/ziglang/zig/issues/5437
1129	return error.SkipZigTest;
1130	}
1131	const cwd = fs.cwd();
1132	const temp_manifest_dir = "no_file_inputs_manifest_dir";
1133	defer cwd.deleteTree(temp_manifest_dir) catch {};
1134
1135	var digest1: [hex_digest_len]u8 = undefined;
1136	var digest2: [hex_digest_len]u8 = undefined;
1137
1138	var cache = Cache{
1139	.gpa = testing.allocator,
1140	.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
1141	};
1142	cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
1143	defer cache.manifest_dir.close();
1144
1145	{
1146	var man = cache.obtain();
1147	defer man.deinit();
1148
1149	man.hash.addBytes("1234");
1150
1151	// There should be nothing in the cache
1152	try testing.expectEqual(false, try man.hit());
1153
1154	digest1 = man.final();
1155
1156	try man.writeManifest();
1157	}
1158	{
1159	var man = cache.obtain();
1160	defer man.deinit();
1161
1162	man.hash.addBytes("1234");
1163
1164	try testing.expect(try man.hit());
1165	digest2 = man.final();
1166	try testing.expectEqual(false, man.have_exclusive_lock);
1167	}
1168
1169	try testing.expectEqual(digest1, digest2);
1170	}
1171
1172	test "Manifest with files added after initial hash work" {
1173	if (builtin.os.tag == .wasi) {
1174	// https://github.com/ziglang/zig/issues/5437
1175	return error.SkipZigTest;
1176	}
1177	const cwd = fs.cwd();
1178
1179	const temp_file1 = "cache_hash_post_file_test1.txt";
1180	const temp_file2 = "cache_hash_post_file_test2.txt";
1181	const temp_manifest_dir = "cache_hash_post_file_manifest_dir";
1182
1183	try cwd.writeFile(temp_file1, "Hello, world!\n");
1184	try cwd.writeFile(temp_file2, "Hello world the second!\n");
1185
1186	// Wait for file timestamps to tick
1187	const initial_time = try testGetCurrentFileTimestamp();
1188	while ((try testGetCurrentFileTimestamp()) == initial_time) {
1189	std.time.sleep(1);
1190	}
1191
1192	var digest1: [hex_digest_len]u8 = undefined;
1193	var digest2: [hex_digest_len]u8 = undefined;
1194	var digest3: [hex_digest_len]u8 = undefined;
1195
1196	{
1197	var cache = Cache{
1198	.gpa = testing.allocator,
1199	.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
1200	};
1201	cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
1202	defer cache.manifest_dir.close();
1203
1204	{
1205	var ch = cache.obtain();
1206	defer ch.deinit();
1207
1208	ch.hash.addBytes("1234");
1209	_ = try ch.addFile(temp_file1, null);
1210
1211	// There should be nothing in the cache
1212	try testing.expectEqual(false, try ch.hit());
1213
1214	_ = try ch.addFilePost(temp_file2);
1215
1216	digest1 = ch.final();
1217	try ch.writeManifest();
1218	}
1219	{
1220	var ch = cache.obtain();
1221	defer ch.deinit();
1222
1223	ch.hash.addBytes("1234");
1224	_ = try ch.addFile(temp_file1, null);
1225
1226	try testing.expect(try ch.hit());
1227	digest2 = ch.final();
1228
1229	try testing.expectEqual(false, ch.have_exclusive_lock);
1230	}
1231	try testing.expect(mem.eql(u8, &digest1, &digest2));
1232
1233	// Modify the file added after initial hash
1234	try cwd.writeFile(temp_file2, "Hello world the second, updated\n");
1235
1236	// Wait for file timestamps to tick
1237	const initial_time2 = try testGetCurrentFileTimestamp();
1238	while ((try testGetCurrentFileTimestamp()) == initial_time2) {
1239	std.time.sleep(1);
1240	}
1241
1242	{
1243	var ch = cache.obtain();
1244	defer ch.deinit();
1245
1246	ch.hash.addBytes("1234");
1247	_ = try ch.addFile(temp_file1, null);
1248
1249	// A file that we depend on has been updated, so the cache should not contain an entry for it
1250	try testing.expectEqual(false, try ch.hit());
1251
1252	_ = try ch.addFilePost(temp_file2);
1253
1254	digest3 = ch.final();
1255
1256	try ch.writeManifest();
1257	}
1258
1259	try testing.expect(!mem.eql(u8, &digest1, &digest3));
1260	}
1261
1262	try cwd.deleteTree(temp_manifest_dir);
1263	try cwd.deleteFile(temp_file1);
1264	try cwd.deleteFile(temp_file2);
1265	}

src/Compilation.zig+42-50

 @@ -26,7 +26,7 @@ const wasi_libc = @import("wasi_libc.zig");
 const fatal = @import("main.zig").fatal;
 const clangMain = @import("main.zig").clangMain;
 const Module = @import("Module.zig");
-const Cache = @import("Cache.zig");
+const Cache = std.Build.Cache;
 const translate_c = @import("translate_c.zig");
 const clang = @import("clang.zig");
 const c_codegen = @import("codegen/c.zig");
 @@ -807,44 +807,7 @@ pub const AllErrors = struct {
     }
 };
-pub const Directory = struct {
+pub const Directory = Cache.Directory;
-    /// This field is redundant for operations that can act on the open directory handle
-    /// directly, but it is needed when passing the directory to a child process.
-    /// `null` means cwd.
-    path: ?[]const u8,
-    handle: std.fs.Dir,
-    pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
-        if (self.path) |p| {
-            // TODO clean way to do this with only 1 allocation
-            const part2 = try std.fs.path.join(allocator, paths);
-            defer allocator.free(part2);
-            return std.fs.path.join(allocator, &[_][]const u8{ p, part2 });
-        } else {
-            return std.fs.path.join(allocator, paths);
-        }
-    }
-    pub fn joinZ(self: Directory, allocator: Allocator, paths: []const []const u8) ![:0]u8 {
-        if (self.path) |p| {
-            // TODO clean way to do this with only 1 allocation
-            const part2 = try std.fs.path.join(allocator, paths);
-            defer allocator.free(part2);
-            return std.fs.path.joinZ(allocator, &[_][]const u8{ p, part2 });
-        } else {
-            return std.fs.path.joinZ(allocator, paths);
-        }
-    }
-    /// Whether or not the handle should be closed, or the path should be freed
-    /// is determined by usage, however this function is provided for convenience
-    /// if it happens to be what the caller needs.
-    pub fn closeAndFree(self: *Directory, gpa: Allocator) void {
-        self.handle.close();
-        if (self.path) |p| gpa.free(p);
-        self.* = undefined;
-    }
-};
 pub const EmitLoc = struct {
     /// If this is `null` it means the file will be output to the cache directory.
 @@ -854,6 +817,35 @@ pub const EmitLoc = struct {
     basename: []const u8,
 };
+pub const cache_helpers = struct {
+    pub fn addEmitLoc(hh: *Cache.HashHelper, emit_loc: EmitLoc) void {
+        hh.addBytes(emit_loc.basename);
+    }
+    pub fn addOptionalEmitLoc(hh: *Cache.HashHelper, optional_emit_loc: ?EmitLoc) void {
+        hh.add(optional_emit_loc != null);
+        addEmitLoc(hh, optional_emit_loc orelse return);
+    }
+    pub fn hashCSource(self: *Cache.Manifest, c_source: Compilation.CSourceFile) !void {
+        _ = try self.addFile(c_source.src_path, null);
+        // Hash the extra flags, with special care to call addFile for file parameters.
+        // TODO this logic can likely be improved by utilizing clang_options_data.zig.
+        const file_args = [_][]const u8{"-include"};
+        var arg_i: usize = 0;
+        while (arg_i < c_source.extra_flags.len) : (arg_i += 1) {
+            const arg = c_source.extra_flags[arg_i];
+            self.hash.addBytes(arg);
+            for (file_args) |file_arg| {
+                if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_source.extra_flags.len) {
+                    arg_i += 1;
+                    _ = try self.addFile(c_source.extra_flags[arg_i], null);
+                }
+            }
+        }
+    }
+};
 pub const ClangPreprocessorMode = enum {
     no,
     /// This means we are doing `zig cc -E -o <path>`.
 @@ -1523,8 +1515,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
         cache.hash.add(link_libunwind);
         cache.hash.add(options.output_mode);
         cache.hash.add(options.machine_code_model);
-        cache.hash.addOptionalEmitLoc(options.emit_bin);
+        cache_helpers.addOptionalEmitLoc(&cache.hash, options.emit_bin);
-        cache.hash.addOptionalEmitLoc(options.emit_implib);
+        cache_helpers.addOptionalEmitLoc(&cache.hash, options.emit_implib);
         cache.hash.addBytes(options.root_name);
         if (options.target.os.tag == .wasi) cache.hash.add(wasi_exec_model);
         // TODO audit this and make sure everything is in it
 @@ -2638,11 +2630,11 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes
         man.hash.addListOfBytes(key.src.extra_flags);
     }
-    man.hash.addOptionalEmitLoc(comp.emit_asm);
+    cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_asm);
-    man.hash.addOptionalEmitLoc(comp.emit_llvm_ir);
+    cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_ir);
-    man.hash.addOptionalEmitLoc(comp.emit_llvm_bc);
+    cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_bc);
-    man.hash.addOptionalEmitLoc(comp.emit_analysis);
+    cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_analysis);
-    man.hash.addOptionalEmitLoc(comp.emit_docs);
+    cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_docs);
     man.hash.addListOfBytes(comp.clang_argv);
 @@ -3961,11 +3953,11 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P
     defer man.deinit();
     man.hash.add(comp.clang_preprocessor_mode);
-    man.hash.addOptionalEmitLoc(comp.emit_asm);
+    cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_asm);
-    man.hash.addOptionalEmitLoc(comp.emit_llvm_ir);
+    cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_ir);
-    man.hash.addOptionalEmitLoc(comp.emit_llvm_bc);
+    cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_bc);
-    try man.hashCSource(c_object.src);
+    try cache_helpers.hashCSource(&man, c_object.src);
     var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
     defer arena_allocator.deinit();

src/DepTokenizer.zig deleted-1069

...	@@ -1,1069 +0,0 @@
1	const Tokenizer = @This();
2
3	index: usize = 0,
4	bytes: []const u8,
5	state: State = .lhs,
6
7	const std = @import("std");
8	const testing = std.testing;
9	const assert = std.debug.assert;
10
11	pub fn next(self: *Tokenizer) ?Token {
12	var start = self.index;
13	var must_resolve = false;
14	while (self.index < self.bytes.len) {
15	const char = self.bytes[self.index];
16	switch (self.state) {
17	.lhs => switch (char) {
18	'\t', '\n', '\r', ' ' => {
19	// silently ignore whitespace
20	self.index += 1;
21	},
22	else => {
23	start = self.index;
24	self.state = .target;
25	},
26	},
27	.target => switch (char) {
28	'\t', '\n', '\r', ' ' => {
29	return errorIllegalChar(.invalid_target, self.index, char);
30	},
31	'$' => {
32	self.state = .target_dollar_sign;
33	self.index += 1;
34	},
35	'\\' => {
36	self.state = .target_reverse_solidus;
37	self.index += 1;
38	},
39	':' => {
40	self.state = .target_colon;
41	self.index += 1;
42	},
43	else => {
44	self.index += 1;
45	},
46	},
47	.target_reverse_solidus => switch (char) {
48	'\t', '\n', '\r' => {
49	return errorIllegalChar(.bad_target_escape, self.index, char);
50	},
51	' ', '#', '\\' => {
52	must_resolve = true;
53	self.state = .target;
54	self.index += 1;
55	},
56	'$' => {
57	self.state = .target_dollar_sign;
58	self.index += 1;
59	},
60	else => {
61	self.state = .target;
62	self.index += 1;
63	},
64	},
65	.target_dollar_sign => switch (char) {
66	'$' => {
67	must_resolve = true;
68	self.state = .target;
69	self.index += 1;
70	},
71	else => {
72	return errorIllegalChar(.expected_dollar_sign, self.index, char);
73	},
74	},
75	.target_colon => switch (char) {
76	'\n', '\r' => {
77	const bytes = self.bytes[start .. self.index - 1];
78	if (bytes.len != 0) {
79	self.state = .lhs;
80	return finishTarget(must_resolve, bytes);
81	}
82	// silently ignore null target
83	self.state = .lhs;
84	},
85	'/', '\\' => {
86	self.state = .target_colon_reverse_solidus;
87	self.index += 1;
88	},
89	else => {
90	const bytes = self.bytes[start .. self.index - 1];
91	if (bytes.len != 0) {
92	self.state = .rhs;
93	return finishTarget(must_resolve, bytes);
94	}
95	// silently ignore null target
96	self.state = .lhs;
97	},
98	},
99	.target_colon_reverse_solidus => switch (char) {
100	'\n', '\r' => {
101	const bytes = self.bytes[start .. self.index - 2];
102	if (bytes.len != 0) {
103	self.state = .lhs;
104	return finishTarget(must_resolve, bytes);
105	}
106	// silently ignore null target
107	self.state = .lhs;
108	},
109	else => {
110	self.state = .target;
111	},
112	},
113	.rhs => switch (char) {
114	'\t', ' ' => {
115	// silently ignore horizontal whitespace
116	self.index += 1;
117	},
118	'\n', '\r' => {
119	self.state = .lhs;
120	},
121	'\\' => {
122	self.state = .rhs_continuation;
123	self.index += 1;
124	},
125	'"' => {
126	self.state = .prereq_quote;
127	self.index += 1;
128	start = self.index;
129	},
130	else => {
131	start = self.index;
132	self.state = .prereq;
133	},
134	},
135	.rhs_continuation => switch (char) {
136	'\n' => {
137	self.state = .rhs;
138	self.index += 1;
139	},
140	'\r' => {
141	self.state = .rhs_continuation_linefeed;
142	self.index += 1;
143	},
144	else => {
145	return errorIllegalChar(.continuation_eol, self.index, char);
146	},
147	},
148	.rhs_continuation_linefeed => switch (char) {
149	'\n' => {
150	self.state = .rhs;
151	self.index += 1;
152	},
153	else => {
154	return errorIllegalChar(.continuation_eol, self.index, char);
155	},
156	},
157	.prereq_quote => switch (char) {
158	'"' => {
159	self.index += 1;
160	self.state = .rhs;
161	return Token{ .prereq = self.bytes[start .. self.index - 1] };
162	},
163	else => {
164	self.index += 1;
165	},
166	},
167	.prereq => switch (char) {
168	'\t', ' ' => {
169	self.state = .rhs;
170	return Token{ .prereq = self.bytes[start..self.index] };
171	},
172	'\n', '\r' => {
173	self.state = .lhs;
174	return Token{ .prereq = self.bytes[start..self.index] };
175	},
176	'\\' => {
177	self.state = .prereq_continuation;
178	self.index += 1;
179	},
180	else => {
181	self.index += 1;
182	},
183	},
184	.prereq_continuation => switch (char) {
185	'\n' => {
186	self.index += 1;
187	self.state = .rhs;
188	return Token{ .prereq = self.bytes[start .. self.index - 2] };
189	},
190	'\r' => {
191	self.state = .prereq_continuation_linefeed;
192	self.index += 1;
193	},
194	else => {
195	// not continuation
196	self.state = .prereq;
197	self.index += 1;
198	},
199	},
200	.prereq_continuation_linefeed => switch (char) {
201	'\n' => {
202	self.index += 1;
203	self.state = .rhs;
204	return Token{ .prereq = self.bytes[start .. self.index - 1] };
205	},
206	else => {
207	return errorIllegalChar(.continuation_eol, self.index, char);
208	},
209	},
210	}
211	} else {
212	switch (self.state) {
213	.lhs,
214	.rhs,
215	.rhs_continuation,
216	.rhs_continuation_linefeed,
217	=> return null,
218	.target => {
219	return errorPosition(.incomplete_target, start, self.bytes[start..]);
220	},
221	.target_reverse_solidus,
222	.target_dollar_sign,
223	=> {
224	const idx = self.index - 1;
225	return errorIllegalChar(.incomplete_escape, idx, self.bytes[idx]);
226	},
227	.target_colon => {
228	const bytes = self.bytes[start .. self.index - 1];
229	if (bytes.len != 0) {
230	self.index += 1;
231	self.state = .rhs;
232	return finishTarget(must_resolve, bytes);
233	}
234	// silently ignore null target
235	self.state = .lhs;
236	return null;
237	},
238	.target_colon_reverse_solidus => {
239	const bytes = self.bytes[start .. self.index - 2];
240	if (bytes.len != 0) {
241	self.index += 1;
242	self.state = .rhs;
243	return finishTarget(must_resolve, bytes);
244	}
245	// silently ignore null target
246	self.state = .lhs;
247	return null;
248	},
249	.prereq_quote => {
250	return errorPosition(.incomplete_quoted_prerequisite, start, self.bytes[start..]);
251	},
252	.prereq => {
253	self.state = .lhs;
254	return Token{ .prereq = self.bytes[start..] };
255	},
256	.prereq_continuation => {
257	self.state = .lhs;
258	return Token{ .prereq = self.bytes[start .. self.index - 1] };
259	},
260	.prereq_continuation_linefeed => {
261	self.state = .lhs;
262	return Token{ .prereq = self.bytes[start .. self.index - 2] };
263	},
264	}
265	}
266	unreachable;
267	}
268
269	fn errorPosition(comptime id: std.meta.Tag(Token), index: usize, bytes: []const u8) Token {
270	return @unionInit(Token, @tagName(id), .{ .index = index, .bytes = bytes });
271	}
272
273	fn errorIllegalChar(comptime id: std.meta.Tag(Token), index: usize, char: u8) Token {
274	return @unionInit(Token, @tagName(id), .{ .index = index, .char = char });
275	}
276
277	fn finishTarget(must_resolve: bool, bytes: []const u8) Token {
278	return if (must_resolve) .{ .target_must_resolve = bytes } else .{ .target = bytes };
279	}
280
281	const State = enum {
282	lhs,
283	target,
284	target_reverse_solidus,
285	target_dollar_sign,
286	target_colon,
287	target_colon_reverse_solidus,
288	rhs,
289	rhs_continuation,
290	rhs_continuation_linefeed,
291	prereq_quote,
292	prereq,
293	prereq_continuation,
294	prereq_continuation_linefeed,
295	};
296
297	pub const Token = union(enum) {
298	target: []const u8,
299	target_must_resolve: []const u8,
300	prereq: []const u8,
301
302	incomplete_quoted_prerequisite: IndexAndBytes,
303	incomplete_target: IndexAndBytes,
304
305	invalid_target: IndexAndChar,
306	bad_target_escape: IndexAndChar,
307	expected_dollar_sign: IndexAndChar,
308	continuation_eol: IndexAndChar,
309	incomplete_escape: IndexAndChar,
310
311	pub const IndexAndChar = struct {
312	index: usize,
313	char: u8,
314	};
315
316	pub const IndexAndBytes = struct {
317	index: usize,
318	bytes: []const u8,
319	};
320
321	/// Resolve escapes in target. Only valid with .target_must_resolve.
322	pub fn resolve(self: Token, writer: anytype) @TypeOf(writer).Error!void {
323	const bytes = self.target_must_resolve; // resolve called on incorrect token
324
325	var state: enum { start, escape, dollar } = .start;
326	for (bytes) \|c\| {
327	switch (state) {
328	.start => {
329	switch (c) {
330	'\\' => state = .escape,
331	'$' => state = .dollar,
332	else => try writer.writeByte(c),
333	}
334	},
335	.escape => {
336	switch (c) {
337	' ', '#', '\\' => {},
338	'$' => {
339	try writer.writeByte('\\');
340	state = .dollar;
341	continue;
342	},
343	else => try writer.writeByte('\\'),
344	}
345	try writer.writeByte(c);
346	state = .start;
347	},
348	.dollar => {
349	try writer.writeByte('$');
350	switch (c) {
351	'$' => {},
352	else => try writer.writeByte(c),
353	}
354	state = .start;
355	},
356	}
357	}
358	}
359
360	pub fn printError(self: Token, writer: anytype) @TypeOf(writer).Error!void {
361	switch (self) {
362	.target, .target_must_resolve, .prereq => unreachable, // not an error
363	.incomplete_quoted_prerequisite,
364	.incomplete_target,
365	=> \|index_and_bytes\| {
366	try writer.print("{s} '", .{self.errStr()});
367	if (self == .incomplete_target) {
368	const tmp = Token{ .target_must_resolve = index_and_bytes.bytes };
369	try tmp.resolve(writer);
370	} else {
371	try printCharValues(writer, index_and_bytes.bytes);
372	}
373	try writer.print("' at position {d}", .{index_and_bytes.index});
374	},
375	.invalid_target,
376	.bad_target_escape,
377	.expected_dollar_sign,
378	.continuation_eol,
379	.incomplete_escape,
380	=> \|index_and_char\| {
381	try writer.writeAll("illegal char ");
382	try printUnderstandableChar(writer, index_and_char.char);
383	try writer.print(" at position {d}: {s}", .{ index_and_char.index, self.errStr() });
384	},
385	}
386	}
387
388	fn errStr(self: Token) []const u8 {
389	return switch (self) {
390	.target, .target_must_resolve, .prereq => unreachable, // not an error
391	.incomplete_quoted_prerequisite => "incomplete quoted prerequisite",
392	.incomplete_target => "incomplete target",
393	.invalid_target => "invalid target",
394	.bad_target_escape => "bad target escape",
395	.expected_dollar_sign => "expecting '$'",
396	.continuation_eol => "continuation expecting end-of-line",
397	.incomplete_escape => "incomplete escape",
398	};
399	}
400	};
401
402	test "empty file" {
403	try depTokenizer("", "");
404	}
405
406	test "empty whitespace" {
407	try depTokenizer("\n", "");
408	try depTokenizer("\r", "");
409	try depTokenizer("\r\n", "");
410	try depTokenizer(" ", "");
411	}
412
413	test "empty colon" {
414	try depTokenizer(":", "");
415	try depTokenizer("\n:", "");
416	try depTokenizer("\r:", "");
417	try depTokenizer("\r\n:", "");
418	try depTokenizer(" :", "");
419	}
420
421	test "empty target" {
422	try depTokenizer("foo.o:", "target = {foo.o}");
423	try depTokenizer(
424	\\foo.o:
425	\\bar.o:
426	\\abcd.o:
427	,
428	\\target = {foo.o}
429	\\target = {bar.o}
430	\\target = {abcd.o}
431	);
432	}
433
434	test "whitespace empty target" {
435	try depTokenizer("\nfoo.o:", "target = {foo.o}");
436	try depTokenizer("\rfoo.o:", "target = {foo.o}");
437	try depTokenizer("\r\nfoo.o:", "target = {foo.o}");
438	try depTokenizer(" foo.o:", "target = {foo.o}");
439	}
440
441	test "escape empty target" {
442	try depTokenizer("\\ foo.o:", "target = { foo.o}");
443	try depTokenizer("\\#foo.o:", "target = {#foo.o}");
444	try depTokenizer("\\\\foo.o:", "target = {\\foo.o}");
445	try depTokenizer("$$foo.o:", "target = {$foo.o}");
446	}
447
448	test "empty target linefeeds" {
449	try depTokenizer("\n", "");
450	try depTokenizer("\r\n", "");
451
452	const expect = "target = {foo.o}";
453	try depTokenizer(
454	\\foo.o:
455	, expect);
456	try depTokenizer(
457	\\foo.o:
458	\\
459	, expect);
460	try depTokenizer(
461	\\foo.o:
462	, expect);
463	try depTokenizer(
464	\\foo.o:
465	\\
466	, expect);
467	}
468
469	test "empty target linefeeds + continuations" {
470	const expect = "target = {foo.o}";
471	try depTokenizer(
472	\\foo.o:\
473	, expect);
474	try depTokenizer(
475	\\foo.o:\
476	\\
477	, expect);
478	try depTokenizer(
479	\\foo.o:\
480	, expect);
481	try depTokenizer(
482	\\foo.o:\
483	\\
484	, expect);
485	}
486
487	test "empty target linefeeds + hspace + continuations" {
488	const expect = "target = {foo.o}";
489	try depTokenizer(
490	\\foo.o: \
491	, expect);
492	try depTokenizer(
493	\\foo.o: \
494	\\
495	, expect);
496	try depTokenizer(
497	\\foo.o: \
498	, expect);
499	try depTokenizer(
500	\\foo.o: \
501	\\
502	, expect);
503	}
504
505	test "prereq" {
506	const expect =
507	\\target = {foo.o}
508	\\prereq = {foo.c}
509	;
510	try depTokenizer("foo.o: foo.c", expect);
511	try depTokenizer(
512	\\foo.o: \
513	\\foo.c
514	, expect);
515	try depTokenizer(
516	\\foo.o: \
517	\\ foo.c
518	, expect);
519	try depTokenizer(
520	\\foo.o: \
521	\\ foo.c
522	, expect);
523	}
524
525	test "prereq continuation" {
526	const expect =
527	\\target = {foo.o}
528	\\prereq = {foo.h}
529	\\prereq = {bar.h}
530	;
531	try depTokenizer(
532	\\foo.o: foo.h\
533	\\bar.h
534	, expect);
535	try depTokenizer(
536	\\foo.o: foo.h\
537	\\bar.h
538	, expect);
539	}
540
541	test "multiple prereqs" {
542	const expect =
543	\\target = {foo.o}
544	\\prereq = {foo.c}
545	\\prereq = {foo.h}
546	\\prereq = {bar.h}
547	;
548	try depTokenizer("foo.o: foo.c foo.h bar.h", expect);
549	try depTokenizer(
550	\\foo.o: \
551	\\foo.c foo.h bar.h
552	, expect);
553	try depTokenizer(
554	\\foo.o: foo.c foo.h bar.h\
555	, expect);
556	try depTokenizer(
557	\\foo.o: foo.c foo.h bar.h\
558	\\
559	, expect);
560	try depTokenizer(
561	\\foo.o: \
562	\\foo.c \
563	\\ foo.h\
564	\\bar.h
565	\\
566	, expect);
567	try depTokenizer(
568	\\foo.o: \
569	\\foo.c \
570	\\ foo.h\
571	\\bar.h\
572	\\
573	, expect);
574	try depTokenizer(
575	\\foo.o: \
576	\\foo.c \
577	\\ foo.h\
578	\\bar.h\
579	, expect);
580	}
581
582	test "multiple targets and prereqs" {
583	try depTokenizer(
584	\\foo.o: foo.c
585	\\bar.o: bar.c a.h b.h c.h
586	\\abc.o: abc.c \
587	\\ one.h two.h \
588	\\ three.h four.h
589	,
590	\\target = {foo.o}
591	\\prereq = {foo.c}
592	\\target = {bar.o}
593	\\prereq = {bar.c}
594	\\prereq = {a.h}
595	\\prereq = {b.h}
596	\\prereq = {c.h}
597	\\target = {abc.o}
598	\\prereq = {abc.c}
599	\\prereq = {one.h}
600	\\prereq = {two.h}
601	\\prereq = {three.h}
602	\\prereq = {four.h}
603	);
604	try depTokenizer(
605	\\ascii.o: ascii.c
606	\\base64.o: base64.c stdio.h
607	\\elf.o: elf.c a.h b.h c.h
608	\\macho.o: \
609	\\ macho.c\
610	\\ a.h b.h c.h
611	,
612	\\target = {ascii.o}
613	\\prereq = {ascii.c}
614	\\target = {base64.o}
615	\\prereq = {base64.c}
616	\\prereq = {stdio.h}
617	\\target = {elf.o}
618	\\prereq = {elf.c}
619	\\prereq = {a.h}
620	\\prereq = {b.h}
621	\\prereq = {c.h}
622	\\target = {macho.o}
623	\\prereq = {macho.c}
624	\\prereq = {a.h}
625	\\prereq = {b.h}
626	\\prereq = {c.h}
627	);
628	try depTokenizer(
629	\\a$$scii.o: ascii.c
630	\\\\base64.o: "\base64.c" "s t#dio.h"
631	\\e\\lf.o: "e\lf.c" "a.h$$" "$$b.h c.h$$"
632	\\macho.o: \
633	\\ "macho!.c" \
634	\\ a.h b.h c.h
635	,
636	\\target = {a$scii.o}
637	\\prereq = {ascii.c}
638	\\target = {\base64.o}
639	\\prereq = {\base64.c}
640	\\prereq = {s t#dio.h}
641	\\target = {e\lf.o}
642	\\prereq = {e\lf.c}
643	\\prereq = {a.h$$}
644	\\prereq = {$$b.h c.h$$}
645	\\target = {macho.o}
646	\\prereq = {macho!.c}
647	\\prereq = {a.h}
648	\\prereq = {b.h}
649	\\prereq = {c.h}
650	);
651	}
652
653	test "windows quoted prereqs" {
654	try depTokenizer(
655	\\c:\foo.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo.c"
656	\\c:\foo2.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo2.c" \
657	\\ "C:\Program Files (x86)\Microsoft Visual Studio\foo1.h" \
658	\\ "C:\Program Files (x86)\Microsoft Visual Studio\foo2.h"
659	,
660	\\target = {c:\foo.o}
661	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo.c}
662	\\target = {c:\foo2.o}
663	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.c}
664	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo1.h}
665	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.h}
666	);
667	}
668
669	test "windows mixed prereqs" {
670	try depTokenizer(
671	\\cimport.o: \
672	\\ C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h \
673	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h" \
674	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h" \
675	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h" \
676	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h" \
677	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h" \
678	\\ C:\msys64\opt\zig\lib\zig\include\vadefs.h \
679	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h" \
680	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h" \
681	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h" \
682	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h" \
683	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h" \
684	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h" \
685	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h" \
686	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h" \
687	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h"
688	,
689	\\target = {cimport.o}
690	\\prereq = {C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h}
691	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h}
692	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h}
693	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h}
694	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h}
695	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h}
696	\\prereq = {C:\msys64\opt\zig\lib\zig\include\vadefs.h}
697	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h}
698	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h}
699	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h}
700	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h}
701	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h}
702	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h}
703	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h}
704	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h}
705	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h}
706	);
707	}
708
709	test "windows funky targets" {
710	try depTokenizer(
711	\\C:\Users\anon\foo.o:
712	\\C:\Users\anon\foo\ .o:
713	\\C:\Users\anon\foo\#.o:
714	\\C:\Users\anon\foo$$.o:
715	\\C:\Users\anon\\\ foo.o:
716	\\C:\Users\anon\\#foo.o:
717	\\C:\Users\anon\$$foo.o:
718	\\C:\Users\anon\\\ \ \ \ \ foo.o:
719	,
720	\\target = {C:\Users\anon\foo.o}
721	\\target = {C:\Users\anon\foo .o}
722	\\target = {C:\Users\anon\foo#.o}
723	\\target = {C:\Users\anon\foo$.o}
724	\\target = {C:\Users\anon\ foo.o}
725	\\target = {C:\Users\anon\#foo.o}
726	\\target = {C:\Users\anon\$foo.o}
727	\\target = {C:\Users\anon\ foo.o}
728	);
729	}
730
731	test "windows drive and forward slashes" {
732	try depTokenizer(
733	\\C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj: \
734	\\ C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c
735	,
736	\\target = {C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj}
737	\\prereq = {C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c}
738	);
739	}
740
741	test "error incomplete escape - reverse_solidus" {
742	try depTokenizer("\\",
743	\\ERROR: illegal char '\' at position 0: incomplete escape
744	);
745	try depTokenizer("\t\\",
746	\\ERROR: illegal char '\' at position 1: incomplete escape
747	);
748	try depTokenizer("\n\\",
749	\\ERROR: illegal char '\' at position 1: incomplete escape
750	);
751	try depTokenizer("\r\\",
752	\\ERROR: illegal char '\' at position 1: incomplete escape
753	);
754	try depTokenizer("\r\n\\",
755	\\ERROR: illegal char '\' at position 2: incomplete escape
756	);
757	try depTokenizer(" \\",
758	\\ERROR: illegal char '\' at position 1: incomplete escape
759	);
760	}
761
762	test "error incomplete escape - dollar_sign" {
763	try depTokenizer("$",
764	\\ERROR: illegal char '$' at position 0: incomplete escape
765	);
766	try depTokenizer("\t$",
767	\\ERROR: illegal char '$' at position 1: incomplete escape
768	);
769	try depTokenizer("\n$",
770	\\ERROR: illegal char '$' at position 1: incomplete escape
771	);
772	try depTokenizer("\r$",
773	\\ERROR: illegal char '$' at position 1: incomplete escape
774	);
775	try depTokenizer("\r\n$",
776	\\ERROR: illegal char '$' at position 2: incomplete escape
777	);
778	try depTokenizer(" $",
779	\\ERROR: illegal char '$' at position 1: incomplete escape
780	);
781	}
782
783	test "error incomplete target" {
784	try depTokenizer("foo.o",
785	\\ERROR: incomplete target 'foo.o' at position 0
786	);
787	try depTokenizer("\tfoo.o",
788	\\ERROR: incomplete target 'foo.o' at position 1
789	);
790	try depTokenizer("\nfoo.o",
791	\\ERROR: incomplete target 'foo.o' at position 1
792	);
793	try depTokenizer("\rfoo.o",
794	\\ERROR: incomplete target 'foo.o' at position 1
795	);
796	try depTokenizer("\r\nfoo.o",
797	\\ERROR: incomplete target 'foo.o' at position 2
798	);
799	try depTokenizer(" foo.o",
800	\\ERROR: incomplete target 'foo.o' at position 1
801	);
802
803	try depTokenizer("\\ foo.o",
804	\\ERROR: incomplete target ' foo.o' at position 0
805	);
806	try depTokenizer("\\#foo.o",
807	\\ERROR: incomplete target '#foo.o' at position 0
808	);
809	try depTokenizer("\\\\foo.o",
810	\\ERROR: incomplete target '\foo.o' at position 0
811	);
812	try depTokenizer("$$foo.o",
813	\\ERROR: incomplete target '$foo.o' at position 0
814	);
815	}
816
817	test "error illegal char at position - bad target escape" {
818	try depTokenizer("\\\t",
819	\\ERROR: illegal char \x09 at position 1: bad target escape
820	);
821	try depTokenizer("\\\n",
822	\\ERROR: illegal char \x0A at position 1: bad target escape
823	);
824	try depTokenizer("\\\r",
825	\\ERROR: illegal char \x0D at position 1: bad target escape
826	);
827	try depTokenizer("\\\r\n",
828	\\ERROR: illegal char \x0D at position 1: bad target escape
829	);
830	}
831
832	test "error illegal char at position - execting dollar_sign" {
833	try depTokenizer("$\t",
834	\\ERROR: illegal char \x09 at position 1: expecting '$'
835	);
836	try depTokenizer("$\n",
837	\\ERROR: illegal char \x0A at position 1: expecting '$'
838	);
839	try depTokenizer("$\r",
840	\\ERROR: illegal char \x0D at position 1: expecting '$'
841	);
842	try depTokenizer("$\r\n",
843	\\ERROR: illegal char \x0D at position 1: expecting '$'
844	);
845	}
846
847	test "error illegal char at position - invalid target" {
848	try depTokenizer("foo\t.o",
849	\\ERROR: illegal char \x09 at position 3: invalid target
850	);
851	try depTokenizer("foo\n.o",
852	\\ERROR: illegal char \x0A at position 3: invalid target
853	);
854	try depTokenizer("foo\r.o",
855	\\ERROR: illegal char \x0D at position 3: invalid target
856	);
857	try depTokenizer("foo\r\n.o",
858	\\ERROR: illegal char \x0D at position 3: invalid target
859	);
860	}
861
862	test "error target - continuation expecting end-of-line" {
863	try depTokenizer("foo.o: \\\t",
864	\\target = {foo.o}
865	\\ERROR: illegal char \x09 at position 8: continuation expecting end-of-line
866	);
867	try depTokenizer("foo.o: \\ ",
868	\\target = {foo.o}
869	\\ERROR: illegal char ' ' at position 8: continuation expecting end-of-line
870	);
871	try depTokenizer("foo.o: \\x",
872	\\target = {foo.o}
873	\\ERROR: illegal char 'x' at position 8: continuation expecting end-of-line
874	);
875	try depTokenizer("foo.o: \\\x0dx",
876	\\target = {foo.o}
877	\\ERROR: illegal char 'x' at position 9: continuation expecting end-of-line
878	);
879	}
880
881	test "error prereq - continuation expecting end-of-line" {
882	try depTokenizer("foo.o: foo.h\\\x0dx",
883	\\target = {foo.o}
884	\\ERROR: illegal char 'x' at position 14: continuation expecting end-of-line
885	);
886	}
887
888	// - tokenize input, emit textual representation, and compare to expect
889	fn depTokenizer(input: []const u8, expect: []const u8) !void {
890	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
891	const arena = arena_allocator.allocator();
892	defer arena_allocator.deinit();
893
894	var it: Tokenizer = .{ .bytes = input };
895	var buffer = std.ArrayList(u8).init(arena);
896	var resolve_buf = std.ArrayList(u8).init(arena);
897	var i: usize = 0;
898	while (it.next()) \|token\| {
899	if (i != 0) try buffer.appendSlice("\n");
900	switch (token) {
901	.target, .prereq => \|bytes\| {
902	try buffer.appendSlice(@tagName(token));
903	try buffer.appendSlice(" = {");
904	for (bytes) \|b\| {
905	try buffer.append(printable_char_tab[b]);
906	}
907	try buffer.appendSlice("}");
908	},
909	.target_must_resolve => {
910	try buffer.appendSlice("target = {");
911	try token.resolve(resolve_buf.writer());
912	for (resolve_buf.items) \|b\| {
913	try buffer.append(printable_char_tab[b]);
914	}
915	resolve_buf.items.len = 0;
916	try buffer.appendSlice("}");
917	},
918	else => {
919	try buffer.appendSlice("ERROR: ");
920	try token.printError(buffer.writer());
921	break;
922	},
923	}
924	i += 1;
925	}
926
927	if (std.mem.eql(u8, expect, buffer.items)) {
928	try testing.expect(true);
929	return;
930	}
931
932	const out = std.io.getStdErr().writer();
933
934	try out.writeAll("\n");
935	try printSection(out, "<<<< input", input);
936	try printSection(out, "==== expect", expect);
937	try printSection(out, ">>>> got", buffer.items);
938	try printRuler(out);
939
940	try testing.expect(false);
941	}
942
943	fn printSection(out: anytype, label: []const u8, bytes: []const u8) !void {
944	try printLabel(out, label, bytes);
945	try hexDump(out, bytes);
946	try printRuler(out);
947	try out.writeAll(bytes);
948	try out.writeAll("\n");
949	}
950
951	fn printLabel(out: anytype, label: []const u8, bytes: []const u8) !void {
952	var buf: [80]u8 = undefined;
953	var text = try std.fmt.bufPrint(buf[0..], "{s} {d} bytes ", .{ label, bytes.len });
954	try out.writeAll(text);
955	var i: usize = text.len;
956	const end = 79;
957	while (i < end) : (i += 1) {
958	try out.writeAll(&[_]u8{label[0]});
959	}
960	try out.writeAll("\n");
961	}
962
963	fn printRuler(out: anytype) !void {
964	var i: usize = 0;
965	const end = 79;
966	while (i < end) : (i += 1) {
967	try out.writeAll("-");
968	}
969	try out.writeAll("\n");
970	}
971
972	fn hexDump(out: anytype, bytes: []const u8) !void {
973	const n16 = bytes.len >> 4;
974	var line: usize = 0;
975	var offset: usize = 0;
976	while (line < n16) : (line += 1) {
977	try hexDump16(out, offset, bytes[offset .. offset + 16]);
978	offset += 16;
979	}
980
981	const n = bytes.len & 0x0f;
982	if (n > 0) {
983	try printDecValue(out, offset, 8);
984	try out.writeAll(":");
985	try out.writeAll(" ");
986	var end1 = std.math.min(offset + n, offset + 8);
987	for (bytes[offset..end1]) \|b\| {
988	try out.writeAll(" ");
989	try printHexValue(out, b, 2);
990	}
991	var end2 = offset + n;
992	if (end2 > end1) {
993	try out.writeAll(" ");
994	for (bytes[end1..end2]) \|b\| {
995	try out.writeAll(" ");
996	try printHexValue(out, b, 2);
997	}
998	}
999	const short = 16 - n;
1000	var i: usize = 0;
1001	while (i < short) : (i += 1) {
1002	try out.writeAll(" ");
1003	}
1004	if (end2 > end1) {
1005	try out.writeAll(" \|");
1006	} else {
1007	try out.writeAll(" \|");
1008	}
1009	try printCharValues(out, bytes[offset..end2]);
1010	try out.writeAll("\|\n");
1011	offset += n;
1012	}
1013
1014	try printDecValue(out, offset, 8);
1015	try out.writeAll(":");
1016	try out.writeAll("\n");
1017	}
1018
1019	fn hexDump16(out: anytype, offset: usize, bytes: []const u8) !void {
1020	try printDecValue(out, offset, 8);
1021	try out.writeAll(":");
1022	try out.writeAll(" ");
1023	for (bytes[0..8]) \|b\| {
1024	try out.writeAll(" ");
1025	try printHexValue(out, b, 2);
1026	}
1027	try out.writeAll(" ");
1028	for (bytes[8..16]) \|b\| {
1029	try out.writeAll(" ");
1030	try printHexValue(out, b, 2);
1031	}
1032	try out.writeAll(" \|");
1033	try printCharValues(out, bytes);
1034	try out.writeAll("\|\n");
1035	}
1036
1037	fn printDecValue(out: anytype, value: u64, width: u8) !void {
1038	var buffer: [20]u8 = undefined;
1039	const len = std.fmt.formatIntBuf(buffer[0..], value, 10, .lower, .{ .width = width, .fill = '0' });
1040	try out.writeAll(buffer[0..len]);
1041	}
1042
1043	fn printHexValue(out: anytype, value: u64, width: u8) !void {
1044	var buffer: [16]u8 = undefined;
1045	const len = std.fmt.formatIntBuf(buffer[0..], value, 16, .lower, .{ .width = width, .fill = '0' });
1046	try out.writeAll(buffer[0..len]);
1047	}
1048
1049	fn printCharValues(out: anytype, bytes: []const u8) !void {
1050	for (bytes) \|b\| {
1051	try out.writeAll(&[_]u8{printable_char_tab[b]});
1052	}
1053	}
1054
1055	fn printUnderstandableChar(out: anytype, char: u8) !void {
1056	if (std.ascii.isPrint(char)) {
1057	try out.print("'{c}'", .{char});
1058	} else {
1059	try out.print("\\x{X:0>2}", .{char});
1060	}
1061	}
1062
1063	// zig fmt: off
1064	const printable_char_tab: [256]u8 = (
1065	"................................ !\"#$%&'()*+,-./0123456789:;<=>?" ++
1066	"@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{\|}~." ++
1067	"................................................................" ++
1068	"................................................................"
1069	).*;

src/Module.zig+1-1

...	@@ -16,7 +16,7 @@ const Ast = std.zig.Ast;	...	@@ -16,7 +16,7 @@ const Ast = std.zig.Ast;
16		16
17	const Module = @This();	17	const Module = @This();
18	const Compilation = @import("Compilation.zig");	18	const Compilation = @import("Compilation.zig");
19	const Cache = @import("Cache.zig");	19	const Cache = std.Build.Cache;
20	const Value = @import("value.zig").Value;	20	const Value = @import("value.zig").Value;
21	const Type = @import("type.zig").Type;	21	const Type = @import("type.zig").Type;
22	const TypedValue = @import("TypedValue.zig");	22	const TypedValue = @import("TypedValue.zig");

src/Package.zig+1-1

...	@@ -13,7 +13,7 @@ const Compilation = @import("Compilation.zig");	...	@@ -13,7 +13,7 @@ const Compilation = @import("Compilation.zig");
13	const Module = @import("Module.zig");	13	const Module = @import("Module.zig");
14	const ThreadPool = @import("ThreadPool.zig");	14	const ThreadPool = @import("ThreadPool.zig");
15	const WaitGroup = @import("WaitGroup.zig");	15	const WaitGroup = @import("WaitGroup.zig");
16	const Cache = @import("Cache.zig");	16	const Cache = std.Build.Cache;
17	const build_options = @import("build_options");	17	const build_options = @import("build_options");
18	const Manifest = @import("Manifest.zig");	18	const Manifest = @import("Manifest.zig");
19		19

src/glibc.zig+1-1

...	@@ -11,7 +11,7 @@ const target_util = @import("target.zig");	...	@@ -11,7 +11,7 @@ const target_util = @import("target.zig");
11	const Compilation = @import("Compilation.zig");	11	const Compilation = @import("Compilation.zig");
12	const build_options = @import("build_options");	12	const build_options = @import("build_options");
13	const trace = @import("tracy.zig").trace;	13	const trace = @import("tracy.zig").trace;
14	const Cache = @import("Cache.zig");	14	const Cache = std.Build.Cache;
15	const Package = @import("Package.zig");	15	const Package = @import("Package.zig");
16		16
17	pub const Lib = struct {	17	pub const Lib = struct {

src/link.zig+1-1

...	@@ -10,7 +10,7 @@ const wasi_libc = @import("wasi_libc.zig");	...	@@ -10,7 +10,7 @@ const wasi_libc = @import("wasi_libc.zig");
10		10
11	const Air = @import("Air.zig");	11	const Air = @import("Air.zig");
12	const Allocator = std.mem.Allocator;	12	const Allocator = std.mem.Allocator;
13	const Cache = @import("Cache.zig");	13	const Cache = std.Build.Cache;
14	const Compilation = @import("Compilation.zig");	14	const Compilation = @import("Compilation.zig");
15	const LibCInstallation = @import("libc_installation.zig").LibCInstallation;	15	const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
16	const Liveness = @import("Liveness.zig");	16	const Liveness = @import("Liveness.zig");

src/link/Coff/lld.zig+1-1

...	@@ -5,6 +5,7 @@ const assert = std.debug.assert;	...	@@ -5,6 +5,7 @@ const assert = std.debug.assert;
5	const fs = std.fs;	5	const fs = std.fs;
6	const log = std.log.scoped(.link);	6	const log = std.log.scoped(.link);
7	const mem = std.mem;	7	const mem = std.mem;
		8	const Cache = std.Build.Cache;
8		9
9	const mingw = @import("../../mingw.zig");	10	const mingw = @import("../../mingw.zig");
10	const link = @import("../../link.zig");	11	const link = @import("../../link.zig");
...	@@ -13,7 +14,6 @@ const trace = @import("../../tracy.zig").trace;	...	@@ -13,7 +14,6 @@ const trace = @import("../../tracy.zig").trace;
13		14
14	const Allocator = mem.Allocator;	15	const Allocator = mem.Allocator;
15		16
16	const Cache = @import("../../Cache.zig");
17	const Coff = @import("../Coff.zig");	17	const Coff = @import("../Coff.zig");
18	const Compilation = @import("../../Compilation.zig");	18	const Compilation = @import("../../Compilation.zig");
19		19

src/link/Elf.zig+1-1

...	@@ -21,7 +21,7 @@ const trace = @import("../tracy.zig").trace;	...	@@ -21,7 +21,7 @@ const trace = @import("../tracy.zig").trace;
21	const Air = @import("../Air.zig");	21	const Air = @import("../Air.zig");
22	const Allocator = std.mem.Allocator;	22	const Allocator = std.mem.Allocator;
23	pub const Atom = @import("Elf/Atom.zig");	23	pub const Atom = @import("Elf/Atom.zig");
24	const Cache = @import("../Cache.zig");	24	const Cache = std.Build.Cache;
25	const Compilation = @import("../Compilation.zig");	25	const Compilation = @import("../Compilation.zig");
26	const Dwarf = @import("Dwarf.zig");	26	const Dwarf = @import("Dwarf.zig");
27	const File = link.File;	27	const File = link.File;

src/link/MachO.zig+1-1

...	@@ -28,7 +28,7 @@ const Air = @import("../Air.zig");	...	@@ -28,7 +28,7 @@ const Air = @import("../Air.zig");
28	const Allocator = mem.Allocator;	28	const Allocator = mem.Allocator;
29	const Archive = @import("MachO/Archive.zig");	29	const Archive = @import("MachO/Archive.zig");
30	pub const Atom = @import("MachO/Atom.zig");	30	pub const Atom = @import("MachO/Atom.zig");
31	const Cache = @import("../Cache.zig");	31	const Cache = std.Build.Cache;
32	const CodeSignature = @import("MachO/CodeSignature.zig");	32	const CodeSignature = @import("MachO/CodeSignature.zig");
33	const Compilation = @import("../Compilation.zig");	33	const Compilation = @import("../Compilation.zig");
34	const Dwarf = File.Dwarf;	34	const Dwarf = File.Dwarf;

src/link/MachO/zld.zig+1-1

...	@@ -20,7 +20,7 @@ const trace = @import("../../tracy.zig").trace;	...	@@ -20,7 +20,7 @@ const trace = @import("../../tracy.zig").trace;
20	const Allocator = mem.Allocator;	20	const Allocator = mem.Allocator;
21	const Archive = @import("Archive.zig");	21	const Archive = @import("Archive.zig");
22	const Atom = @import("ZldAtom.zig");	22	const Atom = @import("ZldAtom.zig");
23	const Cache = @import("../../Cache.zig");	23	const Cache = std.Build.Cache;
24	const CodeSignature = @import("CodeSignature.zig");	24	const CodeSignature = @import("CodeSignature.zig");
25	const Compilation = @import("../../Compilation.zig");	25	const Compilation = @import("../../Compilation.zig");
26	const DwarfInfo = @import("DwarfInfo.zig");	26	const DwarfInfo = @import("DwarfInfo.zig");

src/link/Wasm.zig+1-1

...	@@ -20,7 +20,7 @@ const lldMain = @import("../main.zig").lldMain;	...	@@ -20,7 +20,7 @@ const lldMain = @import("../main.zig").lldMain;
20	const trace = @import("../tracy.zig").trace;	20	const trace = @import("../tracy.zig").trace;
21	const build_options = @import("build_options");	21	const build_options = @import("build_options");
22	const wasi_libc = @import("../wasi_libc.zig");	22	const wasi_libc = @import("../wasi_libc.zig");
23	const Cache = @import("../Cache.zig");	23	const Cache = std.Build.Cache;
24	const Type = @import("../type.zig").Type;	24	const Type = @import("../type.zig").Type;
25	const TypedValue = @import("../TypedValue.zig");	25	const TypedValue = @import("../TypedValue.zig");
26	const LlvmObject = @import("../codegen/llvm.zig").Object;	26	const LlvmObject = @import("../codegen/llvm.zig").Object;

src/main.zig+2-2

...	@@ -20,7 +20,7 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation;	...	@@ -20,7 +20,7 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
20	const wasi_libc = @import("wasi_libc.zig");	20	const wasi_libc = @import("wasi_libc.zig");
21	const translate_c = @import("translate_c.zig");	21	const translate_c = @import("translate_c.zig");
22	const clang = @import("clang.zig");	22	const clang = @import("clang.zig");
23	const Cache = @import("Cache.zig");	23	const Cache = std.Build.Cache;
24	const target_util = @import("target.zig");	24	const target_util = @import("target.zig");
25	const ThreadPool = @import("ThreadPool.zig");	25	const ThreadPool = @import("ThreadPool.zig");
26	const crash_report = @import("crash_report.zig");	26	const crash_report = @import("crash_report.zig");
...	@@ -3615,7 +3615,7 @@ fn cmdTranslateC(comp: *Compilation, arena: Allocator, enable_cache: bool) !void	...	@@ -3615,7 +3615,7 @@ fn cmdTranslateC(comp: *Compilation, arena: Allocator, enable_cache: bool) !void
3615	defer if (enable_cache) man.deinit();	3615	defer if (enable_cache) man.deinit();
3616		3616
3617	man.hash.add(@as(u16, 0xb945)); // Random number to distinguish translate-c from compiling C objects	3617	man.hash.add(@as(u16, 0xb945)); // Random number to distinguish translate-c from compiling C objects
3618	man.hashCSource(c_source_file) catch \|err\| {	3618	Compilation.cache_helpers.hashCSource(&man, c_source_file) catch \|err\| {
3619	fatal("unable to process '{s}': {s}", .{ c_source_file.src_path, @errorName(err) });	3619	fatal("unable to process '{s}': {s}", .{ c_source_file.src_path, @errorName(err) });
3620	};	3620	};
3621		3621

src/mingw.zig+1-1

...	@@ -8,7 +8,7 @@ const log = std.log.scoped(.mingw);	...	@@ -8,7 +8,7 @@ const log = std.log.scoped(.mingw);
8	const builtin = @import("builtin");	8	const builtin = @import("builtin");
9	const Compilation = @import("Compilation.zig");	9	const Compilation = @import("Compilation.zig");
10	const build_options = @import("build_options");	10	const build_options = @import("build_options");
11	const Cache = @import("Cache.zig");	11	const Cache = std.Build.Cache;
12		12
13	pub const CRTFile = enum {	13	pub const CRTFile = enum {
14	crt2_o,	14	crt2_o,

test/tests.zig+2-2

...	@@ -570,7 +570,7 @@ pub fn addCliTests(b: *std.Build, test_filter: ?[]const u8, optimize_modes: []co	...	@@ -570,7 +570,7 @@ pub fn addCliTests(b: *std.Build, test_filter: ?[]const u8, optimize_modes: []co
570	const run_cmd = exe.run();	570	const run_cmd = exe.run();
571	run_cmd.addArgs(&[_][]const u8{	571	run_cmd.addArgs(&[_][]const u8{
572	fs.realpathAlloc(b.allocator, b.zig_exe) catch unreachable,	572	fs.realpathAlloc(b.allocator, b.zig_exe) catch unreachable,
573	b.pathFromRoot(b.cache_root),	573	b.pathFromRoot(b.cache_root.path orelse "."),
574	});	574	});
575		575
576	step.dependOn(&run_cmd.step);	576	step.dependOn(&run_cmd.step);
...	@@ -1059,7 +1059,7 @@ pub const StandaloneContext = struct {	...	@@ -1059,7 +1059,7 @@ pub const StandaloneContext = struct {
1059	}	1059	}
1060		1060
1061	var zig_args = ArrayList([]const u8).init(b.allocator);	1061	var zig_args = ArrayList([]const u8).init(b.allocator);
1062	const rel_zig_exe = fs.path.relative(b.allocator, b.build_root, b.zig_exe) catch unreachable;	1062	const rel_zig_exe = fs.path.relative(b.allocator, b.build_root.path orelse ".", b.zig_exe) catch unreachable;
1063	zig_args.append(rel_zig_exe) catch unreachable;	1063	zig_args.append(rel_zig_exe) catch unreachable;
1064	zig_args.append("build") catch unreachable;	1064	zig_args.append("build") catch unreachable;
1065		1065

author	Andrew Kelley <andrew@ziglang.org> 2023-02-13 17:23:19-05:00
committer	GitHub <noreply@github.com> 2023-02-13 17:23:19-05:00
log	a9e1cf3049d1d90d7e62950c3b7f4b088081caa8
tree	22fe9f9f10ab0b6088124949068e6aeb83162a57
parent	fc48467a97021cb872ff2a947f96e882274c39c1
parent	35bb823131a93d7407e79897d4cd20ae2a98ca54
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #14571 from ziglang/more-build-zig