authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-10 15:09:46-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-12 00:14:08-07:00
loga966eee090d55c7d61484333af675c80115bf188
tree9df97f8551ce59a4f163d4bd23b145c5bcb44858
parentf2856403c6997ff1317c968abed0871df9586c7c

std.Build.Step.WriteFile: fix handling of directories

and add file system watching integration. `addDirectoryWatchInput` now returns a `bool` which helps remind the caller to 1. call addDirectoryWatchInputFromPath on any derived paths 2. but only if the dependency is not already captured by a step dependency edge. The make function now recursively walks all directories and adds the found files to the cache hash rather than incorrectly only adding the directory name to the cache hash. closes #20571

3 files changed, 95 insertions(+), 43 deletions(-)

lib/std/Build/Step.zig+12-2
...@@ -634,7 +634,11 @@ pub fn addWatchInput(step: *Step, lazy_file: Build.LazyPath) Allocator.Error!voi...@@ -634,7 +634,11 @@ pub fn addWatchInput(step: *Step, lazy_file: Build.LazyPath) Allocator.Error!voi
634/// Any changes inside the directory will trigger invalidation.634/// Any changes inside the directory will trigger invalidation.
635///635///
636/// See also `addDirectoryWatchInputFromPath` which takes a `Build.Cache.Path` instead.636/// See also `addDirectoryWatchInputFromPath` which takes a `Build.Cache.Path` instead.
637pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Allocator.Error!void {637///
638/// Paths derived from this directory should also be manually added via
639/// `addDirectoryWatchInputFromPath` if and only if this function returns
640/// `true`.
641pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Allocator.Error!bool {
638 switch (lazy_directory) {642 switch (lazy_directory) {
639 .src_path => |src_path| try addDirectoryWatchInputFromBuilder(step, src_path.owner, src_path.sub_path),643 .src_path => |src_path| try addDirectoryWatchInputFromBuilder(step, src_path.owner, src_path.sub_path),
640 .dependency => |d| try addDirectoryWatchInputFromBuilder(step, d.dependency.builder, d.sub_path),644 .dependency => |d| try addDirectoryWatchInputFromBuilder(step, d.dependency.builder, d.sub_path),
...@@ -648,13 +652,19 @@ pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Alloc...@@ -648,13 +652,19 @@ pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Alloc
648 });652 });
649 },653 },
650 // Nothing to watch because this dependency edge is modeled instead via `dependants`.654 // Nothing to watch because this dependency edge is modeled instead via `dependants`.
651 .generated => {},655 .generated => return false,
652 }656 }
657 return true;
653}658}
654659
655/// Any changes inside the directory will trigger invalidation.660/// Any changes inside the directory will trigger invalidation.
656///661///
657/// See also `addDirectoryWatchInput` which takes a `Build.LazyPath` instead.662/// See also `addDirectoryWatchInput` which takes a `Build.LazyPath` instead.
663///
664/// This function should only be called when it has been verified that the
665/// dependency on `path` is not already accounted for by a `Step` dependency.
666/// In other words, before calling this function, first check that the
667/// `Build.LazyPath` which this `path` is derived from is not `generated`.
658pub fn addDirectoryWatchInputFromPath(step: *Step, path: Build.Cache.Path) !void {668pub fn addDirectoryWatchInputFromPath(step: *Step, path: Build.Cache.Path) !void {
659 return addWatchInputFromPath(step, path, ".");669 return addWatchInputFromPath(step, path, ".");
660}670}
lib/std/Build/Step/InstallDir.zig+3-4
...@@ -63,8 +63,8 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {...@@ -63,8 +63,8 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
63 const arena = b.allocator;63 const arena = b.allocator;
64 const dest_prefix = b.getInstallPath(install_dir.options.install_dir, install_dir.options.install_subdir);64 const dest_prefix = b.getInstallPath(install_dir.options.install_dir, install_dir.options.install_subdir);
65 const src_dir_path = install_dir.options.source_dir.getPath3(b, step);65 const src_dir_path = install_dir.options.source_dir.getPath3(b, step);
66 try step.addDirectoryWatchInput(install_dir.options.source_dir);66 const need_derived_inputs = try step.addDirectoryWatchInput(install_dir.options.source_dir);
67 var src_dir = src_dir_path.root_dir.handle.openDir(src_dir_path.subPathOpt() orelse ".", .{ .iterate = true }) catch |err| {67 var src_dir = src_dir_path.root_dir.handle.openDir(src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| {
68 return step.fail("unable to open source directory '{}': {s}", .{68 return step.fail("unable to open source directory '{}': {s}", .{
69 src_dir_path, @errorName(err),69 src_dir_path, @errorName(err),
70 });70 });
...@@ -96,8 +96,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {...@@ -96,8 +96,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
9696
97 switch (entry.kind) {97 switch (entry.kind) {
98 .directory => {98 .directory => {
99 const subdir_path = try src_dir_path.join(arena, entry.path);99 if (need_derived_inputs) try step.addDirectoryWatchInputFromPath(src_sub_path);
100 try step.addDirectoryWatchInputFromPath(subdir_path);
101 try cwd.makePath(dest_path);100 try cwd.makePath(dest_path);
102 // TODO: set result_cached=false if the directory did not already exist.101 // TODO: set result_cached=false if the directory did not already exist.
103 },102 },
lib/std/Build/Step/WriteFile.zig+80-37
...@@ -40,6 +40,22 @@ pub const Directory = struct {...@@ -40,6 +40,22 @@ pub const Directory = struct {
40 .include_extensions = if (opts.include_extensions) |incs| b.dupeStrings(incs) else null,40 .include_extensions = if (opts.include_extensions) |incs| b.dupeStrings(incs) else null,
41 };41 };
42 }42 }
43
44 pub fn pathIncluded(opts: Options, path: []const u8) bool {
45 for (opts.exclude_extensions) |ext| {
46 if (std.mem.endsWith(u8, path, ext))
47 return false;
48 }
49 if (opts.include_extensions) |incs| {
50 for (incs) |inc| {
51 if (std.mem.endsWith(u8, path, inc))
52 return true;
53 } else {
54 return false;
55 }
56 }
57 return true;
58 }
43 };59 };
44};60};
4561
...@@ -158,7 +174,10 @@ fn maybeUpdateName(write_file: *WriteFile) void {...@@ -158,7 +174,10 @@ fn maybeUpdateName(write_file: *WriteFile) void {
158fn make(step: *Step, prog_node: std.Progress.Node) !void {174fn make(step: *Step, prog_node: std.Progress.Node) !void {
159 _ = prog_node;175 _ = prog_node;
160 const b = step.owner;176 const b = step.owner;
177 const arena = b.allocator;
178 const gpa = arena;
161 const write_file: *WriteFile = @fieldParentPtr("step", step);179 const write_file: *WriteFile = @fieldParentPtr("step", step);
180 step.clearWatchInputs();
162181
163 // The cache is used here not really as a way to speed things up - because writing182 // The cache is used here not really as a way to speed things up - because writing
164 // the data to a file would probably be very fast - but as a way to find a canonical183 // the data to a file would probably be very fast - but as a way to find a canonical
...@@ -173,29 +192,67 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {...@@ -173,29 +192,67 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
173 // Random bytes to make WriteFile unique. Refresh this with192 // Random bytes to make WriteFile unique. Refresh this with
174 // new random bytes when WriteFile implementation is modified193 // new random bytes when WriteFile implementation is modified
175 // in a non-backwards-compatible way.194 // in a non-backwards-compatible way.
176 man.hash.add(@as(u32, 0xd767ee59));195 man.hash.add(@as(u32, 0xc2a287d0));
177196
178 for (write_file.files.items) |file| {197 for (write_file.files.items) |file| {
179 man.hash.addBytes(file.sub_path);198 man.hash.addBytes(file.sub_path);
199
180 switch (file.contents) {200 switch (file.contents) {
181 .bytes => |bytes| {201 .bytes => |bytes| {
182 man.hash.addBytes(bytes);202 man.hash.addBytes(bytes);
183 },203 },
184 .copy => |file_source| {204 .copy => |lazy_path| {
185 _ = try man.addFile(file_source.getPath2(b, step), null);205 const path = lazy_path.getPath3(b, step);
206 _ = try man.addFilePath(path, null);
207 try step.addWatchInput(lazy_path);
186 },208 },
187 }209 }
188 }210 }
189 for (write_file.directories.items) |dir| {211
190 man.hash.addBytes(dir.source.getPath2(b, step));212 const open_dir_cache = try arena.alloc(fs.Dir, write_file.directories.items.len);
213 var open_dirs_count: usize = 0;
214 defer closeDirs(open_dir_cache[0..open_dirs_count]);
215
216 for (write_file.directories.items, open_dir_cache) |dir, *open_dir_cache_elem| {
191 man.hash.addBytes(dir.sub_path);217 man.hash.addBytes(dir.sub_path);
192 for (dir.options.exclude_extensions) |ext| man.hash.addBytes(ext);218 for (dir.options.exclude_extensions) |ext| man.hash.addBytes(ext);
193 if (dir.options.include_extensions) |incs| for (incs) |inc| man.hash.addBytes(inc);219 if (dir.options.include_extensions) |incs| for (incs) |inc| man.hash.addBytes(inc);
220
221 const need_derived_inputs = try step.addDirectoryWatchInput(dir.source);
222 const src_dir_path = dir.source.getPath3(b, step);
223
224 var src_dir = src_dir_path.root_dir.handle.openDir(src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| {
225 return step.fail("unable to open source directory '{}': {s}", .{
226 src_dir_path, @errorName(err),
227 });
228 };
229 open_dir_cache_elem.* = src_dir;
230 open_dirs_count += 1;
231
232 var it = try src_dir.walk(gpa);
233 defer it.deinit();
234 while (try it.next()) |entry| {
235 if (!dir.options.pathIncluded(entry.path)) continue;
236
237 switch (entry.kind) {
238 .directory => {
239 if (need_derived_inputs) {
240 const entry_path = try src_dir_path.join(arena, entry.path);
241 try step.addDirectoryWatchInputFromPath(entry_path);
242 }
243 },
244 .file => {
245 const entry_path = try src_dir_path.join(arena, entry.path);
246 _ = try man.addFilePath(entry_path, null);
247 },
248 else => continue,
249 }
250 }
194 }251 }
195252
196 if (try step.cacheHit(&man)) {253 if (try step.cacheHit(&man)) {
197 const digest = man.final();254 const digest = man.final();
198 write_file.generated_directory.path = try b.cache_root.join(b.allocator, &.{ "o", &digest });255 write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest });
199 step.result_cached = true;256 step.result_cached = true;
200 return;257 return;
201 }258 }
...@@ -203,7 +260,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {...@@ -203,7 +260,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
203 const digest = man.final();260 const digest = man.final();
204 const cache_path = "o" ++ fs.path.sep_str ++ digest;261 const cache_path = "o" ++ fs.path.sep_str ++ digest;
205262
206 write_file.generated_directory.path = try b.cache_root.join(b.allocator, &.{ "o", &digest });263 write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest });
207264
208 var cache_dir = b.cache_root.handle.makeOpenPath(cache_path, .{}) catch |err| {265 var cache_dir = b.cache_root.handle.makeOpenPath(cache_path, .{}) catch |err| {
209 return step.fail("unable to make path '{}{s}': {s}", .{266 return step.fail("unable to make path '{}{s}': {s}", .{
...@@ -256,8 +313,9 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {...@@ -256,8 +313,9 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
256 },313 },
257 }314 }
258 }315 }
259 for (write_file.directories.items) |dir| {316
260 const full_src_dir_path = dir.source.getPath2(b, step);317 for (write_file.directories.items, open_dir_cache) |dir, already_open_dir| {
318 const src_dir_path = dir.source.getPath3(b, step);
261 const dest_dirname = dir.sub_path;319 const dest_dirname = dir.sub_path;
262320
263 if (dest_dirname.len != 0) {321 if (dest_dirname.len != 0) {
...@@ -268,44 +326,25 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {...@@ -268,44 +326,25 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
268 };326 };
269 }327 }
270328
271 var src_dir = b.build_root.handle.openDir(full_src_dir_path, .{ .iterate = true }) catch |err| {329 var it = try already_open_dir.walk(gpa);
272 return step.fail("unable to open source directory '{s}': {s}", .{330 defer it.deinit();
273 full_src_dir_path, @errorName(err),331 while (try it.next()) |entry| {
274 });332 if (!dir.options.pathIncluded(entry.path)) continue;
275 };
276 defer src_dir.close();
277333
278 var it = try src_dir.walk(b.allocator);334 const src_entry_path = try src_dir_path.join(arena, entry.path);
279 next_entry: while (try it.next()) |entry| {
280 for (dir.options.exclude_extensions) |ext| {
281 if (std.mem.endsWith(u8, entry.path, ext)) continue :next_entry;
282 }
283 if (dir.options.include_extensions) |incs| {
284 for (incs) |inc| {
285 if (std.mem.endsWith(u8, entry.path, inc)) break;
286 } else {
287 continue :next_entry;
288 }
289 }
290 const full_src_entry_path = b.pathJoin(&.{ full_src_dir_path, entry.path });
291 const dest_path = b.pathJoin(&.{ dest_dirname, entry.path });335 const dest_path = b.pathJoin(&.{ dest_dirname, entry.path });
292 switch (entry.kind) {336 switch (entry.kind) {
293 .directory => try cache_dir.makePath(dest_path),337 .directory => try cache_dir.makePath(dest_path),
294 .file => {338 .file => {
295 const prev_status = fs.Dir.updateFile(339 const prev_status = fs.Dir.updateFile(
296 cwd,340 src_entry_path.root_dir.handle,
297 full_src_entry_path,341 src_entry_path.sub_path,
298 cache_dir,342 cache_dir,
299 dest_path,343 dest_path,
300 .{},344 .{},
301 ) catch |err| {345 ) catch |err| {
302 return step.fail("unable to update file from '{s}' to '{}{s}{c}{s}': {s}", .{346 return step.fail("unable to update file from '{}' to '{}{s}{c}{s}': {s}", .{
303 full_src_entry_path,347 src_entry_path, b.cache_root, cache_path, fs.path.sep, dest_path, @errorName(err),
304 b.cache_root,
305 cache_path,
306 fs.path.sep,
307 dest_path,
308 @errorName(err),
309 });348 });
310 };349 };
311 _ = prev_status;350 _ = prev_status;
...@@ -317,3 +356,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {...@@ -317,3 +356,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
317356
318 try step.writeManifest(&man);357 try step.writeManifest(&man);
319}358}
359
360fn closeDirs(dirs: []fs.Dir) void {
361 for (dirs) |*d| d.close();
362}