authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-08 20:03:40-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-08 20:03:40-07:00
logcd4397783f7ae86de867bed3346ea10547a94ee1
tree4ce6a807c43bacfabee5312517ca6f35b118ad2d
parent4a2cf388440f6efadf9c7664b63c3949b25d6824

Package.Fetch: improved deletion algorithm

Instead of every file deletion being followed by a recursive attempt to remove the parent directory, while walking the directory, every directory that will have nonzero files deleted from it is tracked in an array hash map. After all the threads have finished deleting and hashing, the parent thread sorts the "suspicious" directories by length, descending, ensuring that children appear before parents, and then iterates over the array hash map, attempting a rmdir operation on each. Any rmdir that succeeds appends the parent directory to the map so that it will be removed if empty.

1 files changed, 47 insertions(+), 16 deletions(-)

src/Package/Fetch.zig+47-16
......@@ -1249,8 +1249,13 @@ fn computeHash(
12491249 var all_files = std.ArrayList(*HashedFile).init(gpa);
12501250 defer all_files.deinit();
12511251
1252 var all_deletions = std.ArrayList(*DeletedFile).init(gpa);
1253 defer all_deletions.deinit();
1252 var deleted_files = std.ArrayList(*DeletedFile).init(gpa);
1253 defer deleted_files.deinit();
1254
1255 // Track directories which had any files deleted from them so that empty directories
1256 // can be deleted.
1257 var sus_dirs: std.StringArrayHashMapUnmanaged(void) = .{};
1258 defer sus_dirs.deinit(gpa);
12541259
12551260 var walker = try @as(fs.IterableDir, .{ .dir = tmp_directory.handle }).walk(gpa);
12561261 defer walker.deinit();
......@@ -1274,16 +1279,22 @@ fn computeHash(
12741279
12751280 if (!filter.includePath(entry.path)) {
12761281 // Delete instead of including in hash calculation.
1282 const fs_path = try arena.dupe(u8, entry.path);
1283
1284 // Also track the parent directory in case it becomes empty.
1285 if (fs.path.dirname(fs_path)) |parent|
1286 try sus_dirs.put(gpa, parent, {});
1287
12771288 const deleted_file = try arena.create(DeletedFile);
12781289 deleted_file.* = .{
1279 .fs_path = try arena.dupe(u8, entry.path),
1290 .fs_path = fs_path,
12801291 .failure = undefined, // to be populated by the worker
12811292 };
12821293 wait_group.start();
12831294 try thread_pool.spawn(workerDeleteFile, .{
12841295 tmp_directory.handle, deleted_file, &wait_group,
12851296 });
1286 try all_deletions.append(deleted_file);
1297 try deleted_files.append(deleted_file);
12871298 continue;
12881299 }
12891300
......@@ -1300,8 +1311,8 @@ fn computeHash(
13001311 if (std.mem.eql(u8, entry.path, Package.build_zig_basename))
13011312 f.has_build_zig = true;
13021313
1303 const hashed_file = try arena.create(HashedFile);
13041314 const fs_path = try arena.dupe(u8, entry.path);
1315 const hashed_file = try arena.create(HashedFile);
13051316 hashed_file.* = .{
13061317 .fs_path = fs_path,
13071318 .normalized_path = try normalizePath(arena, fs_path),
......@@ -1317,6 +1328,36 @@ fn computeHash(
13171328 }
13181329 }
13191330
1331 {
1332 // Sort by length, descending, so that child directories get removed first.
1333 sus_dirs.sortUnstable(@as(struct {
1334 keys: []const []const u8,
1335 pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
1336 return ctx.keys[b_index].len < ctx.keys[a_index].len;
1337 }
1338 }, .{ .keys = sus_dirs.keys() }));
1339
1340 // During this loop, more entries will be added, so we must loop by index.
1341 var i: usize = 0;
1342 while (i < sus_dirs.count()) : (i += 1) {
1343 const sus_dir = sus_dirs.keys()[i];
1344 tmp_directory.handle.deleteDir(sus_dir) catch |err| switch (err) {
1345 error.DirNotEmpty => continue,
1346 error.FileNotFound => continue,
1347 else => |e| {
1348 try eb.addRootErrorMessage(.{ .msg = try eb.printString(
1349 "unable to delete empty directory '{s}': {s}",
1350 .{ sus_dir, @errorName(e) },
1351 ) });
1352 return error.FetchFailed;
1353 },
1354 };
1355 if (fs.path.dirname(sus_dir)) |parent| {
1356 try sus_dirs.put(gpa, parent, {});
1357 }
1358 }
1359 }
1360
13201361 std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);
13211362
13221363 var hasher = Manifest.Hash.init(.{});
......@@ -1332,7 +1373,7 @@ fn computeHash(
13321373 };
13331374 hasher.update(&hashed_file.hash);
13341375 }
1335 for (all_deletions.items) |deleted_file| {
1376 for (deleted_files.items) |deleted_file| {
13361377 deleted_file.failure catch |err| {
13371378 any_failures = true;
13381379 try eb.addRootErrorMessage(.{
......@@ -1382,16 +1423,6 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
13821423
13831424fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {
13841425 try dir.deleteFile(deleted_file.fs_path);
1385 // In case the file was the last remaining file in the parent directory, attempt to
1386 // remove the parent directory.
1387 var opt_parent = fs.path.dirname(deleted_file.fs_path);
1388 while (opt_parent) |parent| : (opt_parent = fs.path.dirname(parent)) {
1389 dir.deleteDir(parent) catch |err| switch (err) {
1390 error.DirNotEmpty => return,
1391 error.FileNotFound => return,
1392 else => |e| return e,
1393 };
1394 }
13951426}
13961427
13971428fn isExecutable(file: fs.File) !bool {