authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-10-12 13:37:06-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-10-12 13:37:06-04:00
logc23b3e6fd924725d21ecbbc85ef086d330dd4322
tree92c033450b0e5e952d83c93b54c6183bca34b06a
parent260b0ff7c57f27f059a74329f1ca354c5c881353
parent1468eb12f339b09baa1155d33694272828f9617a
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #13073 from squeek502/fs-delete-tree-2

`fs.Dir.deleteTree`: Optimize for non-deeply-nested directories

2 files changed, 393 insertions(+), 107 deletions(-)

lib/std/fs.zig+357-107
......@@ -301,7 +301,7 @@ pub const IterableDir = struct {
301301 .macos, .ios, .freebsd, .netbsd, .dragonfly, .openbsd, .solaris => struct {
302302 dir: Dir,
303303 seek: i64,
304 buf: [8192]u8, // TODO align(@alignOf(os.system.dirent)),
304 buf: [1024]u8, // TODO align(@alignOf(os.system.dirent)),
305305 index: usize,
306306 end_index: usize,
307307 first_iter: bool,
......@@ -490,10 +490,16 @@ pub const IterableDir = struct {
490490 };
491491 }
492492 }
493
494 pub fn reset(self: *Self) void {
495 self.index = 0;
496 self.end_index = 0;
497 self.first_iter = true;
498 }
493499 },
494500 .haiku => struct {
495501 dir: Dir,
496 buf: [8192]u8, // TODO align(@alignOf(os.dirent64)),
502 buf: [1024]u8, // TODO align(@alignOf(os.dirent64)),
497503 index: usize,
498504 end_index: usize,
499505 first_iter: bool,
......@@ -577,12 +583,18 @@ pub const IterableDir = struct {
577583 };
578584 }
579585 }
586
587 pub fn reset(self: *Self) void {
588 self.index = 0;
589 self.end_index = 0;
590 self.first_iter = true;
591 }
580592 },
581593 .linux => struct {
582594 dir: Dir,
583595 // The if guard is solely there to prevent compile errors from missing `linux.dirent64`
584596 // definition when compiling for other OSes. It doesn't do anything when compiling for Linux.
585 buf: [8192]u8 align(if (builtin.os.tag != .linux) 1 else @alignOf(linux.dirent64)),
597 buf: [1024]u8 align(if (builtin.os.tag != .linux) 1 else @alignOf(linux.dirent64)),
586598 index: usize,
587599 end_index: usize,
588600 first_iter: bool,
......@@ -655,10 +667,16 @@ pub const IterableDir = struct {
655667 };
656668 }
657669 }
670
671 pub fn reset(self: *Self) void {
672 self.index = 0;
673 self.end_index = 0;
674 self.first_iter = true;
675 }
658676 },
659677 .windows => struct {
660678 dir: Dir,
661 buf: [8192]u8 align(@alignOf(os.windows.FILE_BOTH_DIR_INFORMATION)),
679 buf: [1024]u8 align(@alignOf(os.windows.FILE_BOTH_DIR_INFORMATION)),
662680 index: usize,
663681 end_index: usize,
664682 first_iter: bool,
......@@ -727,10 +745,16 @@ pub const IterableDir = struct {
727745 };
728746 }
729747 }
748
749 pub fn reset(self: *Self) void {
750 self.index = 0;
751 self.end_index = 0;
752 self.first_iter = true;
753 }
730754 },
731755 .wasi => struct {
732756 dir: Dir,
733 buf: [8192]u8, // TODO align(@alignOf(os.wasi.dirent_t)),
757 buf: [1024]u8, // TODO align(@alignOf(os.wasi.dirent_t)),
734758 cookie: u64,
735759 index: usize,
736760 end_index: usize,
......@@ -806,11 +830,28 @@ pub const IterableDir = struct {
806830 };
807831 }
808832 }
833
834 pub fn reset(self: *Self) void {
835 self.index = 0;
836 self.end_index = 0;
837 self.cookie = os.wasi.DIRCOOKIE_START;
838 }
809839 },
810840 else => @compileError("unimplemented"),
811841 };
812842
813843 pub fn iterate(self: IterableDir) Iterator {
844 return self.iterateImpl(true);
845 }
846
847 /// Like `iterate`, but will not reset the directory cursor before the first
848 /// iteration. This should only be used in cases where it is known that the
849 /// `IterableDir` has not had its cursor modified yet (e.g. it was just opened).
850 pub fn iterateAssumeFirstIteration(self: IterableDir) Iterator {
851 return self.iterateImpl(false);
852 }
853
854 fn iterateImpl(self: IterableDir, first_iter_start_value: bool) Iterator {
814855 switch (builtin.os.tag) {
815856 .macos,
816857 .ios,
......@@ -825,20 +866,20 @@ pub const IterableDir = struct {
825866 .index = 0,
826867 .end_index = 0,
827868 .buf = undefined,
828 .first_iter = true,
869 .first_iter = first_iter_start_value,
829870 },
830871 .linux, .haiku => return Iterator{
831872 .dir = self.dir,
832873 .index = 0,
833874 .end_index = 0,
834875 .buf = undefined,
835 .first_iter = true,
876 .first_iter = first_iter_start_value,
836877 },
837878 .windows => return Iterator{
838879 .dir = self.dir,
839880 .index = 0,
840881 .end_index = 0,
841 .first_iter = true,
882 .first_iter = first_iter_start_value,
842883 .buf = undefined,
843884 .name_data = undefined,
844885 },
......@@ -2035,55 +2076,197 @@ pub const Dir = struct {
20352076 /// this function recursively removes its entries and then tries again.
20362077 /// This operation is not atomic on most file systems.
20372078 pub fn deleteTree(self: Dir, sub_path: []const u8) DeleteTreeError!void {
2038 start_over: while (true) {
2039 var got_access_denied = false;
2040
2041 // First, try deleting the item as a file. This way we don't follow sym links.
2042 if (self.deleteFile(sub_path)) {
2043 return;
2044 } else |err| switch (err) {
2045 error.FileNotFound => return,
2046 error.IsDir => {},
2047 error.AccessDenied => got_access_denied = true,
2048
2049 error.InvalidUtf8,
2050 error.SymLinkLoop,
2051 error.NameTooLong,
2052 error.SystemResources,
2053 error.ReadOnlyFileSystem,
2054 error.NotDir,
2055 error.FileSystem,
2056 error.FileBusy,
2057 error.BadPathName,
2058 error.Unexpected,
2059 => |e| return e,
2079 var initial_iterable_dir = (try self.deleteTreeOpenInitialSubpath(sub_path, .File)) orelse return;
2080
2081 const StackItem = struct {
2082 name: []const u8,
2083 parent_dir: Dir,
2084 iter: IterableDir.Iterator,
2085 };
2086
2087 var stack = std.BoundedArray(StackItem, 16){};
2088 defer {
2089 for (stack.slice()) |*item| {
2090 item.iter.dir.close();
20602091 }
2061 var iterable_dir = self.openIterableDir(sub_path, .{ .no_follow = true }) catch |err| switch (err) {
2062 error.NotDir => {
2063 if (got_access_denied) {
2064 return error.AccessDenied;
2092 }
2093
2094 stack.appendAssumeCapacity(StackItem{
2095 .name = sub_path,
2096 .parent_dir = self,
2097 .iter = initial_iterable_dir.iterateAssumeFirstIteration(),
2098 });
2099
2100 process_stack: while (stack.len != 0) {
2101 var top = &(stack.slice()[stack.len - 1]);
2102 while (try top.iter.next()) |entry| {
2103 var treat_as_dir = entry.kind == .Directory;
2104 handle_entry: while (true) {
2105 if (treat_as_dir) {
2106 if (stack.ensureUnusedCapacity(1)) {
2107 var iterable_dir = top.iter.dir.openIterableDir(entry.name, .{ .no_follow = true }) catch |err| switch (err) {
2108 error.NotDir => {
2109 treat_as_dir = false;
2110 continue :handle_entry;
2111 },
2112 error.FileNotFound => {
2113 // That's fine, we were trying to remove this directory anyway.
2114 break :handle_entry;
2115 },
2116
2117 error.InvalidHandle,
2118 error.AccessDenied,
2119 error.SymLinkLoop,
2120 error.ProcessFdQuotaExceeded,
2121 error.NameTooLong,
2122 error.SystemFdQuotaExceeded,
2123 error.NoDevice,
2124 error.SystemResources,
2125 error.Unexpected,
2126 error.InvalidUtf8,
2127 error.BadPathName,
2128 error.DeviceBusy,
2129 => |e| return e,
2130 };
2131 stack.appendAssumeCapacity(StackItem{
2132 .name = entry.name,
2133 .parent_dir = top.iter.dir,
2134 .iter = iterable_dir.iterateAssumeFirstIteration(),
2135 });
2136 continue :process_stack;
2137 } else |_| {
2138 try top.iter.dir.deleteTreeMinStackSizeWithKindHint(entry.name, entry.kind);
2139 break :handle_entry;
2140 }
2141 } else {
2142 if (top.iter.dir.deleteFile(entry.name)) {
2143 break :handle_entry;
2144 } else |err| switch (err) {
2145 error.FileNotFound => break :handle_entry,
2146
2147 // Impossible because we do not pass any path separators.
2148 error.NotDir => unreachable,
2149
2150 error.IsDir => {
2151 treat_as_dir = true;
2152 continue :handle_entry;
2153 },
2154
2155 error.AccessDenied,
2156 error.InvalidUtf8,
2157 error.SymLinkLoop,
2158 error.NameTooLong,
2159 error.SystemResources,
2160 error.ReadOnlyFileSystem,
2161 error.FileSystem,
2162 error.FileBusy,
2163 error.BadPathName,
2164 error.Unexpected,
2165 => |e| return e,
2166 }
20652167 }
2066 continue :start_over;
2067 },
2068 error.FileNotFound => {
2069 // That's fine, we were trying to remove this directory anyway.
2070 continue :start_over;
2071 },
2168 }
2169 }
20722170
2073 error.InvalidHandle,
2074 error.AccessDenied,
2075 error.SymLinkLoop,
2076 error.ProcessFdQuotaExceeded,
2077 error.NameTooLong,
2078 error.SystemFdQuotaExceeded,
2079 error.NoDevice,
2080 error.SystemResources,
2081 error.Unexpected,
2082 error.InvalidUtf8,
2083 error.BadPathName,
2084 error.DeviceBusy,
2085 => |e| return e,
2171 // On Windows, we can't delete until the dir's handle has been closed, so
2172 // close it before we try to delete.
2173 top.iter.dir.close();
2174
2175 // In order to avoid double-closing the directory when cleaning up
2176 // the stack in the case of an error, we save the relevant portions and
2177 // pop the value from the stack.
2178 const parent_dir = top.parent_dir;
2179 const name = top.name;
2180 _ = stack.pop();
2181
2182 var need_to_retry: bool = false;
2183 parent_dir.deleteDir(name) catch |err| switch (err) {
2184 error.FileNotFound => {},
2185 error.DirNotEmpty => need_to_retry = false,
2186 else => |e| return e,
20862187 };
2188
2189 if (need_to_retry) {
2190 // Since we closed the handle that the previous iterator used, we
2191 // need to re-open the dir and re-create the iterator.
2192 var iterable_dir = iterable_dir: {
2193 var treat_as_dir = true;
2194 handle_entry: while (true) {
2195 if (treat_as_dir) {
2196 break :iterable_dir parent_dir.openIterableDir(name, .{ .no_follow = true }) catch |err| switch (err) {
2197 error.NotDir => {
2198 treat_as_dir = false;
2199 continue :handle_entry;
2200 },
2201 error.FileNotFound => {
2202 // That's fine, we were trying to remove this directory anyway.
2203 continue :process_stack;
2204 },
2205
2206 error.InvalidHandle,
2207 error.AccessDenied,
2208 error.SymLinkLoop,
2209 error.ProcessFdQuotaExceeded,
2210 error.NameTooLong,
2211 error.SystemFdQuotaExceeded,
2212 error.NoDevice,
2213 error.SystemResources,
2214 error.Unexpected,
2215 error.InvalidUtf8,
2216 error.BadPathName,
2217 error.DeviceBusy,
2218 => |e| return e,
2219 };
2220 } else {
2221 if (parent_dir.deleteFile(name)) {
2222 continue :process_stack;
2223 } else |err| switch (err) {
2224 error.FileNotFound => continue :process_stack,
2225
2226 // Impossible because we do not pass any path separators.
2227 error.NotDir => unreachable,
2228
2229 error.IsDir => {
2230 treat_as_dir = true;
2231 continue :handle_entry;
2232 },
2233
2234 error.AccessDenied,
2235 error.InvalidUtf8,
2236 error.SymLinkLoop,
2237 error.NameTooLong,
2238 error.SystemResources,
2239 error.ReadOnlyFileSystem,
2240 error.FileSystem,
2241 error.FileBusy,
2242 error.BadPathName,
2243 error.Unexpected,
2244 => |e| return e,
2245 }
2246 }
2247 }
2248 };
2249 // We know there is room on the stack since we are just re-adding
2250 // the StackItem that we previously popped.
2251 stack.appendAssumeCapacity(StackItem{
2252 .name = name,
2253 .parent_dir = parent_dir,
2254 .iter = iterable_dir.iterateAssumeFirstIteration(),
2255 });
2256 continue :process_stack;
2257 }
2258 }
2259 }
2260
2261 /// Like `deleteTree`, but only keeps one `Iterator` active at a time to minimize the function's stack size.
2262 /// This is slower than `deleteTree` but uses less stack space.
2263 pub fn deleteTreeMinStackSize(self: Dir, sub_path: []const u8) DeleteTreeError!void {
2264 return self.deleteTreeMinStackWithKindHint(sub_path, .File);
2265 }
2266
2267 fn deleteTreeMinStackSizeWithKindHint(self: Dir, sub_path: []const u8, kind_hint: File.Kind) DeleteTreeError!void {
2268 start_over: while (true) {
2269 var iterable_dir = (try self.deleteTreeOpenInitialSubpath(sub_path, kind_hint)) orelse return;
20872270 var cleanup_dir_parent: ?IterableDir = null;
20882271 defer if (cleanup_dir_parent) |*d| d.close();
20892272
......@@ -2101,41 +2284,110 @@ pub const Dir = struct {
21012284 // open it, and close the original directory. Repeat. Then start the entire operation over.
21022285
21032286 scan_dir: while (true) {
2104 var dir_it = iterable_dir.iterate();
2105 while (try dir_it.next()) |entry| {
2106 if (iterable_dir.dir.deleteFile(entry.name)) {
2107 continue;
2108 } else |err| switch (err) {
2109 error.FileNotFound => continue,
2110
2111 // Impossible because we do not pass any path separators.
2112 error.NotDir => unreachable,
2287 var dir_it = iterable_dir.iterateAssumeFirstIteration();
2288 dir_it: while (try dir_it.next()) |entry| {
2289 var treat_as_dir = entry.kind == .Directory;
2290 handle_entry: while (true) {
2291 if (treat_as_dir) {
2292 const new_dir = iterable_dir.dir.openIterableDir(entry.name, .{ .no_follow = true }) catch |err| switch (err) {
2293 error.NotDir => {
2294 treat_as_dir = false;
2295 continue :handle_entry;
2296 },
2297 error.FileNotFound => {
2298 // That's fine, we were trying to remove this directory anyway.
2299 continue :dir_it;
2300 },
2301
2302 error.InvalidHandle,
2303 error.AccessDenied,
2304 error.SymLinkLoop,
2305 error.ProcessFdQuotaExceeded,
2306 error.NameTooLong,
2307 error.SystemFdQuotaExceeded,
2308 error.NoDevice,
2309 error.SystemResources,
2310 error.Unexpected,
2311 error.InvalidUtf8,
2312 error.BadPathName,
2313 error.DeviceBusy,
2314 => |e| return e,
2315 };
2316 if (cleanup_dir_parent) |*d| d.close();
2317 cleanup_dir_parent = iterable_dir;
2318 iterable_dir = new_dir;
2319 mem.copy(u8, &dir_name_buf, entry.name);
2320 dir_name = dir_name_buf[0..entry.name.len];
2321 continue :scan_dir;
2322 } else {
2323 if (iterable_dir.dir.deleteFile(entry.name)) {
2324 continue :dir_it;
2325 } else |err| switch (err) {
2326 error.FileNotFound => continue :dir_it,
2327
2328 // Impossible because we do not pass any path separators.
2329 error.NotDir => unreachable,
2330
2331 error.IsDir => {
2332 treat_as_dir = true;
2333 continue :handle_entry;
2334 },
2335
2336 error.AccessDenied,
2337 error.InvalidUtf8,
2338 error.SymLinkLoop,
2339 error.NameTooLong,
2340 error.SystemResources,
2341 error.ReadOnlyFileSystem,
2342 error.FileSystem,
2343 error.FileBusy,
2344 error.BadPathName,
2345 error.Unexpected,
2346 => |e| return e,
2347 }
2348 }
2349 }
2350 }
2351 // Reached the end of the directory entries, which means we successfully deleted all of them.
2352 // Now to remove the directory itself.
2353 iterable_dir.close();
2354 cleanup_dir = false;
21132355
2114 error.IsDir => {},
2115 error.AccessDenied => got_access_denied = true,
2356 if (cleanup_dir_parent) |d| {
2357 d.dir.deleteDir(dir_name) catch |err| switch (err) {
2358 // These two things can happen due to file system race conditions.
2359 error.FileNotFound, error.DirNotEmpty => continue :start_over,
2360 else => |e| return e,
2361 };
2362 continue :start_over;
2363 } else {
2364 self.deleteDir(sub_path) catch |err| switch (err) {
2365 error.FileNotFound => return,
2366 error.DirNotEmpty => continue :start_over,
2367 else => |e| return e,
2368 };
2369 return;
2370 }
2371 }
2372 }
2373 }
21162374
2117 error.InvalidUtf8,
2118 error.SymLinkLoop,
2119 error.NameTooLong,
2120 error.SystemResources,
2121 error.ReadOnlyFileSystem,
2122 error.FileSystem,
2123 error.FileBusy,
2124 error.BadPathName,
2125 error.Unexpected,
2126 => |e| return e,
2127 }
2375 /// On successful delete, returns null.
2376 fn deleteTreeOpenInitialSubpath(self: Dir, sub_path: []const u8, kind_hint: File.Kind) !?IterableDir {
2377 return iterable_dir: {
2378 // Treat as a file by default
2379 var treat_as_dir = kind_hint == .Directory;
21282380
2129 const new_dir = iterable_dir.dir.openIterableDir(entry.name, .{ .no_follow = true }) catch |err| switch (err) {
2381 handle_entry: while (true) {
2382 if (treat_as_dir) {
2383 break :iterable_dir self.openIterableDir(sub_path, .{ .no_follow = true }) catch |err| switch (err) {
21302384 error.NotDir => {
2131 if (got_access_denied) {
2132 return error.AccessDenied;
2133 }
2134 continue :scan_dir;
2385 treat_as_dir = false;
2386 continue :handle_entry;
21352387 },
21362388 error.FileNotFound => {
21372389 // That's fine, we were trying to remove this directory anyway.
2138 continue :scan_dir;
2390 return null;
21392391 },
21402392
21412393 error.InvalidHandle,
......@@ -2152,35 +2404,33 @@ pub const Dir = struct {
21522404 error.DeviceBusy,
21532405 => |e| return e,
21542406 };
2155 if (cleanup_dir_parent) |*d| d.close();
2156 cleanup_dir_parent = iterable_dir;
2157 iterable_dir = new_dir;
2158 mem.copy(u8, &dir_name_buf, entry.name);
2159 dir_name = dir_name_buf[0..entry.name.len];
2160 continue :scan_dir;
2161 }
2162 // Reached the end of the directory entries, which means we successfully deleted all of them.
2163 // Now to remove the directory itself.
2164 iterable_dir.close();
2165 cleanup_dir = false;
2166
2167 if (cleanup_dir_parent) |d| {
2168 d.dir.deleteDir(dir_name) catch |err| switch (err) {
2169 // These two things can happen due to file system race conditions.
2170 error.FileNotFound, error.DirNotEmpty => continue :start_over,
2171 else => |e| return e,
2172 };
2173 continue :start_over;
21742407 } else {
2175 self.deleteDir(sub_path) catch |err| switch (err) {
2176 error.FileNotFound => return,
2177 error.DirNotEmpty => continue :start_over,
2178 else => |e| return e,
2179 };
2180 return;
2408 if (self.deleteFile(sub_path)) {
2409 return null;
2410 } else |err| switch (err) {
2411 error.FileNotFound => return null,
2412
2413 error.IsDir => {
2414 treat_as_dir = true;
2415 continue :handle_entry;
2416 },
2417
2418 error.AccessDenied,
2419 error.InvalidUtf8,
2420 error.SymLinkLoop,
2421 error.NameTooLong,
2422 error.SystemResources,
2423 error.ReadOnlyFileSystem,
2424 error.NotDir,
2425 error.FileSystem,
2426 error.FileBusy,
2427 error.BadPathName,
2428 error.Unexpected,
2429 => |e| return e,
2430 }
21812431 }
21822432 }
2183 }
2433 };
21842434 }
21852435
21862436 /// Writes content to the file system, creating a new file if it does not exist, truncating
lib/std/fs/test.zig+36
......@@ -219,6 +219,42 @@ test "Dir.Iterator twice" {
219219 }
220220}
221221
222test "Dir.Iterator reset" {
223 var tmp_dir = tmpIterableDir(.{});
224 defer tmp_dir.cleanup();
225
226 // First, create a couple of entries to iterate over.
227 const file = try tmp_dir.iterable_dir.dir.createFile("some_file", .{});
228 file.close();
229
230 try tmp_dir.iterable_dir.dir.makeDir("some_dir");
231
232 var arena = ArenaAllocator.init(testing.allocator);
233 defer arena.deinit();
234 const allocator = arena.allocator();
235
236 // Create iterator.
237 var iter = tmp_dir.iterable_dir.iterate();
238
239 var i: u8 = 0;
240 while (i < 2) : (i += 1) {
241 var entries = std.ArrayList(IterableDir.Entry).init(allocator);
242
243 while (try iter.next()) |entry| {
244 // We cannot just store `entry` as on Windows, we're re-using the name buffer
245 // which means we'll actually share the `name` pointer between entries!
246 const name = try allocator.dupe(u8, entry.name);
247 try entries.append(.{ .name = name, .kind = entry.kind });
248 }
249
250 try testing.expect(entries.items.len == 2); // note that the Iterator skips '.' and '..'
251 try testing.expect(contains(&entries, .{ .name = "some_file", .kind = .File }));
252 try testing.expect(contains(&entries, .{ .name = "some_dir", .kind = .Directory }));
253
254 iter.reset();
255 }
256}
257
222258test "Dir.Iterator but dir is deleted during iteration" {
223259 var tmp = std.testing.tmpDir(.{});
224260 defer tmp.cleanup();