authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2025-10-04 22:41:25-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-10-04 22:41:25-07:00
log0a74d73459a8b30b5c91ee444275752206a2a0ea
tree84bbb672630eea58e2052cd789d2846db4ce3d88
parente6e4792a585d4fb462749d78fa73d1403c97caf0
parent894a99171491c9b054ae93b10ca3dc6fa6d8c125
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #25320 from hkupty/walker2

std.fs.Dir: Add `walkSelectively` to provide more control over directory walking

5 files changed, 241 insertions(+), 80 deletions(-)

build.zig+1-1
......@@ -260,7 +260,7 @@ pub fn build(b: *std.Build) !void {
260260 };
261261 const git_describe = mem.trim(u8, git_describe_untrimmed, " \n\r");
262262
263 switch (mem.count(u8, git_describe, "-")) {
263 switch (mem.countScalar(u8, git_describe, '-')) {
264264 0 => {
265265 // Tagged release version (e.g. 0.10.0).
266266 if (!mem.eql(u8, git_describe, version_string)) {
lib/std/fs/Dir.zig+116-55
......@@ -663,35 +663,17 @@ fn iterateImpl(self: Dir, first_iter_start_value: bool) Iterator {
663663 }
664664}
665665
666pub const Walker = struct {
667 stack: std.ArrayListUnmanaged(StackItem),
666pub const SelectiveWalker = struct {
667 stack: std.ArrayListUnmanaged(Walker.StackItem),
668668 name_buffer: std.ArrayListUnmanaged(u8),
669669 allocator: Allocator,
670670
671 pub const Entry = struct {
672 /// The containing directory. This can be used to operate directly on `basename`
673 /// rather than `path`, avoiding `error.NameTooLong` for deeply nested paths.
674 /// The directory remains open until `next` or `deinit` is called.
675 dir: Dir,
676 basename: [:0]const u8,
677 path: [:0]const u8,
678 kind: Dir.Entry.Kind,
679 };
680
681 const StackItem = struct {
682 iter: Dir.Iterator,
683 dirname_len: usize,
684 };
685
686671 /// After each call to this function, and on deinit(), the memory returned
687672 /// from this function becomes invalid. A copy must be made in order to keep
688673 /// a reference to the path.
689 pub fn next(self: *Walker) !?Walker.Entry {
690 const gpa = self.allocator;
691 while (self.stack.items.len != 0) {
692 // `top` and `containing` become invalid after appending to `self.stack`
693 var top = &self.stack.items[self.stack.items.len - 1];
694 var containing = top;
674 pub fn next(self: *SelectiveWalker) !?Walker.Entry {
675 while (self.stack.items.len > 0) {
676 const top = &self.stack.items[self.stack.items.len - 1];
695677 var dirname_len = top.dirname_len;
696678 if (top.iter.next() catch |err| {
697679 // If we get an error, then we want the user to be able to continue
......@@ -703,36 +685,22 @@ pub const Walker = struct {
703685 item.iter.dir.close();
704686 }
705687 return err;
706 }) |base| {
688 }) |entry| {
707689 self.name_buffer.shrinkRetainingCapacity(dirname_len);
708690 if (self.name_buffer.items.len != 0) {
709 try self.name_buffer.append(gpa, fs.path.sep);
691 try self.name_buffer.append(self.allocator, fs.path.sep);
710692 dirname_len += 1;
711693 }
712 try self.name_buffer.ensureUnusedCapacity(gpa, base.name.len + 1);
713 self.name_buffer.appendSliceAssumeCapacity(base.name);
694 try self.name_buffer.ensureUnusedCapacity(self.allocator, entry.name.len + 1);
695 self.name_buffer.appendSliceAssumeCapacity(entry.name);
714696 self.name_buffer.appendAssumeCapacity(0);
715 if (base.kind == .directory) {
716 var new_dir = top.iter.dir.openDir(base.name, .{ .iterate = true }) catch |err| switch (err) {
717 error.NameTooLong => unreachable, // no path sep in base.name
718 else => |e| return e,
719 };
720 {
721 errdefer new_dir.close();
722 try self.stack.append(gpa, .{
723 .iter = new_dir.iterateAssumeFirstIteration(),
724 .dirname_len = self.name_buffer.items.len - 1,
725 });
726 top = &self.stack.items[self.stack.items.len - 1];
727 containing = &self.stack.items[self.stack.items.len - 2];
728 }
729 }
730 return .{
731 .dir = containing.iter.dir,
697 const walker_entry: Walker.Entry = .{
698 .dir = top.iter.dir,
732699 .basename = self.name_buffer.items[dirname_len .. self.name_buffer.items.len - 1 :0],
733700 .path = self.name_buffer.items[0 .. self.name_buffer.items.len - 1 :0],
734 .kind = base.kind,
701 .kind = entry.kind,
735702 };
703 return walker_entry;
736704 } else {
737705 var item = self.stack.pop().?;
738706 if (self.stack.items.len != 0) {
......@@ -743,20 +711,46 @@ pub const Walker = struct {
743711 return null;
744712 }
745713
746 pub fn deinit(self: *Walker) void {
747 const gpa = self.allocator;
748 // Close any remaining directories except the initial one (which is always at index 0)
749 if (self.stack.items.len > 1) {
750 for (self.stack.items[1..]) |*item| {
751 item.iter.dir.close();
714 /// Traverses into the directory, continuing walking one level down.
715 pub fn enter(self: *SelectiveWalker, entry: Walker.Entry) !void {
716 if (entry.kind != .directory) {
717 @branchHint(.cold);
718 return;
719 }
720
721 var new_dir = entry.dir.openDir(entry.basename, .{ .iterate = true }) catch |err| {
722 switch (err) {
723 error.NameTooLong => unreachable,
724 else => |e| return e,
752725 }
726 };
727 errdefer new_dir.close();
728
729 try self.stack.append(self.allocator, .{
730 .iter = new_dir.iterateAssumeFirstIteration(),
731 .dirname_len = self.name_buffer.items.len - 1,
732 });
733 }
734
735 pub fn deinit(self: *SelectiveWalker) void {
736 self.name_buffer.deinit(self.allocator);
737 self.stack.deinit(self.allocator);
738 }
739
740 /// Leaves the current directory, continuing walking one level up.
741 /// If the current entry is a directory entry, then the "current directory"
742 /// will pertain to that entry if `enter` is called before `leave`.
743 pub fn leave(self: *SelectiveWalker) void {
744 var item = self.stack.pop().?;
745 if (self.stack.items.len != 0) {
746 @branchHint(.likely);
747 item.iter.dir.close();
753748 }
754 self.stack.deinit(gpa);
755 self.name_buffer.deinit(gpa);
756749 }
757750};
758751
759/// Recursively iterates over a directory.
752/// Recursively iterates over a directory, but requires the user to
753/// opt-in to recursing into each directory entry.
760754///
761755/// `self` must have been opened with `OpenOptions{.iterate = true}`.
762756///
......@@ -765,7 +759,9 @@ pub const Walker = struct {
765759/// The order of returned file system entries is undefined.
766760///
767761/// `self` will not be closed after walking it.
768pub fn walk(self: Dir, allocator: Allocator) Allocator.Error!Walker {
762///
763/// See also `walk`.
764pub fn walkSelectively(self: Dir, allocator: Allocator) !SelectiveWalker {
769765 var stack: std.ArrayListUnmanaged(Walker.StackItem) = .empty;
770766
771767 try stack.append(allocator, .{
......@@ -780,6 +776,71 @@ pub fn walk(self: Dir, allocator: Allocator) Allocator.Error!Walker {
780776 };
781777}
782778
779pub const Walker = struct {
780 inner: SelectiveWalker,
781
782 pub const Entry = struct {
783 /// The containing directory. This can be used to operate directly on `basename`
784 /// rather than `path`, avoiding `error.NameTooLong` for deeply nested paths.
785 /// The directory remains open until `next` or `deinit` is called.
786 dir: Dir,
787 basename: [:0]const u8,
788 path: [:0]const u8,
789 kind: Dir.Entry.Kind,
790
791 /// Returns the depth of the entry relative to the initial directory.
792 /// Returns 1 for a direct child of the initial directory, 2 for an entry
793 /// within a direct child of the initial directory, etc.
794 pub fn depth(self: Walker.Entry) usize {
795 return mem.countScalar(u8, self.path, fs.path.sep) + 1;
796 }
797 };
798
799 const StackItem = struct {
800 iter: Dir.Iterator,
801 dirname_len: usize,
802 };
803
804 /// After each call to this function, and on deinit(), the memory returned
805 /// from this function becomes invalid. A copy must be made in order to keep
806 /// a reference to the path.
807 pub fn next(self: *Walker) !?Walker.Entry {
808 const entry = try self.inner.next();
809 if (entry != null and entry.?.kind == .directory) {
810 try self.inner.enter(entry.?);
811 }
812 return entry;
813 }
814
815 pub fn deinit(self: *Walker) void {
816 self.inner.deinit();
817 }
818
819 /// Leaves the current directory, continuing walking one level up.
820 /// If the current entry is a directory entry, then the "current directory"
821 /// is the directory pertaining to the current entry.
822 pub fn leave(self: *Walker) void {
823 self.inner.leave();
824 }
825};
826
827/// Recursively iterates over a directory.
828///
829/// `self` must have been opened with `OpenOptions{.iterate = true}`.
830///
831/// `Walker.deinit` releases allocated memory and directory handles.
832///
833/// The order of returned file system entries is undefined.
834///
835/// `self` will not be closed after walking it.
836///
837/// See also `walkSelectively`.
838pub fn walk(self: Dir, allocator: Allocator) Allocator.Error!Walker {
839 return .{
840 .inner = try walkSelectively(self, allocator),
841 };
842}
843
783844pub const OpenError = error{
784845 FileNotFound,
785846 NotDir,
lib/std/fs/test.zig+78-8
......@@ -1765,14 +1765,14 @@ test "walker" {
17651765
17661766 // iteration order of walker is undefined, so need lookup maps to check against
17671767
1768 const expected_paths = std.StaticStringMap(void).initComptime(.{
1769 .{"dir1"},
1770 .{"dir2"},
1771 .{"dir3"},
1772 .{"dir4"},
1773 .{"dir3" ++ fs.path.sep_str ++ "sub1"},
1774 .{"dir3" ++ fs.path.sep_str ++ "sub2"},
1775 .{"dir3" ++ fs.path.sep_str ++ "sub2" ++ fs.path.sep_str ++ "subsub1"},
1768 const expected_paths = std.StaticStringMap(usize).initComptime(.{
1769 .{ "dir1", 1 },
1770 .{ "dir2", 1 },
1771 .{ "dir3", 1 },
1772 .{ "dir4", 1 },
1773 .{ "dir3" ++ fs.path.sep_str ++ "sub1", 2 },
1774 .{ "dir3" ++ fs.path.sep_str ++ "sub2", 2 },
1775 .{ "dir3" ++ fs.path.sep_str ++ "sub2" ++ fs.path.sep_str ++ "subsub1", 3 },
17761776 });
17771777
17781778 const expected_basenames = std.StaticStringMap(void).initComptime(.{
......@@ -1802,6 +1802,76 @@ test "walker" {
18021802 std.debug.print("found unexpected path: {f}\n", .{std.ascii.hexEscape(entry.path, .lower)});
18031803 return err;
18041804 };
1805 testing.expectEqual(expected_paths.get(entry.path).?, entry.depth()) catch |err| {
1806 std.debug.print("path reported unexpected depth: {f}\n", .{std.ascii.hexEscape(entry.path, .lower)});
1807 return err;
1808 };
1809 // make sure that the entry.dir is the containing dir
1810 var entry_dir = try entry.dir.openDir(entry.basename, .{});
1811 defer entry_dir.close();
1812 num_walked += 1;
1813 }
1814 try testing.expectEqual(expected_paths.kvs.len, num_walked);
1815}
1816
1817test "selective walker, skip entries that start with ." {
1818 var tmp = tmpDir(.{ .iterate = true });
1819 defer tmp.cleanup();
1820
1821 const paths_to_create: []const []const u8 = &.{
1822 "dir1/foo/.git/ignored",
1823 ".hidden/bar",
1824 "a/b/c",
1825 "a/baz",
1826 };
1827
1828 // iteration order of walker is undefined, so need lookup maps to check against
1829
1830 const expected_paths = std.StaticStringMap(usize).initComptime(.{
1831 .{ "dir1", 1 },
1832 .{ "dir1" ++ fs.path.sep_str ++ "foo", 2 },
1833 .{ "a", 1 },
1834 .{ "a" ++ fs.path.sep_str ++ "b", 2 },
1835 .{ "a" ++ fs.path.sep_str ++ "b" ++ fs.path.sep_str ++ "c", 3 },
1836 .{ "a" ++ fs.path.sep_str ++ "baz", 2 },
1837 });
1838
1839 const expected_basenames = std.StaticStringMap(void).initComptime(.{
1840 .{"dir1"},
1841 .{"foo"},
1842 .{"a"},
1843 .{"b"},
1844 .{"c"},
1845 .{"baz"},
1846 });
1847
1848 for (paths_to_create) |path| {
1849 try tmp.dir.makePath(path);
1850 }
1851
1852 var walker = try tmp.dir.walkSelectively(testing.allocator);
1853 defer walker.deinit();
1854
1855 var num_walked: usize = 0;
1856 while (try walker.next()) |entry| {
1857 if (entry.basename[0] == '.') continue;
1858 if (entry.kind == .directory) {
1859 try walker.enter(entry);
1860 }
1861
1862 testing.expect(expected_basenames.has(entry.basename)) catch |err| {
1863 std.debug.print("found unexpected basename: {f}\n", .{std.ascii.hexEscape(entry.basename, .lower)});
1864 return err;
1865 };
1866 testing.expect(expected_paths.has(entry.path)) catch |err| {
1867 std.debug.print("found unexpected path: {f}\n", .{std.ascii.hexEscape(entry.path, .lower)});
1868 return err;
1869 };
1870 testing.expectEqual(expected_paths.get(entry.path).?, entry.depth()) catch |err| {
1871 std.debug.print("path reported unexpected depth: {f}\n", .{std.ascii.hexEscape(entry.path, .lower)});
1872 return err;
1873 };
1874
18051875 // make sure that the entry.dir is the containing dir
18061876 var entry_dir = try entry.dir.openDir(entry.basename, .{});
18071877 defer entry_dir.close();
lib/std/mem.zig+20
......@@ -1704,6 +1704,26 @@ test count {
17041704 try testing.expect(count(u8, "owowowu", "owowu") == 1);
17051705}
17061706
1707/// Returns the number of needles inside the haystack
1708pub fn countScalar(comptime T: type, haystack: []const T, needle: T) usize {
1709 var i: usize = 0;
1710 var found: usize = 0;
1711
1712 while (findScalarPos(T, haystack, i, needle)) |idx| {
1713 i = idx + 1;
1714 found += 1;
1715 }
1716
1717 return found;
1718}
1719
1720test countScalar {
1721 try testing.expectEqual(0, countScalar(u8, "", 'h'));
1722 try testing.expectEqual(1, countScalar(u8, "h", 'h'));
1723 try testing.expectEqual(2, countScalar(u8, "hh", 'h'));
1724 try testing.expectEqual(3, countScalar(u8, " abcabc abc", 'b'));
1725}
1726
17071727/// Returns true if the haystack contains expected_count or more needles
17081728/// needle.len must be > 0
17091729/// does not count overlapping needles
tools/update_mingw.zig+26-16
......@@ -109,13 +109,29 @@ pub fn main() !void {
109109
110110 {
111111 // Also add all new def and def.in files.
112 var walker = try src_crt_dir.walk(arena);
112 var walker = try src_crt_dir.walkSelectively(arena);
113113 defer walker.deinit();
114114
115115 var fail = false;
116116
117117 while (try walker.next()) |entry| {
118 if (entry.kind != .file) continue;
118 switch (entry.kind) {
119 .directory => {
120 switch (entry.depth()) {
121 1 => if (def_dirs.has(entry.basename)) {
122 try walker.enter(entry);
123 continue;
124 },
125 else => {
126 // The top-level directory was already validated
127 try walker.enter(entry);
128 continue;
129 },
130 }
131 },
132 .file => {},
133 else => continue,
134 }
119135
120136 const ok_ext = for (def_exts) |ext| {
121137 if (std.mem.endsWith(u8, entry.path, ext)) break true;
......@@ -123,12 +139,6 @@ pub fn main() !void {
123139
124140 if (!ok_ext) continue;
125141
126 const ok_prefix = for (def_dirs) |p| {
127 if (std.mem.startsWith(u8, entry.path, p)) break true;
128 } else false;
129
130 if (!ok_prefix) continue;
131
132142 const blacklisted = for (blacklisted_defs) |item| {
133143 if (std.mem.eql(u8, entry.basename, item)) break true;
134144 } else false;
......@@ -162,14 +172,14 @@ const def_exts = [_][]const u8{
162172 ".def.in",
163173};
164174
165const def_dirs = [_][]const u8{
166 "lib32" ++ std.fs.path.sep_str,
167 "lib64" ++ std.fs.path.sep_str,
168 "libarm32" ++ std.fs.path.sep_str,
169 "libarm64" ++ std.fs.path.sep_str,
170 "lib-common" ++ std.fs.path.sep_str,
171 "def-include" ++ std.fs.path.sep_str,
172};
175const def_dirs = std.StaticStringMap(void).initComptime(.{
176 .{"lib32"},
177 .{"lib64"},
178 .{"libarm32"},
179 .{"libarm64"},
180 .{"lib-common"},
181 .{"def-include"},
182});
173183
174184const blacklisted_defs = [_][]const u8{
175185 "crtdll.def.in",