| author |  Takeshi Yoneda <takeshi@tetrate.io> 2022-12-01 13:56:53+09:00 |
| committer | Takeshi Yoneda <takeshi@tetrate.io> 2022-12-01 13:56:53+09:00 |
| log | 829bf5a03e510e707de3c48a78bef2f35fe193a6 |
| tree | 1700f45c5f79ef8c923c437127bd55860557cd0d |
| parent | a943422672a9f456a74c20b4c167107ccbadfb1a |
wasi: fixes IterableDir.nextWasi for large directory
Signed-off-by: Takeshi Yoneda <takeshi@tetrate.io>2 files changed, 45 insertions(+), 1 deletions(-)
lib/std/fs.zig+8-1
| ... | @@ -809,7 +809,9 @@ pub const IterableDir = struct { | ... | @@ -809,7 +809,9 @@ pub const IterableDir = struct { |
| 809 | // and we avoid the code complexity here. | 809 | // and we avoid the code complexity here. |
| 810 | const w = os.wasi; | 810 | const w = os.wasi; |
| 811 | start_over: while (true) { | 811 | start_over: while (true) { |
| 812 | if (self.index >= self.end_index) { | 812 | // According to the WASI spec, the last entry might be truncated, |
| 813 | // so we need to check if the left buffer contains the whole dirent. | ||
| 814 | if (self.end_index - self.index < @sizeOf(w.dirent_t)) { | ||
| 813 | var bufused: usize = undefined; | 815 | var bufused: usize = undefined; |
| 814 | switch (w.fd_readdir(self.dir.fd, &self.buf, self.buf.len, self.cookie, &bufused)) { | 816 | switch (w.fd_readdir(self.dir.fd, &self.buf, self.buf.len, self.cookie, &bufused)) { |
| 815 | .SUCCESS => {}, | 817 | .SUCCESS => {}, |
| ... | @@ -828,6 +830,11 @@ pub const IterableDir = struct { | ... | @@ -828,6 +830,11 @@ pub const IterableDir = struct { |
| 828 | const entry = @ptrCast(*align(1) w.dirent_t, &self.buf[self.index]); | 830 | const entry = @ptrCast(*align(1) w.dirent_t, &self.buf[self.index]); |
| 829 | const entry_size = @sizeOf(w.dirent_t); | 831 | const entry_size = @sizeOf(w.dirent_t); |
| 830 | const name_index = self.index + entry_size; | 832 | const name_index = self.index + entry_size; |
| 833 | if (name_index + entry.d_namlen > self.end_index) { | ||
| 834 | // This case, the name is truncated, so we need to call readdir to store the entire name. | ||
| 835 | self.end_index = self.index; // Force fd_readdir in the next loop. | ||
| 836 | continue :start_over; | ||
| 837 | } | ||
| 831 | const name = mem.span(self.buf[name_index .. name_index + entry.d_namlen]); | 838 | const name = mem.span(self.buf[name_index .. name_index + entry.d_namlen]); |
| 832 | 839 | ||
| 833 | const next_index = name_index + entry.d_namlen; | 840 | const next_index = name_index + entry.d_namlen; |
lib/std/fs/test.zig+37
| ... | @@ -186,6 +186,43 @@ test "Dir.Iterator" { | ... | @@ -186,6 +186,43 @@ test "Dir.Iterator" { |
| 186 | try testing.expect(contains(&entries, .{ .name = "some_dir", .kind = .Directory })); | 186 | try testing.expect(contains(&entries, .{ .name = "some_dir", .kind = .Directory })); |
| 187 | } | 187 | } |
| 188 | 188 | ||
| 189 | test "Dir.Iterator many entries" { | ||
| 190 | if (builtin.os.tag == .wasi and !builtin.link_libc) try os.initPreopensWasi(std.heap.page_allocator, "/"); | ||
| 191 | |||
| 192 | var tmp_dir = tmpIterableDir(.{}); | ||
| 193 | defer tmp_dir.cleanup(); | ||
| 194 | |||
| 195 | const num = 1024; | ||
| 196 | var i: usize = 0; | ||
| 197 | var buf: [4]u8 = undefined; // Enough to store "1024". | ||
| 198 | while (i < num) : (i += 1) { | ||
| 199 | const name = try std.fmt.bufPrint(&buf, "{}", .{i}); | ||
| 200 | const file = try tmp_dir.iterable_dir.dir.createFile(name, .{}); | ||
| 201 | file.close(); | ||
| 202 | } | ||
| 203 | |||
| 204 | var arena = ArenaAllocator.init(testing.allocator); | ||
| 205 | defer arena.deinit(); | ||
| 206 | const allocator = arena.allocator(); | ||
| 207 | |||
| 208 | var entries = std.ArrayList(IterableDir.Entry).init(allocator); | ||
| 209 | |||
| 210 | // Create iterator. | ||
| 211 | var iter = tmp_dir.iterable_dir.iterate(); | ||
| 212 | while (try iter.next()) |entry| { | ||
| 213 | // We cannot just store `entry` as on Windows, we're re-using the name buffer | ||
| 214 | // which means we'll actually share the `name` pointer between entries! | ||
| 215 | const name = try allocator.dupe(u8, entry.name); | ||
| 216 | try entries.append(.{ .name = name, .kind = entry.kind }); | ||
| 217 | } | ||
| 218 | |||
| 219 | i = 0; | ||
| 220 | while (i < num) : (i += 1) { | ||
| 221 | const name = try std.fmt.bufPrint(&buf, "{}", .{i}); | ||
| 222 | try testing.expect(contains(&entries, .{ .name = name, .kind = .File })); | ||
| 223 | } | ||
| 224 | } | ||
| 225 | |||
| 189 | test "Dir.Iterator twice" { | 226 | test "Dir.Iterator twice" { |
| 190 | var tmp_dir = tmpIterableDir(.{}); | 227 | var tmp_dir = tmpIterableDir(.{}); |
| 191 | defer tmp_dir.cleanup(); | 228 | defer tmp_dir.cleanup(); |