authorgravatar for alex_naskos@hotmail.comAlexandros Naskos <alex_naskos@hotmail.com> 2020-11-16 19:10:33+02:00
committergravatar for alex_naskos@hotmail.comAlexandros Naskos <alex_naskos@hotmail.com> 2020-12-14 21:00:54+02:00
log5112ab8233449c2061237a178087992cbf74dfea
treeecb18734d25566c42b52ad175c762f673027175d
parent8591f30b0d53a597682bebdfcd570f5f44339b26
signaturelock-open Commit is signed but in an unrecognized format.

Fixed std.fs.Watch implementation on Linux

Added .Deleted event to std.fs.Watch on Linux

1 files changed, 151 insertions(+), 111 deletions(-)

lib/std/fs/watch.zig+151-111
...@@ -25,9 +25,7 @@ const WatchEventId = enum {...@@ -25,9 +25,7 @@ const WatchEventId = enum {
25};25};
2626
27fn eqlString(a: []const u16, b: []const u16) bool {27fn eqlString(a: []const u16, b: []const u16) bool {
28 if (a.len != b.len) return false;28 return mem.eql(u16, a, b);
29 if (a.ptr == b.ptr) return true;
30 return mem.compare(u16, a, b) == .Equal;
31}29}
3230
33fn hashString(s: []const u16) u32 {31fn hashString(s: []const u16) u32 {
...@@ -43,7 +41,7 @@ const WatchEventError = error{...@@ -43,7 +41,7 @@ const WatchEventError = error{
4341
44pub fn Watch(comptime V: type) type {42pub fn Watch(comptime V: type) type {
45 return struct {43 return struct {
46 channel: *event.Channel(Event.Error!Event),44 channel: event.Channel(Event.Error!Event),
47 os_data: OsData,45 os_data: OsData,
48 allocator: *Allocator,46 allocator: *Allocator,
4947
...@@ -110,19 +108,14 @@ pub fn Watch(comptime V: type) type {...@@ -110,19 +108,14 @@ pub fn Watch(comptime V: type) type {
110 pub const Event = struct {108 pub const Event = struct {
111 id: Id,109 id: Id,
112 data: V,110 data: V,
111 dirname: []const u8,
112 basename: []const u8,
113113
114 pub const Id = WatchEventId;114 pub const Id = WatchEventId;
115 pub const Error = WatchEventError;115 pub const Error = WatchEventError;
116 };116 };
117117
118 pub fn init(allocator: *Allocator, event_buf_count: usize) !*Self {118 pub fn init(allocator: *Allocator, event_buf_count: usize) !*Self {
119 const channel = try allocator.create(event.Channel(Event.Error!Event));
120 errdefer allocator.destroy(channel);
121 var buf = try allocator.alloc(Event.Error!Event, event_buf_count);
122 errdefer allocator.free(buf);
123 channel.init(buf);
124 errdefer channel.deinit();
125
126 const self = try allocator.create(Self);119 const self = try allocator.create(Self);
127 errdefer allocator.destroy(self);120 errdefer allocator.destroy(self);
128121
...@@ -133,15 +126,17 @@ pub fn Watch(comptime V: type) type {...@@ -133,15 +126,17 @@ pub fn Watch(comptime V: type) type {
133126
134 self.* = Self{127 self.* = Self{
135 .allocator = allocator,128 .allocator = allocator,
136 .channel = channel,129 .channel = undefined,
137 .os_data = OsData{130 .os_data = OsData{
138 .putter_frame = undefined,131 .putter_frame = undefined,
139 .inotify_fd = inotify_fd,132 .inotify_fd = inotify_fd,
140 .wd_table = OsData.WdTable.init(allocator),133 .wd_table = OsData.WdTable.init(allocator),
141 .table_lock = event.Lock.init(),134 .table_lock = event.Lock{},
142 },135 },
143 };136 };
144137
138 var buf = try allocator.alloc(Event.Error!Event, event_buf_count);
139 self.channel.init(buf);
145 self.os_data.putter_frame = async self.linuxEventPutter();140 self.os_data.putter_frame = async self.linuxEventPutter();
146 return self;141 return self;
147 },142 },
...@@ -149,14 +144,16 @@ pub fn Watch(comptime V: type) type {...@@ -149,14 +144,16 @@ pub fn Watch(comptime V: type) type {
149 .windows => {144 .windows => {
150 self.* = Self{145 self.* = Self{
151 .allocator = allocator,146 .allocator = allocator,
152 .channel = channel,147 .channel = undefined,
153 .os_data = OsData{148 .os_data = OsData{
154 .table_lock = event.Lock.init(),149 .table_lock = event.Lock{},
155 .dir_table = OsData.DirTable.init(allocator),150 .dir_table = OsData.DirTable.init(allocator),
156 .ref_count = std.atomic.Int(usize).init(1),151 .ref_count = std.atomic.Int(usize).init(1),
157 .all_putters = std.atomic.Queue(anyframe).init(),152 .all_putters = std.atomic.Queue(WindowsOsData.Put).init(),
158 },153 },
159 };154 };
155 var buf = try allocator.alloc(Event.Error!Event, event_buf_count);
156 self.channel.init(buf);
160 return self;157 return self;
161 },158 },
162159
...@@ -194,6 +191,17 @@ pub fn Watch(comptime V: type) type {...@@ -194,6 +191,17 @@ pub fn Watch(comptime V: type) type {
194 },191 },
195 .linux => {192 .linux => {
196 self.os_data.cancelled = true;193 self.os_data.cancelled = true;
194 {
195 // Remove all directory watches linuxEventPutter will take care of
196 // cleaning up the memory and closing the inotify fd.
197 var dir_it = self.os_data.wd_table.iterator();
198 while (dir_it.next()) |wd_entry| {
199 const rc = os.linux.inotify_rm_watch(self.os_data.inotify_fd, wd_entry.key);
200 // Errno can only be EBADF, EINVAL if either the inotify fs or the wd are invalid
201 std.debug.assert(rc == 0);
202 }
203 }
204
197 await self.os_data.putter_frame;205 await self.os_data.putter_frame;
198 self.allocator.destroy(self);206 self.allocator.destroy(self);
199 },207 },
...@@ -322,19 +330,12 @@ pub fn Watch(comptime V: type) type {...@@ -322,19 +330,12 @@ pub fn Watch(comptime V: type) type {
322330
323 fn addFileLinux(self: *Self, file_path: []const u8, value: V) !?V {331 fn addFileLinux(self: *Self, file_path: []const u8, value: V) !?V {
324 const dirname = std.fs.path.dirname(file_path) orelse ".";332 const dirname = std.fs.path.dirname(file_path) orelse ".";
325 const dirname_with_null = try std.cstr.addNullByte(self.allocator, dirname);
326 var dirname_with_null_consumed = false;
327 defer if (!dirname_with_null_consumed) self.channel.free(dirname_with_null);
328
329 const basename = std.fs.path.basename(file_path);333 const basename = std.fs.path.basename(file_path);
330 const basename_with_null = try std.cstr.addNullByte(self.allocator, basename);
331 var basename_with_null_consumed = false;
332 defer if (!basename_with_null_consumed) self.allocator.free(basename_with_null);
333334
334 const wd = try os.inotify_add_watchZ(335 const wd = try os.inotify_add_watch(
335 self.os_data.inotify_fd,336 self.os_data.inotify_fd,
336 dirname_with_null.ptr,337 dirname,
337 os.linux.IN_CLOSE_WRITE | os.linux.IN_ONLYDIR | os.linux.IN_EXCL_UNLINK,338 os.linux.IN_CLOSE_WRITE | os.linux.IN_ONLYDIR | os.linux.IN_DELETE | os.linux.IN_EXCL_UNLINK,
338 );339 );
339 // wd is either a newly created watch or an existing one.340 // wd is either a newly created watch or an existing one.
340341
...@@ -343,22 +344,21 @@ pub fn Watch(comptime V: type) type {...@@ -343,22 +344,21 @@ pub fn Watch(comptime V: type) type {
343344
344 const gop = try self.os_data.wd_table.getOrPut(wd);345 const gop = try self.os_data.wd_table.getOrPut(wd);
345 if (!gop.found_existing) {346 if (!gop.found_existing) {
346 gop.kv.value = OsData.Dir{347 gop.entry.value = OsData.Dir{
347 .dirname = dirname_with_null,348 .dirname = try self.allocator.dupe(u8, dirname),
348 .file_table = OsData.FileTable.init(self.allocator),349 .file_table = OsData.FileTable.init(self.allocator),
349 };350 };
350 dirname_with_null_consumed = true;
351 }351 }
352 const dir = &gop.kv.value;
353352
354 const file_table_gop = try dir.file_table.getOrPut(basename_with_null);353 const dir = &gop.entry.value;
354 const file_table_gop = try dir.file_table.getOrPut(basename);
355 if (file_table_gop.found_existing) {355 if (file_table_gop.found_existing) {
356 const prev_value = file_table_gop.kv.value;356 const prev_value = file_table_gop.entry.value;
357 file_table_gop.kv.value = value;357 file_table_gop.entry.value = value;
358 return prev_value;358 return prev_value;
359 } else {359 } else {
360 file_table_gop.kv.value = value;360 file_table_gop.entry.key = try self.allocator.dupe(u8, basename);
361 basename_with_null_consumed = true;361 file_table_gop.entry.value = value;
362 return null;362 return null;
363 }363 }
364 }364 }
...@@ -539,76 +539,96 @@ pub fn Watch(comptime V: type) type {...@@ -539,76 +539,96 @@ pub fn Watch(comptime V: type) type {
539 }539 }
540540
541 pub fn removeFile(self: *Self, file_path: []const u8) ?V {541 pub fn removeFile(self: *Self, file_path: []const u8) ?V {
542 @panic("TODO");542 switch (builtin.os.tag) {
543 .linux => {
544 const dirname = std.fs.path.dirname(file_path) orelse ".";
545 const basename = std.fs.path.basename(file_path);
546
547 const held = self.os_data.table_lock.acquire();
548 defer held.release();
549
550 const dir = self.os_data.wd_table.get(dirname) orelse return null;
551 if (dir.file_table.remove(basename)) |file_entry| {
552 self.allocator.free(file_entry.key);
553 return file_entry.value;
554 }
555 return null;
556 },
557 .macos, .freebsd, .netbsd, .dragonfly, .openbsd => @panic("TODO"),
558 .windows => return @panic("TODO"),
559 else => @compileError("Unsupported OS"),
560 }
543 }561 }
544562
545 fn linuxEventPutter(self: *Self) void {563 fn linuxEventPutter(self: *Self) void {
546 global_event_loop.beginOneEvent();564 global_event_loop.beginOneEvent();
547565
548 defer {566 defer {
549 self.os_data.table_lock.deinit();567 std.debug.assert(self.os_data.wd_table.count() == 0);
550 var wd_it = self.os_data.wd_table.iterator();
551 while (wd_it.next()) |wd_entry| {
552 var file_it = wd_entry.value.file_table.iterator();
553 while (file_it.next()) |file_entry| {
554 self.allocator.free(file_entry.key);
555 }
556 self.allocator.free(wd_entry.value.dirname);
557 wd_entry.value.file_table.deinit();
558 }
559 self.os_data.wd_table.deinit();568 self.os_data.wd_table.deinit();
560 global_event_loop.finishOneEvent();
561 os.close(self.os_data.inotify_fd);569 os.close(self.os_data.inotify_fd);
562 self.channel.deinit();
563 self.allocator.free(self.channel.buffer_nodes);570 self.allocator.free(self.channel.buffer_nodes);
571 self.channel.deinit();
572 global_event_loop.finishOneEvent();
564 }573 }
565574
566 var event_buf: [4096]u8 align(@alignOf(os.linux.inotify_event)) = undefined;575 var event_buf: [4096]u8 align(@alignOf(os.linux.inotify_event)) = undefined;
567576
568 while (!self.os_data.cancelled) {577 while (!self.os_data.cancelled) {
569 const rc = os.linux.read(self.os_data.inotify_fd, &event_buf, event_buf.len);578 const bytes_read = global_event_loop.read(self.os_data.inotify_fd, &event_buf, false) catch unreachable;
570 const errno = os.linux.getErrno(rc);579
571 switch (errno) {580 var ptr: [*]u8 = &event_buf;
572 0 => {581 const end_ptr = ptr + bytes_read;
573 // can't use @bytesToSlice because of the special variable length name field582 while (@ptrToInt(ptr) < @ptrToInt(end_ptr)) {
574 var ptr = event_buf[0..].ptr;583 const ev = @ptrCast(*const os.linux.inotify_event, ptr);
575 const end_ptr = ptr + event_buf.len;584 if (ev.mask & os.linux.IN_CLOSE_WRITE == os.linux.IN_CLOSE_WRITE) {
576 var ev: *os.linux.inotify_event = undefined;585 const basename_ptr = ptr + @sizeOf(os.linux.inotify_event);
577 while (@ptrToInt(ptr) < @ptrToInt(end_ptr)) {586 const basename = std.mem.span(@ptrCast([*:0]u8, basename_ptr));
578 ev = @ptrCast(*os.linux.inotify_event, ptr);587
579 if (ev.mask & os.linux.IN_CLOSE_WRITE == os.linux.IN_CLOSE_WRITE) {588 const held = self.os_data.table_lock.acquire();
580 const basename_ptr = ptr + @sizeOf(os.linux.inotify_event);589 defer held.release();
581 // `ev.len` counts all bytes in `ev.name` including terminating null byte.590
582 const basename_with_null = basename_ptr[0..ev.len];591 const dir = &self.os_data.wd_table.get(ev.wd).?;
583 const user_value = blk: {592 if (dir.file_table.getEntry(basename)) |file_value| {
584 const held = self.os_data.table_lock.acquire();593 self.channel.put(Event{
585 defer held.release();594 .id = .CloseWrite,
586595 .data = file_value.value,
587 const dir = &self.os_data.wd_table.get(ev.wd).?.value;596 .dirname = dir.dirname,
588 if (dir.file_table.get(basename_with_null)) |entry| {597 .basename = file_value.key,
589 break :blk entry.value;598 });
590 } else {599 }
591 break :blk null;600 } else if (ev.mask & os.linux.IN_IGNORED == os.linux.IN_IGNORED) {
592 }601 // Directory watch was removed
593 };602 const held = self.os_data.table_lock.acquire();
594 if (user_value) |v| {603 defer held.release();
595 self.channel.put(Event{604 if (self.os_data.wd_table.remove(ev.wd)) |*wd_entry| {
596 .id = WatchEventId.CloseWrite,605 var file_it = wd_entry.value.file_table.iterator();
597 .data = v,606 while (file_it.next()) |file_entry| {
598 });607 self.allocator.free(file_entry.key);
599 }
600 }608 }
601609 self.allocator.free(wd_entry.value.dirname);
602 ptr = @alignCast(@alignOf(os.linux.inotify_event), ptr + @sizeOf(os.linux.inotify_event) + ev.len);610 wd_entry.value.file_table.deinit();
603 }611 }
604 },612 } else if (ev.mask & os.linux.IN_DELETE == os.linux.IN_DELETE) {
605 os.linux.EINTR => continue,613 // File or directory was removed or deleted
606 os.linux.EINVAL => unreachable,614 const basename_ptr = ptr + @sizeOf(os.linux.inotify_event);
607 os.linux.EFAULT => unreachable,615 const basename = std.mem.span(@ptrCast([*:0]u8, basename_ptr));
608 os.linux.EAGAIN => {616
609 global_event_loop.linuxWaitFd(self.os_data.inotify_fd, os.linux.EPOLLET | os.linux.EPOLLIN | os.EPOLLONESHOT);617 const held = self.os_data.table_lock.acquire();
610 },618 defer held.release();
611 else => unreachable,619 const dir = &self.os_data.wd_table.get(ev.wd).?;
620
621 if (dir.file_table.getEntry(basename)) |file_value| {
622 self.channel.put(Event{
623 .id = .Delete,
624 .data = file_value.value,
625 .dirname = dir.dirname,
626 .basename = file_value.key,
627 });
628 }
629 }
630
631 ptr = @alignCast(@alignOf(os.linux.inotify_event), ptr + @sizeOf(os.linux.inotify_event) + ev.len);
612 }632 }
613 }633 }
614 }634 }
...@@ -617,19 +637,19 @@ pub fn Watch(comptime V: type) type {...@@ -617,19 +637,19 @@ pub fn Watch(comptime V: type) type {
617637
618const test_tmp_dir = "std_event_fs_test";638const test_tmp_dir = "std_event_fs_test";
619639
620test "write a file, watch it, write it again" {640test "write a file, watch it, write it again, delete it" {
621 // TODO re-enable this test641 if (!std.io.is_async) return error.SkipZigTest;
622 if (true) return error.SkipZigTest;642 // TODO https://github.com/ziglang/zig/issues/1908
643 if (builtin.single_threaded) return error.SkipZigTest;
623644
624 try fs.cwd().makePath(test_tmp_dir);645 try std.fs.cwd().makePath(test_tmp_dir);
625 defer fs.cwd().deleteTree(test_tmp_dir) catch {};646 defer std.fs.cwd().deleteTree(test_tmp_dir) catch {};
626647
627 const allocator = std.heap.page_allocator;648 return testWriteWatchWriteDelete(std.testing.allocator);
628 return testFsWatch(&allocator);
629}649}
630650
631fn testFsWatch(allocator: *Allocator) !void {651fn testWriteWatchWriteDelete(allocator: *Allocator) !void {
632 const file_path = try std.fs.path.join(allocator, [_][]const u8{ test_tmp_dir, "file.txt" });652 const file_path = try std.fs.path.join(allocator, &[_][]const u8{ test_tmp_dir, "file.txt" });
633 defer allocator.free(file_path);653 defer allocator.free(file_path);
634654
635 const contents =655 const contents =
...@@ -639,9 +659,10 @@ fn testFsWatch(allocator: *Allocator) !void {...@@ -639,9 +659,10 @@ fn testFsWatch(allocator: *Allocator) !void {
639 const line2_offset = 7;659 const line2_offset = 7;
640660
641 // first just write then read the file661 // first just write then read the file
642 try writeFile(allocator, file_path, contents);662 try std.fs.cwd().writeFile(file_path, contents);
643663
644 const read_contents = try readFile(allocator, file_path, 1024 * 1024);664 const read_contents = try std.fs.cwd().readFileAlloc(allocator, file_path, 1024 * 1024);
665 defer allocator.free(read_contents);
645 testing.expectEqualSlices(u8, contents, read_contents);666 testing.expectEqualSlices(u8, contents, read_contents);
646667
647 // now watch the file668 // now watch the file
...@@ -650,28 +671,47 @@ fn testFsWatch(allocator: *Allocator) !void {...@@ -650,28 +671,47 @@ fn testFsWatch(allocator: *Allocator) !void {
650671
651 testing.expect((try watch.addFile(file_path, {})) == null);672 testing.expect((try watch.addFile(file_path, {})) == null);
652673
653 const ev = watch.channel.get();674 var ev = async watch.channel.get();
654 var ev_consumed = false;675 var ev_consumed = false;
655 defer if (!ev_consumed) await ev;676 defer if (!ev_consumed) {
677 _ = await ev;
678 };
656679
657 // overwrite line 2680 // overwrite line 2
658 const fd = try await openReadWrite(file_path, File.default_mode);681 const file = try std.fs.cwd().openFile(file_path, .{ .read = true, .write = true });
659 {682 {
660 defer os.close(fd);683 defer file.close();
661684 const write_contents = "lorem ipsum";
662 try pwritev(allocator, fd, []const []const u8{"lorem ipsum"}, line2_offset);685 var iovec = [_]os.iovec_const{.{
686 .iov_base = write_contents,
687 .iov_len = write_contents.len,
688 }};
689 _ = try file.pwritevAll(&iovec, line2_offset);
663 }690 }
664691
665 ev_consumed = true;
666 switch ((try await ev).id) {692 switch ((try await ev).id) {
667 WatchEventId.CloseWrite => {},693 .CloseWrite => {
668 WatchEventId.Delete => @panic("wrong event"),694 ev_consumed = true;
695 },
696 .Delete => @panic("wrong event"),
669 }697 }
670 const contents_updated = try readFile(allocator, file_path, 1024 * 1024);698
699 const contents_updated = try std.fs.cwd().readFileAlloc(allocator, file_path, 1024 * 1024);
700 defer allocator.free(contents_updated);
701
671 testing.expectEqualSlices(u8,702 testing.expectEqualSlices(u8,
672 \\line 1703 \\line 1
673 \\lorem ipsum704 \\lorem ipsum
674 , contents_updated);705 , contents_updated);
675706
676 // TODO test deleting the file and then re-adding it. we should get events for both707 ev = async watch.channel.get();
708 ev_consumed = false;
709
710 try std.fs.cwd().deleteFile(file_path);
711 switch ((try await ev).id) {
712 .Delete => {
713 ev_consumed = true;
714 },
715 .CloseWrite => @panic("wrong event"),
716 }
677}717}