authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-10-25 04:00:26-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-10-25 04:00:26-07:00
logb8795b4d02dfdba19aac5ed92bd051ff458665af
tree5e441bd1a237868092d42492de0f1afd51fa4e5b
parent03d0e296cb8a8afb2bd81c030b31e96e6e2940c7
parentcc671a2d400c6a8238f77eb3f0bd9a749714b8e5
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #21795 from ziglang/kqueue-watch

Build Runner: Implement File System Watching for kqueue

2 files changed, 319 insertions(+), 94 deletions(-)

lib/compiler/build_runner.zig+1-4
...@@ -447,10 +447,7 @@ pub fn main() !void {...@@ -447,10 +447,7 @@ pub fn main() !void {
447447
448 if (!watch) return cleanExit();448 if (!watch) return cleanExit();
449449
450 switch (builtin.os.tag) {450 if (!Watch.have_impl) fatal("--watch not yet implemented for {s}", .{@tagName(builtin.os.tag)});
451 .linux, .windows => {},
452 else => fatal("--watch not yet implemented for {s}", .{@tagName(builtin.os.tag)}),
453 }
454451
455 try w.update(gpa, run.step_stack.keys());452 try w.update(gpa, run.step_stack.keys());
456453
lib/std/Build/Watch.zig+318-90
...@@ -10,6 +10,8 @@ dir_table: DirTable,...@@ -10,6 +10,8 @@ dir_table: DirTable,
10os: Os,10os: Os,
11generation: Generation,11generation: Generation,
1212
13pub const have_impl = Os != void;
14
13/// Key is the directory to watch which contains one or more files we are15/// Key is the directory to watch which contains one or more files we are
14/// interested in noticing changes to.16/// interested in noticing changes to.
15///17///
...@@ -89,6 +91,38 @@ const Os = switch (builtin.os.tag) {...@@ -89,6 +91,38 @@ const Os = switch (builtin.os.tag) {
89 };91 };
90 };92 };
9193
94 fn init() !Watch {
95 const fan_fd = std.posix.fanotify_init(.{
96 .CLASS = .NOTIF,
97 .CLOEXEC = true,
98 .NONBLOCK = true,
99 .REPORT_NAME = true,
100 .REPORT_DIR_FID = true,
101 .REPORT_FID = true,
102 .REPORT_TARGET_FID = true,
103 }, 0) catch |err| switch (err) {
104 error.UnsupportedFlags => fatal("fanotify_init failed due to old kernel; requires 5.17+", .{}),
105 else => |e| return e,
106 };
107 return .{
108 .dir_table = .{},
109 .os = switch (builtin.os.tag) {
110 .linux => .{
111 .handle_table = .{},
112 .poll_fds = .{
113 .{
114 .fd = fan_fd,
115 .events = std.posix.POLL.IN,
116 .revents = undefined,
117 },
118 },
119 },
120 else => {},
121 },
122 .generation = 0,
123 };
124 }
125
92 fn getDirHandle(gpa: Allocator, path: std.Build.Cache.Path) !FileHandle {126 fn getDirHandle(gpa: Allocator, path: std.Build.Cache.Path) !FileHandle {
93 var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined;127 var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined;
94 var mount_id: i32 = undefined;128 var mount_id: i32 = undefined;
...@@ -236,6 +270,16 @@ const Os = switch (builtin.os.tag) {...@@ -236,6 +270,16 @@ const Os = switch (builtin.os.tag) {
236 w.generation +%= 1;270 w.generation +%= 1;
237 }271 }
238 }272 }
273
274 fn wait(w: *Watch, gpa: Allocator, timeout: Timeout) !WaitResult {
275 const events_len = try std.posix.poll(&w.os.poll_fds, timeout.to_i32_ms());
276 return if (events_len == 0)
277 .timeout
278 else if (try Os.markDirtySteps(w, gpa))
279 .dirty
280 else
281 .clean;
282 }
239 },283 },
240 .windows => struct {284 .windows => struct {
241 const windows = std.os.windows;285 const windows = std.os.windows;
...@@ -356,6 +400,21 @@ const Os = switch (builtin.os.tag) {...@@ -356,6 +400,21 @@ const Os = switch (builtin.os.tag) {
356 }400 }
357 };401 };
358402
403 fn init() !Watch {
404 return .{
405 .dir_table = .{},
406 .os = switch (builtin.os.tag) {
407 .windows => .{
408 .handle_table = .{},
409 .dir_list = .{},
410 .io_cp = null,
411 },
412 else => {},
413 },
414 .generation = 0,
415 };
416 }
417
359 fn getFileId(handle: windows.HANDLE) !FileId {418 fn getFileId(handle: windows.HANDLE) !FileId {
360 var file_id: FileId = undefined;419 var file_id: FileId = undefined;
361 var io_status: windows.IO_STATUS_BLOCK = undefined;420 var io_status: windows.IO_STATUS_BLOCK = undefined;
...@@ -509,59 +568,258 @@ const Os = switch (builtin.os.tag) {...@@ -509,59 +568,258 @@ const Os = switch (builtin.os.tag) {
509 w.generation +%= 1;568 w.generation +%= 1;
510 }569 }
511 }570 }
571
572 fn wait(w: *Watch, gpa: Allocator, timeout: Timeout) !WaitResult {
573 var bytes_transferred: std.os.windows.DWORD = undefined;
574 var key: usize = undefined;
575 var overlapped_ptr: ?*std.os.windows.OVERLAPPED = undefined;
576 return while (true) switch (std.os.windows.GetQueuedCompletionStatus(
577 w.os.io_cp.?,
578 &bytes_transferred,
579 &key,
580 &overlapped_ptr,
581 @bitCast(timeout.to_i32_ms()),
582 )) {
583 .Normal => {
584 if (bytes_transferred == 0)
585 break error.Unexpected;
586
587 // This 'orelse' detects a race condition that happens when we receive a
588 // completion notification for a directory that no longer exists in our list.
589 const dir = w.os.dir_list.get(key) orelse break .clean;
590
591 break if (try Os.markDirtySteps(w, gpa, dir))
592 .dirty
593 else
594 .clean;
595 },
596 .Timeout => break .timeout,
597 // This status is issued because CancelIo was called, skip and try again.
598 .Cancelled => continue,
599 else => break error.Unexpected,
600 };
601 }
512 },602 },
513 else => void,603 .dragonfly, .freebsd, .netbsd, .openbsd, .ios, .macos, .tvos, .visionos, .watchos, .haiku => struct {
514};604 const posix = std.posix;
515605
516pub fn init() !Watch {606 kq_fd: i32,
517 switch (builtin.os.tag) {607 /// Indexes correspond 1:1 with `dir_table`.
518 .linux => {608 handles: std.MultiArrayList(struct {
519 const fan_fd = std.posix.fanotify_init(.{609 rs: ReactionSet,
520 .CLASS = .NOTIF,610 /// If the corresponding dir_table Path has sub_path == "", then it
611 /// suffices as the open directory handle, and this value will be
612 /// -1. Otherwise, it needs to be opened in update(), and will be
613 /// stored here.
614 dir_fd: i32,
615 /// Number of files being watched by this directory handle.
616 ref_count: u32,
617 }),
618
619 const dir_open_flags: posix.O = f: {
620 var f: posix.O = .{
621 .ACCMODE = .RDONLY,
622 .NOFOLLOW = false,
623 .DIRECTORY = true,
521 .CLOEXEC = true,624 .CLOEXEC = true,
522 .NONBLOCK = true,
523 .REPORT_NAME = true,
524 .REPORT_DIR_FID = true,
525 .REPORT_FID = true,
526 .REPORT_TARGET_FID = true,
527 }, 0) catch |err| switch (err) {
528 error.UnsupportedFlags => fatal("fanotify_init failed due to old kernel; requires 5.17+", .{}),
529 else => |e| return e,
530 };
531 return .{
532 .dir_table = .{},
533 .os = switch (builtin.os.tag) {
534 .linux => .{
535 .handle_table = .{},
536 .poll_fds = .{
537 .{
538 .fd = fan_fd,
539 .events = std.posix.POLL.IN,
540 .revents = undefined,
541 },
542 },
543 },
544 else => {},
545 },
546 .generation = 0,
547 };625 };
548 },626 if (@hasField(posix.O, "EVTONLY")) f.EVTONLY = true;
549 .windows => {627 if (@hasField(posix.O, "PATH")) f.PATH = true;
628 break :f f;
629 };
630
631 const EV = std.c.EV;
632 const NOTE = std.c.NOTE;
633
634 fn init() !Watch {
635 const kq_fd = try posix.kqueue();
636 errdefer posix.close(kq_fd);
550 return .{637 return .{
551 .dir_table = .{},638 .dir_table = .{},
552 .os = switch (builtin.os.tag) {639 .os = .{
553 .windows => .{640 .kq_fd = kq_fd,
554 .handle_table = .{},641 .handles = .empty,
555 .dir_list = .{},
556 .io_cp = null,
557 },
558 else => {},
559 },642 },
560 .generation = 0,643 .generation = 0,
561 };644 };
562 },645 }
563 else => @panic("unimplemented"),646
564 }647 fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void {
648 const handles = &w.os.handles;
649 for (steps) |step| {
650 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
651 const reaction_set = rs: {
652 const gop = try w.dir_table.getOrPut(gpa, path);
653 if (!gop.found_existing) {
654 const skip_open_dir = path.sub_path.len == 0;
655 const dir_fd = if (skip_open_dir)
656 path.root_dir.handle.fd
657 else
658 posix.openat(path.root_dir.handle.fd, path.sub_path, dir_open_flags, 0) catch |err| {
659 fatal("failed to open directory {}: {s}", .{ path, @errorName(err) });
660 };
661 // Empirically the dir has to stay open or else no events are triggered.
662 errdefer if (!skip_open_dir) posix.close(dir_fd);
663 const changes = [1]posix.Kevent{.{
664 .ident = @bitCast(@as(isize, dir_fd)),
665 .filter = std.c.EVFILT.VNODE,
666 .flags = EV.ADD | EV.ENABLE | EV.CLEAR,
667 .fflags = NOTE.DELETE | NOTE.WRITE | NOTE.RENAME | NOTE.REVOKE,
668 .data = 0,
669 .udata = gop.index,
670 }};
671 _ = try posix.kevent(w.os.kq_fd, &changes, &.{}, null);
672 assert(handles.len == gop.index);
673 try handles.append(gpa, .{
674 .rs = .{},
675 .dir_fd = if (skip_open_dir) -1 else dir_fd,
676 .ref_count = 1,
677 });
678 } else {
679 handles.items(.ref_count)[gop.index] += 1;
680 }
681 break :rs &handles.items(.rs)[gop.index];
682 };
683 for (files.items) |basename| {
684 const gop = try reaction_set.getOrPut(gpa, basename);
685 if (!gop.found_existing) gop.value_ptr.* = .{};
686 try gop.value_ptr.put(gpa, step, w.generation);
687 }
688 }
689 }
690
691 {
692 // Remove marks for files that are no longer inputs.
693 var i: usize = 0;
694 while (i < handles.len) {
695 {
696 const reaction_set = &handles.items(.rs)[i];
697 var step_set_i: usize = 0;
698 while (step_set_i < reaction_set.entries.len) {
699 const step_set = &reaction_set.values()[step_set_i];
700 var dirent_i: usize = 0;
701 while (dirent_i < step_set.entries.len) {
702 const generations = step_set.values();
703 if (generations[dirent_i] == w.generation) {
704 dirent_i += 1;
705 continue;
706 }
707 step_set.swapRemoveAt(dirent_i);
708 }
709 if (step_set.entries.len > 0) {
710 step_set_i += 1;
711 continue;
712 }
713 reaction_set.swapRemoveAt(step_set_i);
714 }
715 if (reaction_set.entries.len > 0) {
716 i += 1;
717 continue;
718 }
719 }
720
721 const ref_count_ptr = &handles.items(.ref_count)[i];
722 ref_count_ptr.* -= 1;
723 if (ref_count_ptr.* > 0) continue;
724
725 // If the sub_path == "" then this patch has already the
726 // dir fd that we need to use as the ident to remove the
727 // event. If it was opened above with openat() then we need
728 // to access that data via the dir_fd field.
729 const path = w.dir_table.keys()[i];
730 const dir_fd = if (path.sub_path.len == 0)
731 path.root_dir.handle.fd
732 else
733 handles.items(.dir_fd)[i];
734 assert(dir_fd != -1);
735
736 // The changelist also needs to update the udata field of the last
737 // event, since we are doing a swap remove, and we store the dir_table
738 // index in the udata field.
739 const last_dir_fd = fd: {
740 const last_path = w.dir_table.keys()[handles.len - 1];
741 const last_dir_fd = if (last_path.sub_path.len != 0)
742 last_path.root_dir.handle.fd
743 else
744 handles.items(.dir_fd)[i];
745 assert(last_dir_fd != -1);
746 break :fd last_dir_fd;
747 };
748 const changes = [_]posix.Kevent{
749 .{
750 .ident = @bitCast(@as(isize, dir_fd)),
751 .filter = std.c.EVFILT.VNODE,
752 .flags = EV.DELETE,
753 .fflags = 0,
754 .data = 0,
755 .udata = i,
756 },
757 .{
758 .ident = @bitCast(@as(isize, last_dir_fd)),
759 .filter = std.c.EVFILT.VNODE,
760 .flags = EV.ADD,
761 .fflags = NOTE.DELETE | NOTE.WRITE | NOTE.RENAME | NOTE.REVOKE,
762 .data = 0,
763 .udata = i,
764 },
765 };
766 const filtered_changes = if (i == handles.len - 1) changes[0..1] else &changes;
767 _ = try posix.kevent(w.os.kq_fd, filtered_changes, &.{}, null);
768 if (path.sub_path.len != 0) posix.close(dir_fd);
769
770 w.dir_table.swapRemoveAt(i);
771 handles.swapRemove(i);
772 }
773 w.generation +%= 1;
774 }
775 }
776
777 fn wait(w: *Watch, gpa: Allocator, timeout: Timeout) !WaitResult {
778 var timespec_buffer: posix.timespec = undefined;
779 var event_buffer: [100]posix.Kevent = undefined;
780 var n = try posix.kevent(w.os.kq_fd, &.{}, &event_buffer, timeout.toTimespec(&timespec_buffer));
781 if (n == 0) return .timeout;
782 const reaction_sets = w.os.handles.items(.rs);
783 var any_dirty = markDirtySteps(gpa, reaction_sets, event_buffer[0..n], false);
784 timespec_buffer = .{ .sec = 0, .nsec = 0 };
785 while (n == event_buffer.len) {
786 n = try posix.kevent(w.os.kq_fd, &.{}, &event_buffer, &timespec_buffer);
787 if (n == 0) break;
788 any_dirty = markDirtySteps(gpa, reaction_sets, event_buffer[0..n], any_dirty);
789 }
790 return if (any_dirty) .dirty else .clean;
791 }
792
793 fn markDirtySteps(
794 gpa: Allocator,
795 reaction_sets: []ReactionSet,
796 events: []const std.c.Kevent,
797 start_any_dirty: bool,
798 ) bool {
799 var any_dirty = start_any_dirty;
800 for (events) |event| {
801 const index: usize = @intCast(event.udata);
802 const reaction_set = &reaction_sets[index];
803 // If we knew the basename of the changed file, here we would
804 // mark only the step set dirty, and possibly the glob set:
805 //if (reaction_set.getPtr(".")) |glob_set|
806 // any_dirty = markStepSetDirty(gpa, glob_set, any_dirty);
807 //if (reaction_set.getPtr(file_name)) |step_set|
808 // any_dirty = markStepSetDirty(gpa, step_set, any_dirty);
809 // However we don't know the file name so just mark all the
810 // sets dirty for this directory.
811 for (reaction_set.values()) |*step_set| {
812 any_dirty = markStepSetDirty(gpa, step_set, any_dirty);
813 }
814 }
815 return any_dirty;
816 }
817 },
818 else => void,
819};
820
821pub fn init() !Watch {
822 return Os.init();
565}823}
566824
567pub const Match = struct {825pub const Match = struct {
...@@ -609,10 +867,7 @@ fn markStepSetDirty(gpa: Allocator, step_set: *StepSet, any_dirty: bool) bool {...@@ -609,10 +867,7 @@ fn markStepSetDirty(gpa: Allocator, step_set: *StepSet, any_dirty: bool) bool {
609}867}
610868
611pub fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void {869pub fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void {
612 switch (builtin.os.tag) {870 return Os.update(w, gpa, steps);
613 .linux, .windows => return Os.update(w, gpa, steps),
614 else => @compileError("unimplemented"),
615 }
616}871}
617872
618pub const Timeout = union(enum) {873pub const Timeout = union(enum) {
...@@ -625,6 +880,20 @@ pub const Timeout = union(enum) {...@@ -625,6 +880,20 @@ pub const Timeout = union(enum) {
625 .ms => |ms| ms,880 .ms => |ms| ms,
626 };881 };
627 }882 }
883
884 pub fn toTimespec(t: Timeout, buf: *std.posix.timespec) ?*std.posix.timespec {
885 return switch (t) {
886 .none => null,
887 .ms => |ms_u16| {
888 const ms: isize = ms_u16;
889 buf.* = .{
890 .sec = @divTrunc(ms, std.time.ms_per_s),
891 .nsec = @rem(ms, std.time.ms_per_s) * std.time.ns_per_ms,
892 };
893 return buf;
894 },
895 };
896 }
628};897};
629898
630pub const WaitResult = enum {899pub const WaitResult = enum {
...@@ -638,46 +907,5 @@ pub const WaitResult = enum {...@@ -638,46 +907,5 @@ pub const WaitResult = enum {
638};907};
639908
640pub fn wait(w: *Watch, gpa: Allocator, timeout: Timeout) !WaitResult {909pub fn wait(w: *Watch, gpa: Allocator, timeout: Timeout) !WaitResult {
641 switch (builtin.os.tag) {910 return Os.wait(w, gpa, timeout);
642 .linux => {
643 const events_len = try std.posix.poll(&w.os.poll_fds, timeout.to_i32_ms());
644 return if (events_len == 0)
645 .timeout
646 else if (try Os.markDirtySteps(w, gpa))
647 .dirty
648 else
649 .clean;
650 },
651 .windows => {
652 var bytes_transferred: std.os.windows.DWORD = undefined;
653 var key: usize = undefined;
654 var overlapped_ptr: ?*std.os.windows.OVERLAPPED = undefined;
655 return while (true) switch (std.os.windows.GetQueuedCompletionStatus(
656 w.os.io_cp.?,
657 &bytes_transferred,
658 &key,
659 &overlapped_ptr,
660 @bitCast(timeout.to_i32_ms()),
661 )) {
662 .Normal => {
663 if (bytes_transferred == 0)
664 break error.Unexpected;
665
666 // This 'orelse' detects a race condition that happens when we receive a
667 // completion notification for a directory that no longer exists in our list.
668 const dir = w.os.dir_list.get(key) orelse break .clean;
669
670 break if (try Os.markDirtySteps(w, gpa, dir))
671 .dirty
672 else
673 .clean;
674 },
675 .Timeout => break .timeout,
676 // This status is issued because CancelIo was called, skip and try again.
677 .Cancelled => continue,
678 else => break error.Unexpected,
679 };
680 },
681 else => @compileError("unimplemented"),
682 }
683}911}