authorgravatar for kevin@boula.inKevin Boulain <kevin@boula.in> 2025-06-07 18:58:24+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-06-23 06:08:25+02:00
log640a130651ee39f8d8f5afc9498574c10af18ba9
treeed7fee0761f5b27b8b9fcb5491f54f88d65fc2f6
parent1df79ab895a5740611b57fb091d0487ec21a762d

std.Build.Watch: key fanotify file descriptors by mount id

Since marks are limited to a single filesystem. Context: https://github.com/ziglang/zig/issues/20670 Original pull request: https://github.com/ziglang/zig/pull/20672 Co-authored-by: Maciej 'vesim' Kuliński <vesim809@pm.me>

1 files changed, 56 insertions(+), 47 deletions(-)

lib/std/Build/Watch.zig+56-47
...@@ -33,9 +33,12 @@ const Os = switch (builtin.os.tag) {...@@ -33,9 +33,12 @@ const Os = switch (builtin.os.tag) {
3333
34 /// Keyed differently but indexes correspond 1:1 with `dir_table`.34 /// Keyed differently but indexes correspond 1:1 with `dir_table`.
35 handle_table: HandleTable,35 handle_table: HandleTable,
36 poll_fds: [1]posix.pollfd,36 /// fanotify file descriptors are keyed by mount id since marks
37 /// are limited to a single filesystem.
38 poll_fds: std.AutoArrayHashMapUnmanaged(MountId, posix.pollfd),
3739
38 const HandleTable = std.ArrayHashMapUnmanaged(FileHandle, ReactionSet, FileHandle.Adapter, false);40 const MountId = i32;
41 const HandleTable = std.ArrayHashMapUnmanaged(FileHandle, struct { mount_id: MountId, reaction_set: ReactionSet }, FileHandle.Adapter, false);
3942
40 const fan_mask: std.os.linux.fanotify.MarkMask = .{43 const fan_mask: std.os.linux.fanotify.MarkMask = .{
41 .CLOSE_WRITE = true,44 .CLOSE_WRITE = true,
...@@ -92,30 +95,12 @@ const Os = switch (builtin.os.tag) {...@@ -92,30 +95,12 @@ const Os = switch (builtin.os.tag) {
92 };95 };
9396
94 fn init() !Watch {97 fn init() !Watch {
95 const fan_fd = std.posix.fanotify_init(.{
96 .CLASS = .NOTIF,
97 .CLOEXEC = true,
98 .NONBLOCK = true,
99 .REPORT_NAME = true,
100 .REPORT_DIR_FID = true,
101 .REPORT_FID = true,
102 .REPORT_TARGET_FID = true,
103 }, 0) catch |err| switch (err) {
104 error.UnsupportedFlags => fatal("fanotify_init failed due to old kernel; requires 5.17+", .{}),
105 else => |e| return e,
106 };
107 return .{98 return .{
108 .dir_table = .{},99 .dir_table = .{},
109 .os = switch (builtin.os.tag) {100 .os = switch (builtin.os.tag) {
110 .linux => .{101 .linux => .{
111 .handle_table = .{},102 .handle_table = .{},
112 .poll_fds = .{103 .poll_fds = .{},
113 .{
114 .fd = fan_fd,
115 .events = std.posix.POLL.IN,
116 .revents = undefined,
117 },
118 },
119 },104 },
120 else => {},105 else => {},
121 },106 },
...@@ -123,22 +108,20 @@ const Os = switch (builtin.os.tag) {...@@ -123,22 +108,20 @@ const Os = switch (builtin.os.tag) {
123 };108 };
124 }109 }
125110
126 fn getDirHandle(gpa: Allocator, path: std.Build.Cache.Path) !FileHandle {111 fn getDirHandle(gpa: Allocator, path: std.Build.Cache.Path, mount_id: *MountId) !FileHandle {
127 var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined;112 var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined;
128 var mount_id: i32 = undefined;
129 var buf: [std.fs.max_path_bytes]u8 = undefined;113 var buf: [std.fs.max_path_bytes]u8 = undefined;
130 const adjusted_path = if (path.sub_path.len == 0) "./" else std.fmt.bufPrint(&buf, "{s}/", .{114 const adjusted_path = if (path.sub_path.len == 0) "./" else std.fmt.bufPrint(&buf, "{s}/", .{
131 path.sub_path,115 path.sub_path,
132 }) catch return error.NameTooLong;116 }) catch return error.NameTooLong;
133 const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer);117 const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer);
134 stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle);118 stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle);
135 try posix.name_to_handle_at(path.root_dir.handle.fd, adjusted_path, stack_ptr, &mount_id, std.os.linux.AT.HANDLE_FID);119 try posix.name_to_handle_at(path.root_dir.handle.fd, adjusted_path, stack_ptr, mount_id, std.os.linux.AT.HANDLE_FID);
136 const stack_lfh: FileHandle = .{ .handle = stack_ptr };120 const stack_lfh: FileHandle = .{ .handle = stack_ptr };
137 return stack_lfh.clone(gpa);121 return stack_lfh.clone(gpa);
138 }122 }
139123
140 fn markDirtySteps(w: *Watch, gpa: Allocator) !bool {124 fn markDirtySteps(w: *Watch, gpa: Allocator, fan_fd: posix.fd_t) !bool {
141 const fan_fd = w.os.getFanFd();
142 const fanotify = std.os.linux.fanotify;125 const fanotify = std.os.linux.fanotify;
143 const M = fanotify.event_metadata;126 const M = fanotify.event_metadata;
144 var events_buf: [256 + 4096]u8 = undefined;127 var events_buf: [256 + 4096]u8 = undefined;
...@@ -167,10 +150,10 @@ const Os = switch (builtin.os.tag) {...@@ -167,10 +150,10 @@ const Os = switch (builtin.os.tag) {
167 const file_name_z: [*:0]u8 = @ptrCast((&file_handle.f_handle).ptr + file_handle.handle_bytes);150 const file_name_z: [*:0]u8 = @ptrCast((&file_handle.f_handle).ptr + file_handle.handle_bytes);
168 const file_name = std.mem.span(file_name_z);151 const file_name = std.mem.span(file_name_z);
169 const lfh: FileHandle = .{ .handle = file_handle };152 const lfh: FileHandle = .{ .handle = file_handle };
170 if (w.os.handle_table.getPtr(lfh)) |reaction_set| {153 if (w.os.handle_table.getPtr(lfh)) |value| {
171 if (reaction_set.getPtr(".")) |glob_set|154 if (value.reaction_set.getPtr(".")) |glob_set|
172 any_dirty = markStepSetDirty(gpa, glob_set, any_dirty);155 any_dirty = markStepSetDirty(gpa, glob_set, any_dirty);
173 if (reaction_set.getPtr(file_name)) |step_set|156 if (value.reaction_set.getPtr(file_name)) |step_set|
174 any_dirty = markStepSetDirty(gpa, step_set, any_dirty);157 any_dirty = markStepSetDirty(gpa, step_set, any_dirty);
175 }158 }
176 },159 },
...@@ -180,19 +163,38 @@ const Os = switch (builtin.os.tag) {...@@ -180,19 +163,38 @@ const Os = switch (builtin.os.tag) {
180 }163 }
181 }164 }
182165
183 fn getFanFd(os: *const @This()) posix.fd_t {
184 return os.poll_fds[0].fd;
185 }
186
187 fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void {166 fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void {
188 const fan_fd = w.os.getFanFd();
189 // Add missing marks and note persisted ones.167 // Add missing marks and note persisted ones.
190 for (steps) |step| {168 for (steps) |step| {
191 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {169 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
192 const reaction_set = rs: {170 const reaction_set = rs: {
193 const gop = try w.dir_table.getOrPut(gpa, path);171 const gop = try w.dir_table.getOrPut(gpa, path);
194 if (!gop.found_existing) {172 if (!gop.found_existing) {
195 const dir_handle = try Os.getDirHandle(gpa, path);173 var mount_id: MountId = undefined;
174 const dir_handle = try Os.getDirHandle(gpa, path, &mount_id);
175 const fan_fd = blk: {
176 const fd_gop = try w.os.poll_fds.getOrPut(gpa, mount_id);
177 if (!fd_gop.found_existing) {
178 const fan_fd = std.posix.fanotify_init(.{
179 .CLASS = .NOTIF,
180 .CLOEXEC = true,
181 .NONBLOCK = true,
182 .REPORT_NAME = true,
183 .REPORT_DIR_FID = true,
184 .REPORT_FID = true,
185 .REPORT_TARGET_FID = true,
186 }, 0) catch |err| switch (err) {
187 error.UnsupportedFlags => fatal("fanotify_init failed due to old kernel; requires 5.17+", .{}),
188 else => |e| return e,
189 };
190 fd_gop.value_ptr.* = .{
191 .fd = fan_fd,
192 .events = std.posix.POLL.IN,
193 .revents = undefined,
194 };
195 }
196 break :blk fd_gop.value_ptr.*.fd;
197 };
196 // `dir_handle` may already be present in the table in198 // `dir_handle` may already be present in the table in
197 // the case that we have multiple Cache.Path instances199 // the case that we have multiple Cache.Path instances
198 // that compare inequal but ultimately point to the same200 // that compare inequal but ultimately point to the same
...@@ -204,7 +206,7 @@ const Os = switch (builtin.os.tag) {...@@ -204,7 +206,7 @@ const Os = switch (builtin.os.tag) {
204 _ = w.dir_table.pop();206 _ = w.dir_table.pop();
205 } else {207 } else {
206 assert(dh_gop.index == gop.index);208 assert(dh_gop.index == gop.index);
207 dh_gop.value_ptr.* = .{};209 dh_gop.value_ptr.* = .{ .mount_id = mount_id, .reaction_set = .{} };
208 posix.fanotify_mark(fan_fd, .{210 posix.fanotify_mark(fan_fd, .{
209 .ADD = true,211 .ADD = true,
210 .ONLYDIR = true,212 .ONLYDIR = true,
...@@ -212,9 +214,9 @@ const Os = switch (builtin.os.tag) {...@@ -212,9 +214,9 @@ const Os = switch (builtin.os.tag) {
212 fatal("unable to watch {}: {s}", .{ path, @errorName(err) });214 fatal("unable to watch {}: {s}", .{ path, @errorName(err) });
213 };215 };
214 }216 }
215 break :rs dh_gop.value_ptr;217 break :rs &dh_gop.value_ptr.reaction_set;
216 }218 }
217 break :rs &w.os.handle_table.values()[gop.index];219 break :rs &w.os.handle_table.values()[gop.index].reaction_set;
218 };220 };
219 for (files.items) |basename| {221 for (files.items) |basename| {
220 const gop = try reaction_set.getOrPut(gpa, basename);222 const gop = try reaction_set.getOrPut(gpa, basename);
...@@ -229,7 +231,7 @@ const Os = switch (builtin.os.tag) {...@@ -229,7 +231,7 @@ const Os = switch (builtin.os.tag) {
229 var i: usize = 0;231 var i: usize = 0;
230 while (i < w.os.handle_table.entries.len) {232 while (i < w.os.handle_table.entries.len) {
231 {233 {
232 const reaction_set = &w.os.handle_table.values()[i];234 const reaction_set = &w.os.handle_table.values()[i].reaction_set;
233 var step_set_i: usize = 0;235 var step_set_i: usize = 0;
234 while (step_set_i < reaction_set.entries.len) {236 while (step_set_i < reaction_set.entries.len) {
235 const step_set = &reaction_set.values()[step_set_i];237 const step_set = &reaction_set.values()[step_set_i];
...@@ -256,6 +258,8 @@ const Os = switch (builtin.os.tag) {...@@ -256,6 +258,8 @@ const Os = switch (builtin.os.tag) {
256258
257 const path = w.dir_table.keys()[i];259 const path = w.dir_table.keys()[i];
258260
261 const mount_id = w.os.handle_table.values()[i].mount_id;
262 const fan_fd = w.os.poll_fds.getEntry(mount_id).?.value_ptr.fd;
259 posix.fanotify_mark(fan_fd, .{263 posix.fanotify_mark(fan_fd, .{
260 .REMOVE = true,264 .REMOVE = true,
261 .ONLYDIR = true,265 .ONLYDIR = true,
...@@ -272,13 +276,14 @@ const Os = switch (builtin.os.tag) {...@@ -272,13 +276,14 @@ const Os = switch (builtin.os.tag) {
272 }276 }
273277
274 fn wait(w: *Watch, gpa: Allocator, timeout: Timeout) !WaitResult {278 fn wait(w: *Watch, gpa: Allocator, timeout: Timeout) !WaitResult {
275 const events_len = try std.posix.poll(&w.os.poll_fds, timeout.to_i32_ms());279 const events_len = try std.posix.poll(w.os.poll_fds.values(), timeout.to_i32_ms());
276 return if (events_len == 0)280 if (events_len == 0)
277 .timeout281 return .timeout;
278 else if (try Os.markDirtySteps(w, gpa))282 for (w.os.poll_fds.values()) |poll_fd| {
279 .dirty283 if (poll_fd.revents & std.posix.POLL.IN == std.posix.POLL.IN and try Os.markDirtySteps(w, gpa, poll_fd.fd))
280 else284 return .dirty;
281 .clean;285 }
286 return .clean;
282 }287 }
283 },288 },
284 .windows => struct {289 .windows => struct {
...@@ -838,7 +843,11 @@ pub const Match = struct {...@@ -838,7 +843,11 @@ pub const Match = struct {
838};843};
839844
840fn markAllFilesDirty(w: *Watch, gpa: Allocator) void {845fn markAllFilesDirty(w: *Watch, gpa: Allocator) void {
841 for (w.os.handle_table.values()) |reaction_set| {846 for (w.os.handle_table.values()) |value| {
847 const reaction_set = switch (builtin.os.tag) {
848 .linux => value.reaction_set,
849 else => value,
850 };
842 for (reaction_set.values()) |step_set| {851 for (reaction_set.values()) |step_set| {
843 for (step_set.keys()) |step| {852 for (step_set.keys()) |step| {
844 step.recursiveReset(gpa);853 step.recursiveReset(gpa);