authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-28 20:35:18-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-07-28 20:35:18-07:00
log3a0da431dba3ba4a56fd297f9601f7c959f7d3cc
treeb2af2c343167b78bb675192d6feba35594f2db9a
parent939fcce5ef415142631e246a91473317a8d3fa90
parent2de0e2eca36f39c077b48d0dc4e4d7ddebe0d553
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #20667 from jayrod246/windows-watch

Build Runner: Initial Implementation for File System Watching on Windows

4 files changed, 347 insertions(+), 11 deletions(-)

lib/compiler/build_runner.zig+1-1
......@@ -409,7 +409,7 @@ pub fn main() !void {
409409 if (!watch) return cleanExit();
410410
411411 switch (builtin.os.tag) {
412 .linux => {},
412 .linux, .windows => {},
413413 else => fatal("--watch not yet implemented for {s}", .{@tagName(builtin.os.tag)}),
414414 }
415415
lib/std/Build/Watch.zig+307-1
......@@ -237,6 +237,273 @@ const Os = switch (builtin.os.tag) {
237237 }
238238 }
239239 },
240 .windows => struct {
241 const windows = std.os.windows;
242
243 /// Keyed differently but indexes correspond 1:1 with `dir_table`.
244 handle_table: HandleTable,
245 dir_list: std.ArrayListUnmanaged(*Directory),
246 io_cp: ?windows.HANDLE,
247
248 const HandleTable = std.AutoArrayHashMapUnmanaged(FileId, ReactionSet);
249
250 const FileId = struct {
251 volumeSerialNumber: windows.ULONG,
252 indexNumber: windows.LARGE_INTEGER,
253 };
254
255 const Directory = struct {
256 handle: windows.HANDLE,
257 id: FileId,
258 overlapped: windows.OVERLAPPED,
259 // 64 KB is the packet size limit when monitoring over a network.
260 // https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-readdirectorychangesw#remarks
261 buffer: [64 * 1024]u8 align(@alignOf(windows.FILE_NOTIFY_INFORMATION)) = undefined,
262
263 fn readChanges(self: *@This()) !void {
264 const r = windows.kernel32.ReadDirectoryChangesW(
265 self.handle,
266 @ptrCast(&self.buffer),
267 self.buffer.len,
268 0,
269 .{
270 .creation = true,
271 .dir_name = true,
272 .file_name = true,
273 .last_write = true,
274 .size = true,
275 },
276 null,
277 &self.overlapped,
278 null,
279 );
280 if (r == windows.FALSE) {
281 switch (windows.GetLastError()) {
282 .INVALID_FUNCTION => return error.ReadDirectoryChangesUnsupported,
283 else => |err| return windows.unexpectedError(err),
284 }
285 }
286 }
287
288 fn init(gpa: Allocator, path: Cache.Path) !*@This() {
289 // The following code is a drawn out NtCreateFile call. (mostly adapted from std.fs.Dir.makeOpenDirAccessMaskW)
290 // It's necessary in order to get the specific flags that are required when calling ReadDirectoryChangesW.
291 var dir_handle: windows.HANDLE = undefined;
292 const root_fd = path.root_dir.handle.fd;
293 const sub_path = path.subPathOrDot();
294 const sub_path_w = try windows.sliceToPrefixedFileW(root_fd, sub_path);
295 const path_len_bytes = std.math.cast(u16, sub_path_w.len * 2) orelse return error.NameTooLong;
296
297 var nt_name = windows.UNICODE_STRING{
298 .Length = @intCast(path_len_bytes),
299 .MaximumLength = @intCast(path_len_bytes),
300 .Buffer = @constCast(sub_path_w.span().ptr),
301 };
302 var attr = windows.OBJECT_ATTRIBUTES{
303 .Length = @sizeOf(windows.OBJECT_ATTRIBUTES),
304 .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(sub_path_w.span())) null else root_fd,
305 .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here.
306 .ObjectName = &nt_name,
307 .SecurityDescriptor = null,
308 .SecurityQualityOfService = null,
309 };
310 var io: windows.IO_STATUS_BLOCK = undefined;
311
312 switch (windows.ntdll.NtCreateFile(
313 &dir_handle,
314 windows.SYNCHRONIZE | windows.GENERIC_READ | windows.FILE_LIST_DIRECTORY,
315 &attr,
316 &io,
317 null,
318 0,
319 windows.FILE_SHARE_READ | windows.FILE_SHARE_WRITE | windows.FILE_SHARE_DELETE,
320 windows.FILE_OPEN,
321 windows.FILE_DIRECTORY_FILE | windows.FILE_OPEN_FOR_BACKUP_INTENT,
322 null,
323 0,
324 )) {
325 .SUCCESS => {},
326 .OBJECT_NAME_INVALID => return error.BadPathName,
327 .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
328 .OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
329 .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
330 .NOT_A_DIRECTORY => return error.NotDir,
331 // This can happen if the directory has 'List folder contents' permission set to 'Deny'
332 .ACCESS_DENIED => return error.AccessDenied,
333 .INVALID_PARAMETER => unreachable,
334 else => |rc| return windows.unexpectedStatus(rc),
335 }
336 assert(dir_handle != windows.INVALID_HANDLE_VALUE);
337 errdefer windows.CloseHandle(dir_handle);
338
339 const dir_id = try getFileId(dir_handle);
340
341 const dir_ptr = try gpa.create(@This());
342 dir_ptr.* = .{
343 .handle = dir_handle,
344 .id = dir_id,
345 .overlapped = std.mem.zeroes(windows.OVERLAPPED),
346 };
347 return dir_ptr;
348 }
349
350 fn deinit(self: *@This(), gpa: Allocator) void {
351 _ = windows.kernel32.CancelIo(self.handle);
352 windows.CloseHandle(self.handle);
353 gpa.destroy(self);
354 }
355 };
356
357 fn getFileId(handle: windows.HANDLE) !FileId {
358 var file_id: FileId = undefined;
359 var io_status: windows.IO_STATUS_BLOCK = undefined;
360 var volume_info: windows.FILE_FS_VOLUME_INFORMATION = undefined;
361 switch (windows.ntdll.NtQueryVolumeInformationFile(
362 handle,
363 &io_status,
364 &volume_info,
365 @sizeOf(windows.FILE_FS_VOLUME_INFORMATION),
366 .FileFsVolumeInformation,
367 )) {
368 .SUCCESS => {},
369 // Buffer overflow here indicates that there is more information available than was able to be stored in the buffer
370 // size provided. This is treated as success because the type of variable-length information that this would be relevant for
371 // (name, volume name, etc) we don't care about.
372 .BUFFER_OVERFLOW => {},
373 else => |rc| return windows.unexpectedStatus(rc),
374 }
375 file_id.volumeSerialNumber = volume_info.VolumeSerialNumber;
376 var internal_info: windows.FILE_INTERNAL_INFORMATION = undefined;
377 switch (windows.ntdll.NtQueryInformationFile(
378 handle,
379 &io_status,
380 &internal_info,
381 @sizeOf(windows.FILE_INTERNAL_INFORMATION),
382 .FileInternalInformation,
383 )) {
384 .SUCCESS => {},
385 else => |rc| return windows.unexpectedStatus(rc),
386 }
387 file_id.indexNumber = internal_info.IndexNumber;
388 return file_id;
389 }
390
391 fn markDirtySteps(w: *Watch, gpa: Allocator, dir: *Directory) !bool {
392 var any_dirty = false;
393 const bytes_returned = try windows.GetOverlappedResult(dir.handle, &dir.overlapped, false);
394 if (bytes_returned == 0) {
395 std.log.warn("file system watch queue overflowed; falling back to fstat", .{});
396 markAllFilesDirty(w, gpa);
397 return true;
398 }
399 var file_name_buf: [std.fs.max_path_bytes]u8 = undefined;
400 var notify: *align(1) windows.FILE_NOTIFY_INFORMATION = undefined;
401 var offset: usize = 0;
402 while (true) {
403 notify = @ptrCast(&dir.buffer[offset]);
404 const file_name_field: [*]u16 = @ptrFromInt(@intFromPtr(notify) + @sizeOf(windows.FILE_NOTIFY_INFORMATION));
405 const file_name_len = std.unicode.wtf16LeToWtf8(&file_name_buf, file_name_field[0 .. notify.FileNameLength / 2]);
406 const file_name = file_name_buf[0..file_name_len];
407 if (w.os.handle_table.getIndex(dir.id)) |reaction_set_i| {
408 const reaction_set = w.os.handle_table.values()[reaction_set_i];
409 if (reaction_set.getPtr(".")) |glob_set|
410 any_dirty = markStepSetDirty(gpa, glob_set, any_dirty);
411 if (reaction_set.getPtr(file_name)) |step_set| {
412 any_dirty = markStepSetDirty(gpa, step_set, any_dirty);
413 }
414 }
415 if (notify.NextEntryOffset == 0)
416 break;
417
418 offset += notify.NextEntryOffset;
419 }
420
421 try dir.readChanges();
422 return any_dirty;
423 }
424
425 fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void {
426 // Add missing marks and note persisted ones.
427 for (steps) |step| {
428 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
429 const reaction_set = rs: {
430 const gop = try w.dir_table.getOrPut(gpa, path);
431 if (!gop.found_existing) {
432 const dir = try Os.Directory.init(gpa, path);
433 errdefer dir.deinit(gpa);
434 // `dir.id` may already be present in the table in
435 // the case that we have multiple Cache.Path instances
436 // that compare inequal but ultimately point to the same
437 // directory on the file system.
438 // In such case, we must revert adding this directory, but keep
439 // the additions to the step set.
440 const dh_gop = try w.os.handle_table.getOrPut(gpa, dir.id);
441 if (dh_gop.found_existing) {
442 dir.deinit(gpa);
443 _ = w.dir_table.pop();
444 } else {
445 assert(dh_gop.index == gop.index);
446 dh_gop.value_ptr.* = .{};
447 try dir.readChanges();
448 try w.os.dir_list.insert(gpa, dh_gop.index, dir);
449 w.os.io_cp = try windows.CreateIoCompletionPort(
450 dir.handle,
451 w.os.io_cp,
452 dh_gop.index,
453 0,
454 );
455 }
456 break :rs &w.os.handle_table.values()[dh_gop.index];
457 }
458 break :rs &w.os.handle_table.values()[gop.index];
459 };
460 for (files.items) |basename| {
461 const gop = try reaction_set.getOrPut(gpa, basename);
462 if (!gop.found_existing) gop.value_ptr.* = .{};
463 try gop.value_ptr.put(gpa, step, w.generation);
464 }
465 }
466 }
467
468 {
469 // Remove marks for files that are no longer inputs.
470 var i: usize = 0;
471 while (i < w.os.handle_table.entries.len) {
472 {
473 const reaction_set = &w.os.handle_table.values()[i];
474 var step_set_i: usize = 0;
475 while (step_set_i < reaction_set.entries.len) {
476 const step_set = &reaction_set.values()[step_set_i];
477 var dirent_i: usize = 0;
478 while (dirent_i < step_set.entries.len) {
479 const generations = step_set.values();
480 if (generations[dirent_i] == w.generation) {
481 dirent_i += 1;
482 continue;
483 }
484 step_set.swapRemoveAt(dirent_i);
485 }
486 if (step_set.entries.len > 0) {
487 step_set_i += 1;
488 continue;
489 }
490 reaction_set.swapRemoveAt(step_set_i);
491 }
492 if (reaction_set.entries.len > 0) {
493 i += 1;
494 continue;
495 }
496 }
497
498 w.os.dir_list.items[i].deinit(gpa);
499 _ = w.os.dir_list.swapRemove(i);
500 w.dir_table.swapRemoveAt(i);
501 w.os.handle_table.swapRemoveAt(i);
502 }
503 w.generation +%= 1;
504 }
505 }
506 },
240507 else => void,
241508};
242509
......@@ -270,6 +537,20 @@ pub fn init() !Watch {
270537 .generation = 0,
271538 };
272539 },
540 .windows => {
541 return .{
542 .dir_table = .{},
543 .os = switch (builtin.os.tag) {
544 .windows => .{
545 .handle_table = .{},
546 .dir_list = .{},
547 .io_cp = null,
548 },
549 else => {},
550 },
551 .generation = 0,
552 };
553 },
273554 else => @panic("unimplemented"),
274555 }
275556}
......@@ -320,7 +601,7 @@ fn markStepSetDirty(gpa: Allocator, step_set: *StepSet, any_dirty: bool) bool {
320601
321602pub fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void {
322603 switch (builtin.os.tag) {
323 .linux => return Os.update(w, gpa, steps),
604 .linux, .windows => return Os.update(w, gpa, steps),
324605 else => @compileError("unimplemented"),
325606 }
326607}
......@@ -358,6 +639,31 @@ pub fn wait(w: *Watch, gpa: Allocator, timeout: Timeout) !WaitResult {
358639 else
359640 .clean;
360641 },
642 .windows => {
643 var bytes_transferred: std.os.windows.DWORD = undefined;
644 var key: usize = undefined;
645 var overlapped_ptr: ?*std.os.windows.OVERLAPPED = undefined;
646 return while (true) switch (std.os.windows.GetQueuedCompletionStatus(
647 w.os.io_cp.?,
648 &bytes_transferred,
649 &key,
650 &overlapped_ptr,
651 @bitCast(timeout.to_i32_ms()),
652 )) {
653 .Normal => {
654 if (bytes_transferred == 0)
655 break error.Unexpected;
656 break if (try Os.markDirtySteps(w, gpa, w.os.dir_list.items[key]))
657 .dirty
658 else
659 .clean;
660 },
661 .Timeout => break .timeout,
662 // This status is issued because CancelIo was called, skip and try again.
663 .Cancelled => continue,
664 else => break error.Unexpected,
665 };
666 },
361667 else => @compileError("unimplemented"),
362668 }
363669}
lib/std/os/windows.zig+27-9
......@@ -452,7 +452,7 @@ pub fn WaitForSingleObjectEx(handle: HANDLE, milliseconds: DWORD, alertable: boo
452452}
453453
454454pub fn WaitForMultipleObjectsEx(handles: []const HANDLE, waitAll: bool, milliseconds: DWORD, alertable: bool) !u32 {
455 assert(handles.len < MAXIMUM_WAIT_OBJECTS);
455 assert(handles.len > 0 and handles.len <= MAXIMUM_WAIT_OBJECTS);
456456 const nCount: DWORD = @as(DWORD, @intCast(handles.len));
457457 switch (kernel32.WaitForMultipleObjectsEx(
458458 nCount,
......@@ -516,6 +516,7 @@ pub const GetQueuedCompletionStatusResult = enum {
516516 Aborted,
517517 Cancelled,
518518 EOF,
519 Timeout,
519520};
520521
521522pub fn GetQueuedCompletionStatus(
......@@ -536,6 +537,7 @@ pub fn GetQueuedCompletionStatus(
536537 .ABANDONED_WAIT_0 => return GetQueuedCompletionStatusResult.Aborted,
537538 .OPERATION_ABORTED => return GetQueuedCompletionStatusResult.Cancelled,
538539 .HANDLE_EOF => return GetQueuedCompletionStatusResult.EOF,
540 .WAIT_TIMEOUT => return GetQueuedCompletionStatusResult.Timeout,
539541 else => |err| {
540542 if (std.debug.runtime_safety) {
541543 @setEvalBranchQuota(2500);
......@@ -3206,6 +3208,15 @@ pub const FILE_FS_DEVICE_INFORMATION = extern struct {
32063208 Characteristics: ULONG,
32073209};
32083210
3211pub const FILE_FS_VOLUME_INFORMATION = extern struct {
3212 VolumeCreationTime: LARGE_INTEGER,
3213 VolumeSerialNumber: ULONG,
3214 VolumeLabelLength: ULONG,
3215 SupportsObjects: BOOLEAN,
3216 // Flexible array member
3217 VolumeLabel: [1]WCHAR,
3218};
3219
32093220pub const FS_INFORMATION_CLASS = enum(c_int) {
32103221 FileFsVolumeInformation = 1,
32113222 FileFsLabelInformation,
......@@ -3927,14 +3938,21 @@ pub const FILE_ACTION_RENAMED_NEW_NAME = 0x00000005;
39273938
39283939pub const LPOVERLAPPED_COMPLETION_ROUTINE = ?*const fn (DWORD, DWORD, *OVERLAPPED) callconv(.C) void;
39293940
3930pub const FILE_NOTIFY_CHANGE_CREATION = 64;
3931pub const FILE_NOTIFY_CHANGE_SIZE = 8;
3932pub const FILE_NOTIFY_CHANGE_SECURITY = 256;
3933pub const FILE_NOTIFY_CHANGE_LAST_ACCESS = 32;
3934pub const FILE_NOTIFY_CHANGE_LAST_WRITE = 16;
3935pub const FILE_NOTIFY_CHANGE_DIR_NAME = 2;
3936pub const FILE_NOTIFY_CHANGE_FILE_NAME = 1;
3937pub const FILE_NOTIFY_CHANGE_ATTRIBUTES = 4;
3941pub const FileNotifyChangeFilter = packed struct(DWORD) {
3942 file_name: bool = false,
3943 dir_name: bool = false,
3944 attributes: bool = false,
3945 size: bool = false,
3946 last_write: bool = false,
3947 last_access: bool = false,
3948 creation: bool = false,
3949 ea: bool = false,
3950 security: bool = false,
3951 stream_name: bool = false,
3952 stream_size: bool = false,
3953 stream_write: bool = false,
3954 _pad: u20 = 0,
3955};
39383956
39393957pub const CONSOLE_SCREEN_BUFFER_INFO = extern struct {
39403958 dwSize: COORD,
lib/std/os/windows/kernel32.zig+12
......@@ -45,6 +45,18 @@ const WINAPI = windows.WINAPI;
4545const WORD = windows.WORD;
4646
4747// I/O - Filesystem
48
49pub extern "kernel32" fn ReadDirectoryChangesW(
50 hDirectory: windows.HANDLE,
51 lpBuffer: [*]align(@alignOf(windows.FILE_NOTIFY_INFORMATION)) u8,
52 nBufferLength: windows.DWORD,
53 bWatchSubtree: windows.BOOL,
54 dwNotifyFilter: windows.FileNotifyChangeFilter,
55 lpBytesReturned: ?*windows.DWORD,
56 lpOverlapped: ?*windows.OVERLAPPED,
57 lpCompletionRoutine: windows.LPOVERLAPPED_COMPLETION_ROUTINE,
58) callconv(windows.WINAPI) windows.BOOL;
59
4860// TODO: Wrapper around NtCancelIoFile.
4961pub extern "kernel32" fn CancelIo(
5062 hFile: HANDLE,