authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-08 23:42:20-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-12 00:14:07-07:00
logbbd90a562efd6e802ed41df2649a05fad763a4de
treec5276879ad6ade65af80393019ac649f9e6c1734
parentdeea36250ffe458d92b32b1ad090b8a958ba8082

build runner: implement --watch (work-in-progress)

I'm still learning how the fanotify API works but I think after playing with it in this commit, I finally know how to implement it, at least on Linux. This commit does not accomplish the goal but I want to take the code in a different direction and still be able to reference this point in time by viewing a source control diff. I think the move is going to be saving the file_handle for the parent directory, which combined with the dirent names is how we can correlate the events back to the Step instances that have registered file system inputs. I predict this to be similar to implementations on other operating systems.

4 files changed, 407 insertions(+), 144 deletions(-)

lib/compiler/build_runner.zig+352-66
...@@ -8,6 +8,7 @@ const process = std.process;...@@ -8,6 +8,7 @@ const process = std.process;
8const ArrayList = std.ArrayList;8const ArrayList = std.ArrayList;
9const File = std.fs.File;9const File = std.fs.File;
10const Step = std.Build.Step;10const Step = std.Build.Step;
11const Allocator = std.mem.Allocator;
1112
12pub const root = @import("@build");13pub const root = @import("@build");
13pub const dependencies = @import("@dependencies");14pub const dependencies = @import("@dependencies");
...@@ -74,7 +75,6 @@ pub fn main() !void {...@@ -74,7 +75,6 @@ pub fn main() !void {
74 .query = .{},75 .query = .{},
75 .result = try std.zig.system.resolveTargetQuery(.{}),76 .result = try std.zig.system.resolveTargetQuery(.{}),
76 },77 },
77 .watch = null,
78 };78 };
7979
80 graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() });80 graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() });
...@@ -105,6 +105,7 @@ pub fn main() !void {...@@ -105,6 +105,7 @@ pub fn main() !void {
105 var help_menu = false;105 var help_menu = false;
106 var steps_menu = false;106 var steps_menu = false;
107 var output_tmp_nonce: ?[16]u8 = null;107 var output_tmp_nonce: ?[16]u8 = null;
108 var watch = false;
108109
109 while (nextArg(args, &arg_idx)) |arg| {110 while (nextArg(args, &arg_idx)) |arg| {
110 if (mem.startsWith(u8, arg, "-Z")) {111 if (mem.startsWith(u8, arg, "-Z")) {
...@@ -229,9 +230,7 @@ pub fn main() !void {...@@ -229,9 +230,7 @@ pub fn main() !void {
229 } else if (mem.eql(u8, arg, "--prominent-compile-errors")) {230 } else if (mem.eql(u8, arg, "--prominent-compile-errors")) {
230 prominent_compile_errors = true;231 prominent_compile_errors = true;
231 } else if (mem.eql(u8, arg, "--watch")) {232 } else if (mem.eql(u8, arg, "--watch")) {
232 const watch = try arena.create(std.Build.Watch);233 watch = true;
233 watch.* = std.Build.Watch.init;
234 graph.watch = watch;
235 } else if (mem.eql(u8, arg, "-fwine")) {234 } else if (mem.eql(u8, arg, "-fwine")) {
236 builder.enable_wine = true;235 builder.enable_wine = true;
237 } else if (mem.eql(u8, arg, "-fno-wine")) {236 } else if (mem.eql(u8, arg, "-fno-wine")) {
...@@ -297,6 +296,7 @@ pub fn main() !void {...@@ -297,6 +296,7 @@ pub fn main() !void {
297 const main_progress_node = std.Progress.start(.{296 const main_progress_node = std.Progress.start(.{
298 .disable_printing = (color == .off),297 .disable_printing = (color == .off),
299 });298 });
299 defer main_progress_node.end();
300300
301 builder.debug_log_scopes = debug_log_scopes.items;301 builder.debug_log_scopes = debug_log_scopes.items;
302 builder.resolveInstallPrefix(install_prefix, dir_list);302 builder.resolveInstallPrefix(install_prefix, dir_list);
...@@ -345,13 +345,16 @@ pub fn main() !void {...@@ -345,13 +345,16 @@ pub fn main() !void {
345 .max_rss_is_default = false,345 .max_rss_is_default = false,
346 .max_rss_mutex = .{},346 .max_rss_mutex = .{},
347 .skip_oom_steps = skip_oom_steps,347 .skip_oom_steps = skip_oom_steps,
348 .watch = watch,
348 .memory_blocked_steps = std.ArrayList(*Step).init(arena),349 .memory_blocked_steps = std.ArrayList(*Step).init(arena),
350 .step_stack = .{},
349 .prominent_compile_errors = prominent_compile_errors,351 .prominent_compile_errors = prominent_compile_errors,
350352
351 .claimed_rss = 0,353 .claimed_rss = 0,
352 .summary = summary orelse if (graph.watch != null) .new else .failures,354 .summary = summary orelse if (watch) .new else .failures,
353 .ttyconf = ttyconf,355 .ttyconf = ttyconf,
354 .stderr = stderr,356 .stderr = stderr,
357 .thread_pool = undefined,
355 };358 };
356359
357 if (run.max_rss == 0) {360 if (run.max_rss == 0) {
...@@ -359,30 +362,311 @@ pub fn main() !void {...@@ -359,30 +362,311 @@ pub fn main() !void {
359 run.max_rss_is_default = true;362 run.max_rss_is_default = true;
360 }363 }
361364
362 runStepNames(365 const gpa = arena;
363 arena,366 prepare(gpa, arena, builder, targets.items, &run, seed) catch |err| switch (err) {
364 builder,367 error.UncleanExit => process.exit(1),
365 targets.items,
366 main_progress_node,
367 thread_pool_options,
368 &run,
369 seed,
370 ) catch |err| switch (err) {
371 error.UncleanExit => {
372 if (graph.watch == null)
373 process.exit(1);
374 },
375 else => return err,368 else => return err,
376 };369 };
370
371 var w = Watch.init;
372 if (watch) {
373 w.fan_fd = try std.posix.fanotify_init(.{
374 .CLASS = .NOTIF,
375 .CLOEXEC = true,
376 .NONBLOCK = true,
377 .REPORT_NAME = true,
378 .REPORT_DIR_FID = true,
379 .REPORT_FID = true,
380 .REPORT_TARGET_FID = true,
381 }, 0);
382 }
383
384 try run.thread_pool.init(thread_pool_options);
385 defer run.thread_pool.deinit();
386
387 rebuild: while (true) {
388 runStepNames(
389 gpa,
390 builder,
391 targets.items,
392 main_progress_node,
393 &run,
394 ) catch |err| switch (err) {
395 error.UncleanExit => {
396 assert(!run.watch);
397 process.exit(1);
398 },
399 else => return err,
400 };
401 if (!watch) return cleanExit();
402
403 // Clear all file handles.
404 for (w.handle_table.keys(), w.handle_table.values()) |lfh, *step_set| {
405 lfh.destroy(gpa);
406 step_set.clearAndFree(gpa);
407 }
408 w.handle_table.clearRetainingCapacity();
409
410 // Add missing marks and note persisted ones.
411 for (run.step_stack.keys()) |step| {
412 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
413 {
414 const gop = try w.dir_table.getOrPut(gpa, path);
415 gop.value_ptr.* = w.generation;
416 if (!gop.found_existing) {
417 try std.posix.fanotify_mark(w.fan_fd, .{
418 .ADD = true,
419 .ONLYDIR = true,
420 }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOpt());
421 }
422 }
423 for (files.items) |basename| {
424 const file_handle = try Watch.getFileHandle(gpa, path, basename);
425 std.debug.print("watching file_handle '{}{s}' = {}\n", .{
426 path, basename, std.fmt.fmtSliceHexLower(file_handle.slice()),
427 });
428 const gop = try w.handle_table.getOrPut(gpa, file_handle);
429 if (!gop.found_existing) gop.value_ptr.* = .{};
430 try gop.value_ptr.put(gpa, step, {});
431 }
432 }
433 }
434
435 {
436 // Remove marks for files that are no longer inputs.
437 var i: usize = 0;
438 while (i < w.dir_table.entries.len) {
439 const generations = w.dir_table.values();
440 if (generations[i] == w.generation) {
441 i += 1;
442 continue;
443 }
444
445 const path = w.dir_table.keys()[i];
446
447 try std.posix.fanotify_mark(w.fan_fd, .{
448 .REMOVE = true,
449 .ONLYDIR = true,
450 }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOpt());
451
452 w.dir_table.swapRemoveAt(i);
453 }
454 w.generation +%= 1;
455 }
456
457 // Wait until a file system notification arrives. Read all such events
458 // until the buffer is empty. Then wait for a debounce interval, resetting
459 // if any more events come in. After the debounce interval has passed,
460 // trigger a rebuild on all steps with modified inputs, as well as their
461 // recursive dependants.
462 const debounce_interval_ms = 10;
463 var poll_fds: [1]std.posix.pollfd = .{
464 .{
465 .fd = w.fan_fd,
466 .events = std.posix.POLL.IN,
467 .revents = undefined,
468 },
469 };
470 var caption_buf: [40]u8 = undefined;
471 const caption = std.fmt.bufPrint(&caption_buf, "Watching {d} Directories", .{
472 w.dir_table.entries.len,
473 }) catch &caption_buf;
474 var debouncing_node = main_progress_node.start(caption, 0);
475 var debouncing = false;
476 while (true) {
477 const timeout: i32 = if (debouncing) debounce_interval_ms else -1;
478 const events_len = try std.posix.poll(&poll_fds, timeout);
479 if (events_len == 0) {
480 debouncing_node.end();
481 continue :rebuild;
482 }
483 if (try markDirtySteps(&w)) {
484 if (!debouncing) {
485 debouncing = true;
486 debouncing_node.end();
487 debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0);
488 }
489 }
490 }
491 }
377}492}
378493
494fn markDirtySteps(w: *Watch) !bool {
495 const fanotify = std.os.linux.fanotify;
496 const M = fanotify.event_metadata;
497 var events_buf: [256 + 4096]u8 = undefined;
498 var any_dirty = false;
499 while (true) {
500 var len = std.posix.read(w.fan_fd, &events_buf) catch |err| switch (err) {
501 error.WouldBlock => return any_dirty,
502 else => |e| return e,
503 };
504 //std.debug.dump_hex(events_buf[0..len]);
505 var meta: [*]align(1) M = @ptrCast(&events_buf);
506 while (len >= @sizeOf(M) and meta[0].event_len >= @sizeOf(M) and meta[0].event_len <= len) : ({
507 len -= meta[0].event_len;
508 meta = @ptrCast(@as([*]u8, @ptrCast(meta)) + meta[0].event_len);
509 }) {
510 assert(meta[0].vers == M.VERSION);
511 std.debug.print("meta = {any}\n", .{meta[0]});
512 const fid: *align(1) fanotify.event_info_fid = @ptrCast(meta + 1);
513 switch (fid.hdr.info_type) {
514 .DFID_NAME => {
515 const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle);
516 const file_name_z: [*:0]u8 = @ptrCast((&file_handle.f_handle).ptr + file_handle.handle_bytes);
517 const file_name = mem.span(file_name_z);
518 std.debug.print("DFID_NAME file_handle = {any}, found: '{s}'\n", .{ file_handle.*, file_name });
519 const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle };
520 if (w.handle_table.get(lfh)) |step_set| {
521 for (step_set.keys()) |step| {
522 std.debug.print("DFID_NAME marking step '{s}' dirty\n", .{step.name});
523 step.state = .precheck_done;
524 any_dirty = true;
525 }
526 } else {
527 std.debug.print("DFID_NAME changed file did not match any steps: '{}'\n", .{
528 std.fmt.fmtSliceHexLower(lfh.slice()),
529 });
530 }
531 },
532 .FID => {
533 const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle);
534 const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle };
535 if (w.handle_table.get(lfh)) |step_set| {
536 for (step_set.keys()) |step| {
537 std.debug.print("FID marking step '{s}' dirty\n", .{step.name});
538 step.state = .precheck_done;
539 any_dirty = true;
540 }
541 } else {
542 std.debug.print("FID changed file did not match any steps: '{}'\n", .{
543 std.fmt.fmtSliceHexLower(lfh.slice()),
544 });
545 }
546 },
547 .DFID => {
548 const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle);
549 const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle };
550 if (w.handle_table.get(lfh)) |step_set| {
551 for (step_set.keys()) |step| {
552 std.debug.print("DFID marking step '{s}' dirty\n", .{step.name});
553 step.state = .precheck_done;
554 any_dirty = true;
555 }
556 } else {
557 std.debug.print("DFID changed file did not match any steps\n", .{});
558 }
559 },
560 else => |t| {
561 std.debug.panic("TODO: received event type '{s}'", .{@tagName(t)});
562 },
563 }
564 }
565 }
566}
567
568const Watch = struct {
569 dir_table: DirTable,
570 handle_table: HandleTable,
571 fan_fd: std.posix.fd_t,
572 generation: u8,
573
574 const fan_mask: std.os.linux.fanotify.MarkMask = .{
575 .CLOSE_WRITE = true,
576 .DELETE = true,
577 .MOVED_FROM = true,
578 .MOVED_TO = true,
579 .EVENT_ON_CHILD = true,
580 };
581
582 const init: Watch = .{
583 .dir_table = .{},
584 .handle_table = .{},
585 .fan_fd = -1,
586 .generation = 0,
587 };
588
589 /// Key is the directory to watch which contains one or more files we are
590 /// interested in noticing changes to.
591 ///
592 /// Value is generation.
593 const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, u8, Cache.Path.TableAdapter, false);
594
595 const HandleTable = std.ArrayHashMapUnmanaged(LinuxFileHandle, StepSet, LinuxFileHandle.Adapter, false);
596 const StepSet = std.AutoArrayHashMapUnmanaged(*Step, void);
597
598 const Hash = std.hash.Wyhash;
599 const Cache = std.Build.Cache;
600
601 const LinuxFileHandle = struct {
602 handle: *align(1) std.os.linux.file_handle,
603
604 fn clone(lfh: LinuxFileHandle, gpa: Allocator) Allocator.Error!LinuxFileHandle {
605 const bytes = lfh.slice();
606 const new_ptr = try gpa.alignedAlloc(
607 u8,
608 @alignOf(std.os.linux.file_handle),
609 @sizeOf(std.os.linux.file_handle) + bytes.len,
610 );
611 const new_header: *std.os.linux.file_handle = @ptrCast(new_ptr);
612 new_header.* = lfh.handle.*;
613 const new: LinuxFileHandle = .{ .handle = new_header };
614 @memcpy(new.slice(), lfh.slice());
615 return new;
616 }
617
618 fn destroy(lfh: LinuxFileHandle, gpa: Allocator) void {
619 const ptr: [*]u8 = @ptrCast(lfh.handle);
620 const allocated_slice = ptr[0 .. @sizeOf(std.os.linux.file_handle) + lfh.handle.handle_bytes];
621 return gpa.free(allocated_slice);
622 }
623
624 fn slice(lfh: LinuxFileHandle) []u8 {
625 const ptr: [*]u8 = &lfh.handle.f_handle;
626 return ptr[0..lfh.handle.handle_bytes];
627 }
628
629 const Adapter = struct {
630 pub fn hash(self: Adapter, a: LinuxFileHandle) u32 {
631 _ = self;
632 const unsigned_type: u32 = @bitCast(a.handle.handle_type);
633 return @truncate(Hash.hash(unsigned_type, a.slice()));
634 }
635 pub fn eql(self: Adapter, a: LinuxFileHandle, b: LinuxFileHandle, b_index: usize) bool {
636 _ = self;
637 _ = b_index;
638 return a.handle.handle_type == b.handle.handle_type and mem.eql(u8, a.slice(), b.slice());
639 }
640 };
641 };
642
643 fn getFileHandle(gpa: Allocator, path: std.Build.Cache.Path, basename: []const u8) !LinuxFileHandle {
644 var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined;
645 var mount_id: i32 = undefined;
646 var buf: [std.fs.max_path_bytes]u8 = undefined;
647 const joined_path = if (path.sub_path.len == 0) basename else path: {
648 break :path std.fmt.bufPrint(&buf, "{s}" ++ std.fs.path.sep_str ++ "{s}", .{
649 path.sub_path, basename,
650 }) catch return error.NameTooLong;
651 };
652 const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer);
653 stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle);
654 try std.posix.name_to_handle_at(path.root_dir.handle.fd, joined_path, stack_ptr, &mount_id, 0);
655 const stack_lfh: LinuxFileHandle = .{ .handle = stack_ptr };
656 return stack_lfh.clone(gpa);
657 }
658};
659
379const Run = struct {660const Run = struct {
380 max_rss: u64,661 max_rss: u64,
381 max_rss_is_default: bool,662 max_rss_is_default: bool,
382 max_rss_mutex: std.Thread.Mutex,663 max_rss_mutex: std.Thread.Mutex,
383 skip_oom_steps: bool,664 skip_oom_steps: bool,
665 watch: bool,
384 memory_blocked_steps: std.ArrayList(*Step),666 memory_blocked_steps: std.ArrayList(*Step),
667 step_stack: std.AutoArrayHashMapUnmanaged(*Step, void),
385 prominent_compile_errors: bool,668 prominent_compile_errors: bool,
669 thread_pool: std.Thread.Pool,
386670
387 claimed_rss: usize,671 claimed_rss: usize,
388 summary: Summary,672 summary: Summary,
...@@ -390,18 +674,15 @@ const Run = struct {...@@ -390,18 +674,15 @@ const Run = struct {
390 stderr: File,674 stderr: File,
391};675};
392676
393fn runStepNames(677fn prepare(
394 arena: std.mem.Allocator,678 gpa: Allocator,
679 arena: Allocator,
395 b: *std.Build,680 b: *std.Build,
396 step_names: []const []const u8,681 step_names: []const []const u8,
397 parent_prog_node: std.Progress.Node,
398 thread_pool_options: std.Thread.Pool.Options,
399 run: *Run,682 run: *Run,
400 seed: u32,683 seed: u32,
401) !void {684) !void {
402 const gpa = b.allocator;685 const step_stack = &run.step_stack;
403 var step_stack: std.AutoArrayHashMapUnmanaged(*Step, void) = .{};
404 defer step_stack.deinit(gpa);
405686
406 if (step_names.len == 0) {687 if (step_names.len == 0) {
407 try step_stack.put(gpa, b.default_step, {});688 try step_stack.put(gpa, b.default_step, {});
...@@ -424,7 +705,7 @@ fn runStepNames(...@@ -424,7 +705,7 @@ fn runStepNames(
424 rand.shuffle(*Step, starting_steps);705 rand.shuffle(*Step, starting_steps);
425706
426 for (starting_steps) |s| {707 for (starting_steps) |s| {
427 constructGraphAndCheckForDependencyLoop(b, s, &step_stack, rand) catch |err| switch (err) {708 constructGraphAndCheckForDependencyLoop(b, s, &run.step_stack, rand) catch |err| switch (err) {
428 error.DependencyLoopDetected => return uncleanExit(),709 error.DependencyLoopDetected => return uncleanExit(),
429 else => |e| return e,710 else => |e| return e,
430 };711 };
...@@ -453,14 +734,19 @@ fn runStepNames(...@@ -453,14 +734,19 @@ fn runStepNames(
453 return uncleanExit();734 return uncleanExit();
454 }735 }
455 }736 }
737}
456738
457 var thread_pool: std.Thread.Pool = undefined;739fn runStepNames(
458 try thread_pool.init(thread_pool_options);740 gpa: Allocator,
459 defer thread_pool.deinit();741 b: *std.Build,
742 step_names: []const []const u8,
743 parent_prog_node: std.Progress.Node,
744 run: *Run,
745) !void {
746 const step_stack = &run.step_stack;
747 const thread_pool = &run.thread_pool;
460748
461 {749 {
462 defer parent_prog_node.end();
463
464 const step_prog = parent_prog_node.start("steps", step_stack.count());750 const step_prog = parent_prog_node.start("steps", step_stack.count());
465 defer step_prog.end();751 defer step_prog.end();
466752
...@@ -476,7 +762,7 @@ fn runStepNames(...@@ -476,7 +762,7 @@ fn runStepNames(
476 if (step.state == .skipped_oom) continue;762 if (step.state == .skipped_oom) continue;
477763
478 thread_pool.spawnWg(&wait_group, workerMakeOneStep, .{764 thread_pool.spawnWg(&wait_group, workerMakeOneStep, .{
479 &wait_group, &thread_pool, b, step, step_prog, run,765 &wait_group, b, step, step_prog, run,
480 });766 });
481 }767 }
482 }768 }
...@@ -493,8 +779,6 @@ fn runStepNames(...@@ -493,8 +779,6 @@ fn runStepNames(
493 var failure_count: usize = 0;779 var failure_count: usize = 0;
494 var pending_count: usize = 0;780 var pending_count: usize = 0;
495 var total_compile_errors: usize = 0;781 var total_compile_errors: usize = 0;
496 var compile_error_steps: std.ArrayListUnmanaged(*Step) = .{};
497 defer compile_error_steps.deinit(gpa);
498782
499 for (step_stack.keys()) |s| {783 for (step_stack.keys()) |s| {
500 test_fail_count += s.test_results.fail_count;784 test_fail_count += s.test_results.fail_count;
...@@ -524,7 +808,6 @@ fn runStepNames(...@@ -524,7 +808,6 @@ fn runStepNames(
524 const compile_errors_len = s.result_error_bundle.errorMessageCount();808 const compile_errors_len = s.result_error_bundle.errorMessageCount();
525 if (compile_errors_len > 0) {809 if (compile_errors_len > 0) {
526 total_compile_errors += compile_errors_len;810 total_compile_errors += compile_errors_len;
527 try compile_error_steps.append(gpa, s);
528 }811 }
529 },812 },
530 }813 }
...@@ -537,8 +820,8 @@ fn runStepNames(...@@ -537,8 +820,8 @@ fn runStepNames(
537 else => false,820 else => false,
538 };821 };
539 if (failure_count == 0 and failures_only) {822 if (failure_count == 0 and failures_only) {
540 if (b.graph.watch != null) return;823 if (!run.watch) cleanExit();
541 return cleanExit();824 return;
542 }825 }
543826
544 const ttyconf = run.ttyconf;827 const ttyconf = run.ttyconf;
...@@ -561,10 +844,13 @@ fn runStepNames(...@@ -561,10 +844,13 @@ fn runStepNames(
561 stderr.writeAll("\n") catch {};844 stderr.writeAll("\n") catch {};
562845
563 // Print a fancy tree with build results.846 // Print a fancy tree with build results.
847 var step_stack_copy = try step_stack.clone(gpa);
848 defer step_stack_copy.deinit(gpa);
849
564 var print_node: PrintNode = .{ .parent = null };850 var print_node: PrintNode = .{ .parent = null };
565 if (step_names.len == 0) {851 if (step_names.len == 0) {
566 print_node.last = true;852 print_node.last = true;
567 printTreeStep(b, b.default_step, run, stderr, ttyconf, &print_node, &step_stack) catch {};853 printTreeStep(b, b.default_step, run, stderr, ttyconf, &print_node, &step_stack_copy) catch {};
568 } else {854 } else {
569 const last_index = if (run.summary == .all) b.top_level_steps.count() else blk: {855 const last_index = if (run.summary == .all) b.top_level_steps.count() else blk: {
570 var i: usize = step_names.len;856 var i: usize = step_names.len;
...@@ -583,44 +869,34 @@ fn runStepNames(...@@ -583,44 +869,34 @@ fn runStepNames(
583 for (step_names, 0..) |step_name, i| {869 for (step_names, 0..) |step_name, i| {
584 const tls = b.top_level_steps.get(step_name).?;870 const tls = b.top_level_steps.get(step_name).?;
585 print_node.last = i + 1 == last_index;871 print_node.last = i + 1 == last_index;
586 printTreeStep(b, &tls.step, run, stderr, ttyconf, &print_node, &step_stack) catch {};872 printTreeStep(b, &tls.step, run, stderr, ttyconf, &print_node, &step_stack_copy) catch {};
587 }873 }
588 }874 }
589 }875 }
590876
591 if (failure_count == 0) {877 if (failure_count == 0) {
592 if (b.graph.watch != null) return;878 if (!run.watch) cleanExit();
593 return cleanExit();879 return;
594 }880 }
595881
596 // Finally, render compile errors at the bottom of the terminal.882 // Finally, render compile errors at the bottom of the terminal.
597 // We use a separate compile_error_steps array list because step_stack is destructively
598 // mutated in printTreeStep above.
599 if (run.prominent_compile_errors and total_compile_errors > 0) {883 if (run.prominent_compile_errors and total_compile_errors > 0) {
600 for (compile_error_steps.items) |s| {884 for (step_stack.keys()) |s| {
601 if (s.result_error_bundle.errorMessageCount() > 0) {885 if (s.result_error_bundle.errorMessageCount() > 0) {
602 s.result_error_bundle.renderToStdErr(renderOptions(ttyconf));886 s.result_error_bundle.renderToStdErr(renderOptions(ttyconf));
603 }887 }
604 }888 }
605889
606 if (b.graph.watch != null) return uncleanExit();890 if (!run.watch) {
607891 // Signal to parent process that we have printed compile errors. The
608 // Signal to parent process that we have printed compile errors. The892 // parent process may choose to omit the "following command failed"
609 // parent process may choose to omit the "following command failed"893 // line in this case.
610 // line in this case.894 std.debug.lockStdErr();
611 process.exit(2);895 process.exit(2);
896 }
612 }897 }
613898
614 return uncleanExit();899 if (!run.watch) return uncleanExit();
615}
616
617fn uncleanExit() error{UncleanExit}!void {
618 if (builtin.mode == .Debug) {
619 return error.UncleanExit;
620 } else {
621 std.debug.lockStdErr();
622 process.exit(1);
623 }
624}900}
625901
626const PrintNode = struct {902const PrintNode = struct {
...@@ -912,12 +1188,13 @@ fn constructGraphAndCheckForDependencyLoop(...@@ -912,12 +1188,13 @@ fn constructGraphAndCheckForDependencyLoop(
9121188
913fn workerMakeOneStep(1189fn workerMakeOneStep(
914 wg: *std.Thread.WaitGroup,1190 wg: *std.Thread.WaitGroup,
915 thread_pool: *std.Thread.Pool,
916 b: *std.Build,1191 b: *std.Build,
917 s: *Step,1192 s: *Step,
918 prog_node: std.Progress.Node,1193 prog_node: std.Progress.Node,
919 run: *Run,1194 run: *Run,
920) void {1195) void {
1196 const thread_pool = &run.thread_pool;
1197
921 // First, check the conditions for running this step. If they are not met,1198 // First, check the conditions for running this step. If they are not met,
922 // then we return without doing the step, relying on another worker to1199 // then we return without doing the step, relying on another worker to
923 // queue this step up again when dependencies are met.1200 // queue this step up again when dependencies are met.
...@@ -997,7 +1274,7 @@ fn workerMakeOneStep(...@@ -997,7 +1274,7 @@ fn workerMakeOneStep(
997 // Successful completion of a step, so we queue up its dependants as well.1274 // Successful completion of a step, so we queue up its dependants as well.
998 for (s.dependants.items) |dep| {1275 for (s.dependants.items) |dep| {
999 thread_pool.spawnWg(wg, workerMakeOneStep, .{1276 thread_pool.spawnWg(wg, workerMakeOneStep, .{
1000 wg, thread_pool, b, dep, prog_node, run,1277 wg, b, dep, prog_node, run,
1001 });1278 });
1002 }1279 }
1003 }1280 }
...@@ -1022,7 +1299,7 @@ fn workerMakeOneStep(...@@ -1022,7 +1299,7 @@ fn workerMakeOneStep(
1022 remaining -= dep.max_rss;1299 remaining -= dep.max_rss;
10231300
1024 thread_pool.spawnWg(wg, workerMakeOneStep, .{1301 thread_pool.spawnWg(wg, workerMakeOneStep, .{
1025 wg, thread_pool, b, dep, prog_node, run,1302 wg, b, dep, prog_node, run,
1026 });1303 });
1027 } else {1304 } else {
1028 run.memory_blocked_steps.items[i] = dep;1305 run.memory_blocked_steps.items[i] = dep;
...@@ -1242,13 +1519,22 @@ fn argsRest(args: [][:0]const u8, idx: usize) ?[][:0]const u8 {...@@ -1242,13 +1519,22 @@ fn argsRest(args: [][:0]const u8, idx: usize) ?[][:0]const u8 {
1242 return args[idx..];1519 return args[idx..];
1243}1520}
12441521
1522/// Perhaps in the future there could be an Advanced Options flag such as
1523/// --debug-build-runner-leaks which would make this function return instead of
1524/// calling exit.
1245fn cleanExit() void {1525fn cleanExit() void {
1246 // Perhaps in the future there could be an Advanced Options flag such as1526 std.debug.lockStdErr();
1247 // --debug-build-runner-leaks which would make this function return instead
1248 // of calling exit.
1249 process.exit(0);1527 process.exit(0);
1250}1528}
12511529
1530/// Perhaps in the future there could be an Advanced Options flag such as
1531/// --debug-build-runner-leaks which would make this function return instead of
1532/// calling exit.
1533fn uncleanExit() error{UncleanExit} {
1534 std.debug.lockStdErr();
1535 process.exit(1);
1536}
1537
1252const Color = std.zig.Color;1538const Color = std.zig.Color;
1253const Summary = enum { all, new, failures, none };1539const Summary = enum { all, new, failures, none };
12541540
lib/std/Build.zig-55
...@@ -119,61 +119,6 @@ pub const Graph = struct {...@@ -119,61 +119,6 @@ pub const Graph = struct {
119 needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .{},119 needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .{},
120 /// Information about the native target. Computed before build() is invoked.120 /// Information about the native target. Computed before build() is invoked.
121 host: ResolvedTarget,121 host: ResolvedTarget,
122 /// When `--watch` is provided, collects the set of files that should be
123 /// watched and the state to required to poll the system for changes.
124 watch: ?*Watch,
125};
126
127pub const Watch = struct {
128 table: Table,
129
130 pub const init: Watch = .{
131 .table = .{},
132 };
133
134 /// Key is the directory to watch which contains one or more files we are
135 /// interested in noticing changes to.
136 pub const Table = std.ArrayHashMapUnmanaged(Cache.Path, ReactionSet, TableContext, false);
137
138 const Hash = std.hash.Wyhash;
139
140 pub const TableContext = struct {
141 pub fn hash(self: TableContext, a: Cache.Path) u32 {
142 _ = self;
143 const seed: u32 = @bitCast(a.root_dir.handle.fd);
144 return @truncate(Hash.hash(seed, a.sub_path));
145 }
146 pub fn eql(self: TableContext, a: Cache.Path, b: Cache.Path, b_index: usize) bool {
147 _ = self;
148 _ = b_index;
149 return a.eql(b);
150 }
151 };
152
153 pub const ReactionSet = std.ArrayHashMapUnmanaged(Match, void, Match.Context, false);
154
155 pub const Match = struct {
156 /// Relative to the watched directory, the file path that triggers this
157 /// match.
158 basename: []const u8,
159 /// The step to re-run when file corresponding to `basename` is changed.
160 step: *Step,
161
162 pub const Context = struct {
163 pub fn hash(self: Context, a: Match) u32 {
164 _ = self;
165 var hasher = Hash.init(0);
166 std.hash.autoHash(&hasher, a.step);
167 hasher.update(a.basename);
168 return @truncate(hasher.final());
169 }
170 pub fn eql(self: Context, a: Match, b: Match, b_index: usize) bool {
171 _ = self;
172 _ = b_index;
173 return a.step == b.step and mem.eql(u8, a.basename, b.basename);
174 }
175 };
176 };
177};122};
178123
179const AvailableDeps = []const struct { []const u8, []const u8 };124const AvailableDeps = []const struct { []const u8, []const u8 };
lib/std/Build/Step.zig+51-22
...@@ -7,6 +7,16 @@ dependencies: std.ArrayList(*Step),...@@ -7,6 +7,16 @@ dependencies: std.ArrayList(*Step),
7/// This field is empty during execution of the user's build script, and7/// This field is empty during execution of the user's build script, and
8/// then populated during dependency loop checking in the build runner.8/// then populated during dependency loop checking in the build runner.
9dependants: std.ArrayListUnmanaged(*Step),9dependants: std.ArrayListUnmanaged(*Step),
10/// Collects the set of files that retrigger this step to run.
11///
12/// This is used by the build system's implementation of `--watch` but it can
13/// also be potentially useful for IDEs to know what effects editing a
14/// particular file has.
15///
16/// Populated within `make`. Implementation may choose to clear and repopulate,
17/// retain previous value, or update.
18inputs: Inputs,
19
10state: State,20state: State,
11/// Set this field to declare an upper bound on the amount of bytes of memory it will21/// Set this field to declare an upper bound on the amount of bytes of memory it will
12/// take to run the step. Zero means no limit.22/// take to run the step. Zero means no limit.
...@@ -63,6 +73,11 @@ pub const MakeFn = *const fn (step: *Step, prog_node: std.Progress.Node) anyerro...@@ -63,6 +73,11 @@ pub const MakeFn = *const fn (step: *Step, prog_node: std.Progress.Node) anyerro
63pub const State = enum {73pub const State = enum {
64 precheck_unstarted,74 precheck_unstarted,
65 precheck_started,75 precheck_started,
76 /// This is also used to indicate "dirty" steps that have been modified
77 /// after a previous build completed, in which case, the step may or may
78 /// not have been completed before. Either way, one or more of its direct
79 /// file system inputs have been modified, meaning that the step needs to
80 /// be re-evaluated.
66 precheck_done,81 precheck_done,
67 running,82 running,
68 dependency_failure,83 dependency_failure,
...@@ -134,6 +149,26 @@ pub const Run = @import("Step/Run.zig");...@@ -134,6 +149,26 @@ pub const Run = @import("Step/Run.zig");
134pub const TranslateC = @import("Step/TranslateC.zig");149pub const TranslateC = @import("Step/TranslateC.zig");
135pub const WriteFile = @import("Step/WriteFile.zig");150pub const WriteFile = @import("Step/WriteFile.zig");
136151
152pub const Inputs = struct {
153 table: Table,
154
155 pub const init: Inputs = .{
156 .table = .{},
157 };
158
159 pub const Table = std.ArrayHashMapUnmanaged(Build.Cache.Path, Files, Build.Cache.Path.TableAdapter, false);
160 pub const Files = std.ArrayListUnmanaged([]const u8);
161
162 pub fn populated(inputs: *Inputs) bool {
163 return inputs.table.count() != 0;
164 }
165
166 pub fn clear(inputs: *Inputs, gpa: Allocator) void {
167 for (inputs.table.values()) |*files| files.deinit(gpa);
168 inputs.table.clearRetainingCapacity();
169 }
170};
171
137pub const StepOptions = struct {172pub const StepOptions = struct {
138 id: Id,173 id: Id,
139 name: []const u8,174 name: []const u8,
...@@ -153,6 +188,7 @@ pub fn init(options: StepOptions) Step {...@@ -153,6 +188,7 @@ pub fn init(options: StepOptions) Step {
153 .makeFn = options.makeFn,188 .makeFn = options.makeFn,
154 .dependencies = std.ArrayList(*Step).init(arena),189 .dependencies = std.ArrayList(*Step).init(arena),
155 .dependants = .{},190 .dependants = .{},
191 .inputs = Inputs.init,
156 .state = .precheck_unstarted,192 .state = .precheck_unstarted,
157 .max_rss = options.max_rss,193 .max_rss = options.max_rss,
158 .debug_stack_trace = blk: {194 .debug_stack_trace = blk: {
...@@ -542,19 +578,19 @@ pub fn allocPrintCmd2(...@@ -542,19 +578,19 @@ pub fn allocPrintCmd2(
542 return buf.toOwnedSlice(arena);578 return buf.toOwnedSlice(arena);
543}579}
544580
545pub fn cacheHit(s: *Step, man: *std.Build.Cache.Manifest) !bool {581pub fn cacheHit(s: *Step, man: *Build.Cache.Manifest) !bool {
546 s.result_cached = man.hit() catch |err| return failWithCacheError(s, man, err);582 s.result_cached = man.hit() catch |err| return failWithCacheError(s, man, err);
547 return s.result_cached;583 return s.result_cached;
548}584}
549585
550fn failWithCacheError(s: *Step, man: *const std.Build.Cache.Manifest, err: anyerror) anyerror {586fn failWithCacheError(s: *Step, man: *const Build.Cache.Manifest, err: anyerror) anyerror {
551 const i = man.failed_file_index orelse return err;587 const i = man.failed_file_index orelse return err;
552 const pp = man.files.keys()[i].prefixed_path;588 const pp = man.files.keys()[i].prefixed_path;
553 const prefix = man.cache.prefixes()[pp.prefix].path orelse "";589 const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
554 return s.fail("{s}: {s}/{s}", .{ @errorName(err), prefix, pp.sub_path });590 return s.fail("{s}: {s}/{s}", .{ @errorName(err), prefix, pp.sub_path });
555}591}
556592
557pub fn writeManifest(s: *Step, man: *std.Build.Cache.Manifest) !void {593pub fn writeManifest(s: *Step, man: *Build.Cache.Manifest) !void {
558 if (s.test_results.isSuccess()) {594 if (s.test_results.isSuccess()) {
559 man.writeManifest() catch |err| {595 man.writeManifest() catch |err| {
560 try s.addError("unable to write cache manifest: {s}", .{@errorName(err)});596 try s.addError("unable to write cache manifest: {s}", .{@errorName(err)});
...@@ -568,44 +604,37 @@ fn oom(err: anytype) noreturn {...@@ -568,44 +604,37 @@ fn oom(err: anytype) noreturn {
568 }604 }
569}605}
570606
571pub fn addWatchInput(step: *Step, lazy_path: std.Build.LazyPath) void {607pub fn addWatchInput(step: *Step, lazy_path: Build.LazyPath) void {
572 errdefer |err| oom(err);608 errdefer |err| oom(err);
573 const w = step.owner.graph.watch orelse return;
574 switch (lazy_path) {609 switch (lazy_path) {
575 .src_path => |src_path| try addWatchInputFromBuilder(step, w, src_path.owner, src_path.sub_path),610 .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path),
576 .dependency => |d| try addWatchInputFromBuilder(step, w, d.dependency.builder, d.sub_path),611 .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path),
577 .cwd_relative => |path_string| {612 .cwd_relative => |path_string| {
578 try addWatchInputFromPath(w, .{613 try addWatchInputFromPath(step, .{
579 .root_dir = .{614 .root_dir = .{
580 .path = null,615 .path = null,
581 .handle = std.fs.cwd(),616 .handle = std.fs.cwd(),
582 },617 },
583 .sub_path = std.fs.path.dirname(path_string) orelse "",618 .sub_path = std.fs.path.dirname(path_string) orelse "",
584 }, .{619 }, std.fs.path.basename(path_string));
585 .step = step,
586 .basename = std.fs.path.basename(path_string),
587 });
588 },620 },
589 // Nothing to watch because this dependency edge is modeled instead via `dependants`.621 // Nothing to watch because this dependency edge is modeled instead via `dependants`.
590 .generated => {},622 .generated => {},
591 }623 }
592}624}
593625
594fn addWatchInputFromBuilder(step: *Step, w: *std.Build.Watch, builder: *std.Build, sub_path: []const u8) !void {626fn addWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void {
595 return addWatchInputFromPath(w, .{627 return addWatchInputFromPath(step, .{
596 .root_dir = builder.build_root,628 .root_dir = builder.build_root,
597 .sub_path = std.fs.path.dirname(sub_path) orelse "",629 .sub_path = std.fs.path.dirname(sub_path) orelse "",
598 }, .{630 }, std.fs.path.basename(sub_path));
599 .step = step,
600 .basename = std.fs.path.basename(sub_path),
601 });
602}631}
603632
604fn addWatchInputFromPath(w: *std.Build.Watch, path: std.Build.Cache.Path, match: std.Build.Watch.Match) !void {633fn addWatchInputFromPath(step: *Step, path: Build.Cache.Path, basename: []const u8) !void {
605 const gpa = match.step.owner.allocator;634 const gpa = step.owner.allocator;
606 const gop = try w.table.getOrPut(gpa, path);635 const gop = try step.inputs.table.getOrPut(gpa, path);
607 if (!gop.found_existing) gop.value_ptr.* = .{};636 if (!gop.found_existing) gop.value_ptr.* = .{};
608 try gop.value_ptr.put(gpa, match, {});637 try gop.value_ptr.append(gpa, basename);
609}638}
610639
611test {640test {
lib/std/Build/Step/InstallFile.zig+4-1
...@@ -39,7 +39,10 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {...@@ -39,7 +39,10 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
39 _ = prog_node;39 _ = prog_node;
40 const b = step.owner;40 const b = step.owner;
41 const install_file: *InstallFile = @fieldParentPtr("step", step);41 const install_file: *InstallFile = @fieldParentPtr("step", step);
42 step.addWatchInput(install_file.source);42
43 // Inputs never change when re-running `make`.
44 if (!step.inputs.populated()) step.addWatchInput(install_file.source);
45
43 const full_src_path = install_file.source.getPath2(b, step);46 const full_src_path = install_file.source.getPath2(b, step);
44 const full_dest_path = b.getInstallPath(install_file.dir, install_file.dest_rel_path);47 const full_dest_path = b.getInstallPath(install_file.dir, install_file.dest_rel_path);
45 const cwd = std.fs.cwd();48 const cwd = std.fs.cwd();