authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-18 13:12:18-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-05-25 18:54:33-07:00
log3262698fb171fb2ed33cf6786c0573e922ce9a80
treef1c5d8a56c7daf90b9d9f9f6bf11e937d092fa79
parentafd7507a197d516c7240f92fc19e4f43adb0b79c

make runner: execute step graph


5 files changed, 514 insertions(+), 422 deletions(-)

lib/compiler/maker.zig+411-351
...@@ -512,12 +512,10 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -512,12 +512,10 @@ pub fn main(init: process.Init.Minimal) !void {
512 else => |e| return e,512 else => |e| return e,
513 };513 };
514514
515 if (true) @panic("TODO");
516
517 var w: Watch = w: {515 var w: Watch = w: {
518 if (!watch) break :w undefined;516 if (!watch) break :w undefined;
519 if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag});517 if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag});
520 break :w try .init(graph.cache.cwd);518 break :w try .init(graph.cache.cwd, &scanned_config.configuration, run.steps);
521 };519 };
522520
523 const now = Io.Clock.Timestamp.now(io, .awake);521 const now = Io.Clock.Timestamp.now(io, .awake);
...@@ -532,6 +530,7 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -532,6 +530,7 @@ pub fn main(init: process.Init.Minimal) !void {
532 .watch = watch,530 .watch = watch,
533 .listen_address = listen_address,531 .listen_address = listen_address,
534 .base_timestamp = now,532 .base_timestamp = now,
533 .configuration = &scanned_config.configuration,
535 });534 });
536 } else null;535 } else null;
537536
...@@ -546,7 +545,7 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -546,7 +545,7 @@ pub fn main(init: process.Init.Minimal) !void {
546 }) {545 }) {
547 if (run.web_server) |*ws| ws.startBuild();546 if (run.web_server) |*ws| ws.startBuild();
548547
549 try run.makeStepNames(step_names, main_progress_node, fuzz);548 try run.makeStepNames(step_names.items, main_progress_node, fuzz);
550549
551 if (run.web_server) |*web_server| {550 if (run.web_server) |*web_server| {
552 if (fuzz) |mode| if (mode != .forever) fatal(551 if (fuzz) |mode| if (mode != .forever) fatal(
...@@ -558,12 +557,15 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -558,12 +557,15 @@ pub fn main(init: process.Init.Minimal) !void {
558 }557 }
559558
560 if (run.web_server) |*ws| {559 if (run.web_server) |*ws| {
560 const c = &scanned_config.configuration;
561 assert(!watch); // fatal error after CLI parsing561 assert(!watch); // fatal error after CLI parsing
562 while (true) switch (try ws.wait()) {562 while (true) switch (try ws.wait()) {
563 .rebuild => {563 .rebuild => {
564 for (run.step_stack.keys()) |step| {564 for (run.step_stack.keys()) |step_index| {
565 const step = run.stepByIndex(step_index);
565 step.state = .precheck_done;566 step.state = .precheck_done;
566 step.pending_deps = @intCast(step.dependencies.items.len);567 const deps = step_index.ptr(c).deps.slice(c);
568 step.pending_deps = @intCast(deps.len);
567 step.reset(gpa);569 step.reset(gpa);
568 }570 }
569 continue :rebuild;571 continue :rebuild;
...@@ -583,7 +585,7 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -583,7 +585,7 @@ pub fn main(init: process.Init.Minimal) !void {
583 // recursive dependants.585 // recursive dependants.
584 var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined;586 var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined;
585 const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{587 const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{
586 w.dir_count, countSubProcesses(run.step_stack.keys()),588 w.dir_count, countSubProcesses(run.steps, run.step_stack.keys()),
587 }) catch &caption_buf;589 }) catch &caption_buf;
588 var debouncing_node = main_progress_node.start(caption, 0);590 var debouncing_node = main_progress_node.start(caption, 0);
589 var in_debounce = false;591 var in_debounce = false;
...@@ -591,7 +593,7 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -591,7 +593,7 @@ pub fn main(init: process.Init.Minimal) !void {
591 .timeout => {593 .timeout => {
592 assert(in_debounce);594 assert(in_debounce);
593 debouncing_node.end();595 debouncing_node.end();
594 markFailedStepsDirty(gpa, run.step_stack.keys());596 markFailedStepsDirty(gpa, run.steps, run.step_stack.keys());
595 continue :rebuild;597 continue :rebuild;
596 },598 },
597 .dirty => if (!in_debounce) {599 .dirty => if (!in_debounce) {
...@@ -604,22 +606,29 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -604,22 +606,29 @@ pub fn main(init: process.Init.Minimal) !void {
604 }606 }
605}607}
606608
607fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void {609fn markFailedStepsDirty(gpa: Allocator, make_steps: []Step, all_steps: []const Configuration.Step.Index) void {
608 for (all_steps) |step| switch (step.state) {610 for (all_steps) |step_index| {
609 .dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa),611 const step = &make_steps[@intFromEnum(step_index)];
610 else => continue,612 switch (step.state) {
611 };613 .dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa),
614 else => continue,
615 }
616 }
612 // Now that all dirty steps have been found, the remaining steps that617 // Now that all dirty steps have been found, the remaining steps that
613 // succeeded from last run shall be marked "cached".618 // succeeded from last run shall be marked "cached".
614 for (all_steps) |step| switch (step.state) {619 for (all_steps) |step_index| {
615 .success => step.result_cached = true,620 const step = &make_steps[@intFromEnum(step_index)];
616 else => continue,621 switch (step.state) {
617 };622 .success => step.result_cached = true,
623 else => continue,
624 }
625 }
618}626}
619627
620fn countSubProcesses(all_steps: []const *Step) usize {628fn countSubProcesses(make_steps: []Step, all_steps: []const Configuration.Step.Index) usize {
621 var count: usize = 0;629 var count: usize = 0;
622 for (all_steps) |s| {630 for (all_steps) |step_index| {
631 const s = &make_steps[@intFromEnum(step_index)];
623 count += @intFromBool(s.getZigProcess() != null);632 count += @intFromBool(s.getZigProcess() != null);
624 }633 }
625 return count;634 return count;
...@@ -707,7 +716,7 @@ const Run = struct {...@@ -707,7 +716,7 @@ const Run = struct {
707 if (run.skip_oom_steps) {716 if (run.skip_oom_steps) {
708 make_step.state = .skipped_oom;717 make_step.state = .skipped_oom;
709 for (make_step.dependants.items) |dependant| {718 for (make_step.dependants.items) |dependant| {
710 dependant.pending_deps -= 1;719 run.stepByIndex(dependant).pending_deps -= 1;
711 }720 }
712 } else {721 } else {
713 log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{722 log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{
...@@ -742,17 +751,19 @@ const Run = struct {...@@ -742,17 +751,19 @@ const Run = struct {
742 const io = graph.io;751 const io = graph.io;
743 const step_stack = &run.step_stack;752 const step_stack = &run.step_stack;
744 const top_level_steps = &run.scanned_config.top_level_steps;753 const top_level_steps = &run.scanned_config.top_level_steps;
754 const c = &run.scanned_config.configuration;
745755
746 {756 {
747 // Collect the initial set of tasks (those with no outstanding dependencies) into a buffer,757 // Collect the initial set of tasks (those with no outstanding dependencies) into a buffer,
748 // then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking758 // then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking
749 // a step is initial when it actually became ready due to an earlier initial step.759 // a step is initial when it actually became ready due to an earlier initial step.
750 var initial_set: std.ArrayList(*Step) = .empty;760 var initial_set: std.ArrayList(Configuration.Step.Index) = .empty;
751 defer initial_set.deinit(gpa);761 defer initial_set.deinit(gpa);
752 try initial_set.ensureUnusedCapacity(gpa, step_stack.count());762 try initial_set.ensureUnusedCapacity(gpa, step_stack.count());
753 for (step_stack.keys()) |s| {763 for (step_stack.keys()) |step_index| {
764 const s = run.stepByIndex(step_index);
754 if (s.state == .precheck_done and s.pending_deps == 0) {765 if (s.state == .precheck_done and s.pending_deps == 0) {
755 initial_set.appendAssumeCapacity(s);766 initial_set.appendAssumeCapacity(step_index);
756 }767 }
757 }768 }
758769
...@@ -762,7 +773,7 @@ const Run = struct {...@@ -762,7 +773,7 @@ const Run = struct {
762 var group: Io.Group = .init;773 var group: Io.Group = .init;
763 defer group.cancel(io);774 defer group.cancel(io);
764 // Start working on all of the initial steps...775 // Start working on all of the initial steps...
765 for (initial_set.items) |s| try stepReady(&group, s, step_prog, run);776 for (initial_set.items) |step_index| try stepReady(run, &group, step_index, step_prog);
766 // ...and `makeStep` will trigger every other step when their last dependency finishes.777 // ...and `makeStep` will trigger every other step when their last dependency finishes.
767 try group.await(io);778 try group.await(io);
768 }779 }
...@@ -786,16 +797,17 @@ const Run = struct {...@@ -786,16 +797,17 @@ const Run = struct {
786 var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() });797 var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() });
787 defer cleanup_task.await(io);798 defer cleanup_task.await(io);
788799
789 for (step_stack.keys()) |s| {800 for (step_stack.keys()) |step_index| {
790 test_pass_count += s.test_results.passCount();801 const make_step = run.stepByIndex(step_index);
791 test_skip_count += s.test_results.skip_count;802 test_pass_count += make_step.test_results.passCount();
792 test_fail_count += s.test_results.fail_count;803 test_skip_count += make_step.test_results.skip_count;
793 test_crash_count += s.test_results.crash_count;804 test_fail_count += make_step.test_results.fail_count;
794 test_timeout_count += s.test_results.timeout_count;805 test_crash_count += make_step.test_results.crash_count;
806 test_timeout_count += make_step.test_results.timeout_count;
795807
796 test_count += s.test_results.test_count;808 test_count += make_step.test_results.test_count;
797809
798 switch (s.state) {810 switch (make_step.state) {
799 .precheck_unstarted => unreachable,811 .precheck_unstarted => unreachable,
800 .precheck_started => unreachable,812 .precheck_started => unreachable,
801 .precheck_done => unreachable,813 .precheck_done => unreachable,
...@@ -804,7 +816,7 @@ const Run = struct {...@@ -804,7 +816,7 @@ const Run = struct {
804 .skipped, .skipped_oom => skipped_count += 1,816 .skipped, .skipped_oom => skipped_count += 1,
805 .failure => {817 .failure => {
806 failure_count += 1;818 failure_count += 1;
807 const compile_errors_len = s.result_error_bundle.errorMessageCount();819 const compile_errors_len = make_step.result_error_bundle.errorMessageCount();
808 if (compile_errors_len > 0) {820 if (compile_errors_len > 0) {
809 total_compile_errors += compile_errors_len;821 total_compile_errors += compile_errors_len;
810 }822 }
...@@ -925,13 +937,17 @@ const Run = struct {...@@ -925,13 +937,17 @@ const Run = struct {
925 var print_node: PrintNode = .{ .parent = null };937 var print_node: PrintNode = .{ .parent = null };
926 if (step_names.len == 0) {938 if (step_names.len == 0) {
927 print_node.last = true;939 print_node.last = true;
928 printTreeStep(graph, graph.default_step, run, t, &print_node, &step_stack_copy) catch {};940 printTreeStep(run, c.default_step, t, &print_node, &step_stack_copy) catch |err| switch (err) {
941 error.Canceled => |e| return e,
942 else => {},
943 };
929 } else {944 } else {
930 const last_index = if (run.summary == .all) top_level_steps.count() else blk: {945 const last_index = if (run.summary == .all) top_level_steps.count() else blk: {
931 var i: usize = step_names.len;946 var i: usize = step_names.len;
932 while (i > 0) {947 while (i > 0) {
933 i -= 1;948 i -= 1;
934 const step = top_level_steps.get(step_names[i]).?.step;949 const step_index = top_level_steps.get(step_names[i]).?;
950 const step = run.stepByIndex(step_index);
935 const found = switch (run.summary) {951 const found = switch (run.summary) {
936 .all, .line, .none => unreachable,952 .all, .line, .none => unreachable,
937 .failures => step.state != .success,953 .failures => step.state != .success,
...@@ -942,9 +958,12 @@ const Run = struct {...@@ -942,9 +958,12 @@ const Run = struct {
942 break :blk top_level_steps.count();958 break :blk top_level_steps.count();
943 };959 };
944 for (step_names, 0..) |step_name, i| {960 for (step_names, 0..) |step_name, i| {
945 const tls = top_level_steps.get(step_name).?;961 const step_index = top_level_steps.get(step_name).?;
946 print_node.last = i + 1 == last_index;962 print_node.last = i + 1 == last_index;
947 printTreeStep(graph, &tls.step, run, t, &print_node, &step_stack_copy) catch {};963 printTreeStep(run, step_index, t, &print_node, &step_stack_copy) catch |err| switch (err) {
964 error.Canceled => |e| return e,
965 else => {},
966 };
948 }967 }
949 }968 }
950 w.writeByte('\n') catch {};969 w.writeByte('\n') catch {};
...@@ -964,120 +983,331 @@ const Run = struct {...@@ -964,120 +983,331 @@ const Run = struct {
964 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};983 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
965 process.exit(code);984 process.exit(code);
966 }985 }
967};
968986
969const PrintNode = struct {987 fn stepReady(
970 parent: ?*PrintNode,988 run: *Run,
971 last: bool = false,989 group: *Io.Group,
972};990 step_index: Configuration.Step.Index,
991 root_prog_node: std.Progress.Node,
992 ) Io.Cancelable!void {
993 const graph = run.graph;
994 const io = graph.io;
995 const c = &run.scanned_config.configuration;
996 const max_rss = step_index.ptr(c).max_rss.toBytes();
997 if (max_rss != 0) {
998 try run.max_rss_mutex.lock(io);
999 defer run.max_rss_mutex.unlock(io);
1000 if (run.available_rss < max_rss) {
1001 // Running this step right now could possibly exceed the allotted RSS.
1002 run.memory_blocked_steps.append(run.gpa, step_index) catch
1003 @panic("TODO eliminate memory allocation here");
1004 return;
1005 }
1006 run.available_rss -= max_rss;
1007 }
1008 group.async(io, makeStep, .{ run, group, step_index, root_prog_node });
1009 }
9731010
974fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void {1011 /// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready
975 const parent = node.parent orelse return;1012 /// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must
976 const writer = stderr.writer;1013 /// have already subtracted this value from `run.available_rss`. This function will release the RSS
977 if (parent.parent == null) return;1014 /// claim (i.e. add `s.max_rss` back into `run.available_rss`) and queue any viable memory-blocked
978 try printPrefix(parent, stderr);1015 /// steps after "make" completes for `s`.
979 if (parent.last) {1016 fn makeStep(
980 try writer.writeAll(" ");1017 run: *Run,
981 } else {1018 group: *Io.Group,
982 try writer.writeAll(switch (stderr.mode) {1019 step_index: Configuration.Step.Index,
983 .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │1020 root_prog_node: std.Progress.Node,
984 else => "| ",1021 ) Io.Cancelable!void {
985 });1022 const graph = run.graph;
1023 const io = graph.io;
1024 const gpa = run.gpa;
1025 const c = &run.scanned_config.configuration;
1026 const conf_step = step_index.ptr(c);
1027 const step_name = conf_step.name.slice(c);
1028 const deps = conf_step.deps.slice(c);
1029 const make_step = run.stepByIndex(step_index);
1030
1031 {
1032 const step_prog_node = root_prog_node.start(step_name, 0);
1033 defer step_prog_node.end();
1034
1035 if (run.web_server) |*ws| ws.updateStepStatus(step_index, .wip);
1036
1037 const new_state: Step.State = for (deps) |dep_index| {
1038 const dep_make_step = run.stepByIndex(dep_index);
1039 switch (@atomicLoad(Step.State, &dep_make_step.state, .monotonic)) {
1040 .precheck_unstarted => unreachable,
1041 .precheck_started => unreachable,
1042 .precheck_done => unreachable,
1043
1044 .failure,
1045 .dependency_failure,
1046 .skipped_oom,
1047 => break .dependency_failure,
1048
1049 .success, .skipped => {},
1050 }
1051 } else if (make_step.make(.{
1052 .progress_node = step_prog_node,
1053 .watch = run.watch,
1054 .web_server = if (run.web_server) |*ws| ws else null,
1055 .unit_test_timeout_ns = run.unit_test_timeout_ns,
1056 .gpa = gpa,
1057 })) state: {
1058 break :state .success;
1059 } else |err| switch (err) {
1060 error.MakeFailed => .failure,
1061 error.MakeSkipped => .skipped,
1062 };
1063
1064 @atomicStore(Step.State, &make_step.state, new_state, .monotonic);
1065
1066 switch (new_state) {
1067 .precheck_unstarted => unreachable,
1068 .precheck_started => unreachable,
1069 .precheck_done => unreachable,
1070
1071 .failure,
1072 .dependency_failure,
1073 .skipped_oom,
1074 => {
1075 if (run.web_server) |*ws| ws.updateStepStatus(step_index, .failure);
1076 std.Progress.setStatus(.failure_working);
1077 },
1078
1079 .success,
1080 .skipped,
1081 => {
1082 if (run.web_server) |*ws| ws.updateStepStatus(step_index, .success);
1083 },
1084 }
1085 }
1086
1087 // No matter the result, we want to display error/warning messages.
1088 if (make_step.result_error_bundle.errorMessageCount() > 0 or
1089 make_step.result_error_msgs.items.len > 0 or
1090 make_step.result_stderr.len > 0)
1091 {
1092 const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode);
1093 defer io.unlockStderr();
1094 printErrorMessages(gpa, c, run.steps, step_index, .{}, stderr.terminal(), run.error_style, run.multiline_errors) catch |err| switch (err) {
1095 error.Canceled => |e| return e,
1096 error.WriteFailed => switch (stderr.file_writer.err.?) {
1097 error.Canceled => |e| return e,
1098 else => {},
1099 },
1100 else => {},
1101 };
1102 }
1103
1104 const max_rss = conf_step.max_rss.toBytes();
1105 if (max_rss != 0) {
1106 var dispatch_set: std.ArrayList(Configuration.Step.Index) = .empty;
1107 defer dispatch_set.deinit(gpa);
1108
1109 // Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set`
1110 // as a staging buffer to avoid recursing into `makeStep` while `run.max_rss_mutex` is held.
1111 {
1112 try run.max_rss_mutex.lock(io);
1113 defer run.max_rss_mutex.unlock(io);
1114 run.available_rss += max_rss;
1115 dispatch_set.ensureUnusedCapacity(gpa, run.memory_blocked_steps.items.len) catch
1116 @panic("TODO eliminate memory allocation here");
1117 while (run.memory_blocked_steps.getLast()) |candidate_index| {
1118 const candidate_max_rss = candidate_index.ptr(c).max_rss.toBytes();
1119 if (run.available_rss < candidate_max_rss) break;
1120 assert(run.memory_blocked_steps.pop() == candidate_index);
1121 dispatch_set.appendAssumeCapacity(candidate_index);
1122 }
1123 }
1124 for (dispatch_set.items) |candidate| {
1125 group.async(io, makeStep, .{ run, group, candidate, root_prog_node });
1126 }
1127 }
1128
1129 for (make_step.dependants.items) |dependant_index| {
1130 const dependant = run.stepByIndex(dependant_index);
1131 // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0.
1132 if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) {
1133 try stepReady(run, group, dependant_index, root_prog_node);
1134 }
1135 }
986 }1136 }
987}
9881137
989fn printChildNodePrefix(stderr: Io.Terminal) !void {1138 fn printTreeStep(
990 try stderr.writer.writeAll(switch (stderr.mode) {1139 run: *const Run,
991 .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─1140 step_index: Configuration.Step.Index,
992 else => "+- ",1141 stderr: Io.Terminal,
993 });1142 parent_node: *PrintNode,
994}1143 step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void),
1144 ) !void {
1145 const writer = stderr.writer;
1146 const first = step_stack.swapRemove(step_index);
1147 const summary = run.summary;
1148 const c = &run.scanned_config.configuration;
1149 const conf_step = step_index.ptr(c);
1150 const make_step = run.stepByIndex(step_index);
1151 const skip = switch (summary) {
1152 .none, .line => unreachable,
1153 .all => false,
1154 .new => make_step.result_cached,
1155 .failures => make_step.state == .success,
1156 };
1157 if (skip) return;
1158 try printPrefix(parent_node, stderr);
9951159
996fn printStepStatus(s: *Step, stderr: Io.Terminal, run: *const Run) !void {1160 if (parent_node.parent != null) {
997 const writer = stderr.writer;1161 if (parent_node.last) {
998 switch (s.state) {1162 try printChildNodePrefix(stderr);
999 .precheck_unstarted => unreachable,1163 } else {
1000 .precheck_started => unreachable,1164 try writer.writeAll(switch (stderr.mode) {
1001 .precheck_done => unreachable,1165 .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─
10021166 else => "+- ",
1003 .dependency_failure => {1167 });
1004 try stderr.setColor(.dim);1168 }
1005 try writer.writeAll(" transitive failure\n");1169 }
1006 try stderr.setColor(.reset);
1007 },
10081170
1009 .success => {1171 if (!first) try stderr.setColor(.dim);
1010 try stderr.setColor(.green);1172
1011 if (s.result_cached) {1173 // dep_prefix omitted here because it is redundant with the tree.
1012 try writer.writeAll(" cached");1174 try writer.writeAll(conf_step.name.slice(c));
1013 } else if (s.test_results.test_count > 0) {1175
1014 const pass_count = s.test_results.passCount();1176 const deps = conf_step.deps.slice(c);
1015 assert(s.test_results.test_count == pass_count + s.test_results.skip_count);1177
1016 try writer.print(" {d} pass", .{pass_count});1178 if (first) {
1017 if (s.test_results.skip_count > 0) {1179 try printStepStatus(run, step_index, stderr);
1018 try stderr.setColor(.reset);1180
1019 try writer.writeAll(", ");1181 const last_index = if (summary == .all) deps.len -| 1 else blk: {
1020 try stderr.setColor(.yellow);1182 var i: usize = deps.len;
1021 try writer.print("{d} skip", .{s.test_results.skip_count});1183 while (i > 0) {
1184 i -= 1;
1185
1186 const dep_index = deps[i];
1187 const dep = run.stepByIndex(dep_index);
1188 const found = switch (summary) {
1189 .all, .line, .none => unreachable,
1190 .failures => dep.state != .success,
1191 .new => !dep.result_cached,
1192 };
1193 if (found) break :blk i;
1022 }1194 }
1023 try stderr.setColor(.reset);1195 break :blk deps.len -| 1;
1024 try writer.print(" ({d} total)", .{s.test_results.test_count});1196 };
1197 for (deps, 0..) |dep, i| {
1198 var print_node: PrintNode = .{
1199 .parent = parent_node,
1200 .last = i == last_index,
1201 };
1202 try printTreeStep(run, dep, stderr, &print_node, step_stack);
1203 }
1204 } else {
1205 if (deps.len == 0) {
1206 try writer.writeAll(" (reused)\n");
1025 } else {1207 } else {
1026 try writer.writeAll(" success");1208 try writer.print(" (+{d} more reused dependencies)\n", .{deps.len});
1027 }1209 }
1028 try stderr.setColor(.reset);1210 try stderr.setColor(.reset);
1029 if (s.result_duration_ns) |ns| {1211 }
1212 }
1213
1214 fn printStepStatus(run: *const Run, step_index: Configuration.Step.Index, stderr: Io.Terminal) !void {
1215 const s = run.stepByIndex(step_index);
1216 const writer = stderr.writer;
1217 switch (s.state) {
1218 .precheck_unstarted => unreachable,
1219 .precheck_started => unreachable,
1220 .precheck_done => unreachable,
1221
1222 .dependency_failure => {
1030 try stderr.setColor(.dim);1223 try stderr.setColor(.dim);
1031 if (ns >= std.time.ns_per_min) {1224 try writer.writeAll(" transitive failure\n");
1032 try writer.print(" {d}m", .{ns / std.time.ns_per_min});1225 try stderr.setColor(.reset);
1033 } else if (ns >= std.time.ns_per_s) {1226 },
1034 try writer.print(" {d}s", .{ns / std.time.ns_per_s});1227
1035 } else if (ns >= std.time.ns_per_ms) {1228 .success => {
1036 try writer.print(" {d}ms", .{ns / std.time.ns_per_ms});1229 try stderr.setColor(.green);
1037 } else if (ns >= std.time.ns_per_us) {1230 if (s.result_cached) {
1038 try writer.print(" {d}us", .{ns / std.time.ns_per_us});1231 try writer.writeAll(" cached");
1232 } else if (s.test_results.test_count > 0) {
1233 const pass_count = s.test_results.passCount();
1234 assert(s.test_results.test_count == pass_count + s.test_results.skip_count);
1235 try writer.print(" {d} pass", .{pass_count});
1236 if (s.test_results.skip_count > 0) {
1237 try stderr.setColor(.reset);
1238 try writer.writeAll(", ");
1239 try stderr.setColor(.yellow);
1240 try writer.print("{d} skip", .{s.test_results.skip_count});
1241 }
1242 try stderr.setColor(.reset);
1243 try writer.print(" ({d} total)", .{s.test_results.test_count});
1039 } else {1244 } else {
1040 try writer.print(" {d}ns", .{ns});1245 try writer.writeAll(" success");
1041 }1246 }
1042 try stderr.setColor(.reset);1247 try stderr.setColor(.reset);
1043 }1248 if (s.result_duration_ns) |ns| {
1044 if (s.result_peak_rss != 0) {1249 try stderr.setColor(.dim);
1045 const rss = s.result_peak_rss;1250 if (ns >= std.time.ns_per_min) {
1046 try stderr.setColor(.dim);1251 try writer.print(" {d}m", .{ns / std.time.ns_per_min});
1047 if (rss >= 1000_000_000) {1252 } else if (ns >= std.time.ns_per_s) {
1048 try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000});1253 try writer.print(" {d}s", .{ns / std.time.ns_per_s});
1049 } else if (rss >= 1000_000) {1254 } else if (ns >= std.time.ns_per_ms) {
1050 try writer.print(" MaxRSS:{d}M", .{rss / 1000_000});1255 try writer.print(" {d}ms", .{ns / std.time.ns_per_ms});
1051 } else if (rss >= 1000) {1256 } else if (ns >= std.time.ns_per_us) {
1052 try writer.print(" MaxRSS:{d}K", .{rss / 1000});1257 try writer.print(" {d}us", .{ns / std.time.ns_per_us});
1053 } else {1258 } else {
1054 try writer.print(" MaxRSS:{d}B", .{rss});1259 try writer.print(" {d}ns", .{ns});
1260 }
1261 try stderr.setColor(.reset);
1262 }
1263 if (s.result_peak_rss != 0) {
1264 const rss = s.result_peak_rss;
1265 try stderr.setColor(.dim);
1266 if (rss >= 1000_000_000) {
1267 try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000});
1268 } else if (rss >= 1000_000) {
1269 try writer.print(" MaxRSS:{d}M", .{rss / 1000_000});
1270 } else if (rss >= 1000) {
1271 try writer.print(" MaxRSS:{d}K", .{rss / 1000});
1272 } else {
1273 try writer.print(" MaxRSS:{d}B", .{rss});
1274 }
1275 try stderr.setColor(.reset);
1055 }1276 }
1277 try writer.writeAll("\n");
1278 },
1279 .skipped => {
1280 try stderr.setColor(.yellow);
1281 try writer.writeAll(" skipped\n");
1056 try stderr.setColor(.reset);1282 try stderr.setColor(.reset);
1057 }1283 },
1058 try writer.writeAll("\n");1284 .skipped_oom => {
1059 },1285 const c = &run.scanned_config.configuration;
1060 .skipped => {1286 const max_rss = step_index.ptr(c).max_rss.toBytes();
1061 try stderr.setColor(.yellow);1287 try stderr.setColor(.yellow);
1062 try writer.writeAll(" skipped\n");1288 try writer.writeAll(" skipped (not enough memory)");
1063 try stderr.setColor(.reset);1289 try stderr.setColor(.dim);
1064 },1290 try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{
1065 .skipped_oom => {1291 max_rss, run.available_rss,
1066 try stderr.setColor(.yellow);1292 });
1067 try writer.writeAll(" skipped (not enough memory)");1293 try stderr.setColor(.reset);
1068 try stderr.setColor(.dim);1294 },
1069 try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{ s.max_rss, run.available_rss });1295 .failure => {
1070 try stderr.setColor(.reset);1296 try printStepFailure(run.steps, step_index, stderr, false);
1071 },1297 try stderr.setColor(.reset);
1072 .failure => {1298 },
1073 try printStepFailure(s, stderr, false);1299 }
1074 try stderr.setColor(.reset);
1075 },
1076 }1300 }
1077}1301};
10781302
1079fn printStepFailure(s: *Step, stderr: Io.Terminal, dim: bool) !void {1303fn printStepFailure(
1304 make_steps: []Step,
1305 step_index: Configuration.Step.Index,
1306 stderr: Io.Terminal,
1307 dim: bool,
1308) !void {
1080 const w = stderr.writer;1309 const w = stderr.writer;
1310 const s = &make_steps[@intFromEnum(step_index)];
1081 if (s.result_error_bundle.errorMessageCount() > 0) {1311 if (s.result_error_bundle.errorMessageCount() > 0) {
1082 try stderr.setColor(.red);1312 try stderr.setColor(.red);
1083 try w.print(" {d} errors\n", .{1313 try w.print(" {d} errors\n", .{
...@@ -1160,79 +1390,33 @@ fn printStepFailure(s: *Step, stderr: Io.Terminal, dim: bool) !void {...@@ -1160,79 +1390,33 @@ fn printStepFailure(s: *Step, stderr: Io.Terminal, dim: bool) !void {
1160 }1390 }
1161}1391}
11621392
1163fn printTreeStep(1393const PrintNode = struct {
1164 graph: *Graph,1394 parent: ?*PrintNode,
1165 s: *Step,1395 last: bool = false,
1166 run: *const Run,1396};
1167 stderr: Io.Terminal,
1168 parent_node: *PrintNode,
1169 step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void),
1170) !void {
1171 const writer = stderr.writer;
1172 const first = step_stack.swapRemove(s);
1173 const summary = run.summary;
1174 const skip = switch (summary) {
1175 .none, .line => unreachable,
1176 .all => false,
1177 .new => s.result_cached,
1178 .failures => s.state == .success,
1179 };
1180 if (skip) return;
1181 try printPrefix(parent_node, stderr);
1182
1183 if (parent_node.parent != null) {
1184 if (parent_node.last) {
1185 try printChildNodePrefix(stderr);
1186 } else {
1187 try writer.writeAll(switch (stderr.mode) {
1188 .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─
1189 else => "+- ",
1190 });
1191 }
1192 }
1193
1194 if (!first) try stderr.setColor(.dim);
1195
1196 // dep_prefix omitted here because it is redundant with the tree.
1197 try writer.writeAll(s.name);
1198
1199 if (first) {
1200 try printStepStatus(s, stderr, run);
1201
1202 const last_index = if (summary == .all) s.dependencies.items.len -| 1 else blk: {
1203 var i: usize = s.dependencies.items.len;
1204 while (i > 0) {
1205 i -= 1;
12061397
1207 const step = s.dependencies.items[i];1398fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void {
1208 const found = switch (summary) {1399 const parent = node.parent orelse return;
1209 .all, .line, .none => unreachable,1400 const writer = stderr.writer;
1210 .failures => step.state != .success,1401 if (parent.parent == null) return;
1211 .new => !step.result_cached,1402 try printPrefix(parent, stderr);
1212 };1403 if (parent.last) {
1213 if (found) break :blk i;1404 try writer.writeAll(" ");
1214 }
1215 break :blk s.dependencies.items.len -| 1;
1216 };
1217 for (s.dependencies.items, 0..) |dep, i| {
1218 var print_node: PrintNode = .{
1219 .parent = parent_node,
1220 .last = i == last_index,
1221 };
1222 try printTreeStep(graph, dep, run, stderr, &print_node, step_stack);
1223 }
1224 } else {1405 } else {
1225 if (s.dependencies.items.len == 0) {1406 try writer.writeAll(switch (stderr.mode) {
1226 try writer.writeAll(" (reused)\n");1407 .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │
1227 } else {1408 else => "| ",
1228 try writer.print(" (+{d} more reused dependencies)\n", .{1409 });
1229 s.dependencies.items.len,
1230 });
1231 }
1232 try stderr.setColor(.reset);
1233 }1410 }
1234}1411}
12351412
1413fn printChildNodePrefix(stderr: Io.Terminal) !void {
1414 try stderr.writer.writeAll(switch (stderr.mode) {
1415 .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─
1416 else => "+- ",
1417 });
1418}
1419
1236/// Traverse the dependency graph depth-first and make it undirected by having1420/// Traverse the dependency graph depth-first and make it undirected by having
1237/// steps know their dependants (they only know dependencies at start).1421/// steps know their dependants (they only know dependencies at start).
1238/// Along the way, check that there is no dependency loop, and record the steps1422/// Along the way, check that there is no dependency loop, and record the steps
...@@ -1252,14 +1436,14 @@ fn constructGraphAndCheckForDependencyLoop(...@@ -1252,14 +1436,14 @@ fn constructGraphAndCheckForDependencyLoop(
1252 step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void),1436 step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void),
1253 rand: std.Random,1437 rand: std.Random,
1254) error{ DependencyLoopDetected, OutOfMemory }!void {1438) error{ DependencyLoopDetected, OutOfMemory }!void {
1255 const s: *Step = &steps[@intFromEnum(step_index)];1439 const make_step: *Step = &steps[@intFromEnum(step_index)];
1256 switch (s.state) {1440 switch (make_step.state) {
1257 .precheck_started => {1441 .precheck_started => {
1258 log.err("dependency loop detected: {s}", .{step_index.ptr(c).name.slice(c)});1442 log.err("dependency loop detected: {s}", .{step_index.ptr(c).name.slice(c)});
1259 return error.DependencyLoopDetected;1443 return error.DependencyLoopDetected;
1260 },1444 },
1261 .precheck_unstarted => {1445 .precheck_unstarted => {
1262 s.state = .precheck_started;1446 make_step.state = .precheck_started;
12631447
1264 const step = step_index.ptr(c);1448 const step = step_index.ptr(c);
1265 const dependencies = step.deps.slice(c);1449 const dependencies = step.deps.slice(c);
...@@ -1275,7 +1459,7 @@ fn constructGraphAndCheckForDependencyLoop(...@@ -1275,7 +1459,7 @@ fn constructGraphAndCheckForDependencyLoop(
1275 for (deps) |dep| {1459 for (deps) |dep| {
1276 const dep_step: *Step = &steps[@intFromEnum(dep)];1460 const dep_step: *Step = &steps[@intFromEnum(dep)];
1277 try step_stack.put(gpa, dep, {});1461 try step_stack.put(gpa, dep, {});
1278 try dep_step.dependants.append(gpa, s);1462 try dep_step.dependants.append(gpa, step_index);
1279 constructGraphAndCheckForDependencyLoop(gpa, c, steps, dep, step_stack, rand) catch |err| switch (err) {1463 constructGraphAndCheckForDependencyLoop(gpa, c, steps, dep, step_stack, rand) catch |err| switch (err) {
1280 error.DependencyLoopDetected => {1464 error.DependencyLoopDetected => {
1281 log.info("needed by: {s}", .{step_index.ptr(c).name.slice(c)});1465 log.info("needed by: {s}", .{step_index.ptr(c).name.slice(c)});
...@@ -1285,8 +1469,8 @@ fn constructGraphAndCheckForDependencyLoop(...@@ -1285,8 +1469,8 @@ fn constructGraphAndCheckForDependencyLoop(
1285 };1469 };
1286 }1470 }
12871471
1288 s.state = .precheck_done;1472 make_step.state = .precheck_done;
1289 s.pending_deps = @intCast(dependencies.len);1473 make_step.pending_deps = @intCast(dependencies.len);
1290 },1474 },
1291 .precheck_done => {},1475 .precheck_done => {},
12921476
...@@ -1299,140 +1483,11 @@ fn constructGraphAndCheckForDependencyLoop(...@@ -1299,140 +1483,11 @@ fn constructGraphAndCheckForDependencyLoop(
1299 }1483 }
1300}1484}
13011485
1302/// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready
1303/// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must
1304/// have already subtracted this value from `run.available_rss`. This function will release the RSS
1305/// claim (i.e. add `s.max_rss` back into `run.available_rss`) and queue any viable memory-blocked
1306/// steps after "make" completes for `s`.
1307fn makeStep(
1308 graph: *Graph,
1309 group: *Io.Group,
1310 s: *Step,
1311 root_prog_node: std.Progress.Node,
1312 run: *Run,
1313) Io.Cancelable!void {
1314 const io = graph.io;
1315 const gpa = run.gpa;
1316
1317 {
1318 const step_prog_node = root_prog_node.start(s.name, 0);
1319 defer step_prog_node.end();
1320
1321 if (run.web_server) |*ws| ws.updateStepStatus(s, .wip);
1322
1323 const new_state: Step.State = for (s.dependencies.items) |dep| {
1324 switch (@atomicLoad(Step.State, &dep.state, .monotonic)) {
1325 .precheck_unstarted => unreachable,
1326 .precheck_started => unreachable,
1327 .precheck_done => unreachable,
1328
1329 .failure,
1330 .dependency_failure,
1331 .skipped_oom,
1332 => break .dependency_failure,
1333
1334 .success, .skipped => {},
1335 }
1336 } else if (s.make(.{
1337 .progress_node = step_prog_node,
1338 .watch = run.watch,
1339 .web_server = if (run.web_server) |*ws| ws else null,
1340 .unit_test_timeout_ns = run.unit_test_timeout_ns,
1341 .gpa = gpa,
1342 })) state: {
1343 break :state .success;
1344 } else |err| switch (err) {
1345 error.MakeFailed => .failure,
1346 error.MakeSkipped => .skipped,
1347 };
1348
1349 @atomicStore(Step.State, &s.state, new_state, .monotonic);
1350
1351 switch (new_state) {
1352 .precheck_unstarted => unreachable,
1353 .precheck_started => unreachable,
1354 .precheck_done => unreachable,
1355
1356 .failure,
1357 .dependency_failure,
1358 .skipped_oom,
1359 => {
1360 if (run.web_server) |*ws| ws.updateStepStatus(s, .failure);
1361 std.Progress.setStatus(.failure_working);
1362 },
1363
1364 .success,
1365 .skipped,
1366 => {
1367 if (run.web_server) |*ws| ws.updateStepStatus(s, .success);
1368 },
1369 }
1370 }
1371
1372 // No matter the result, we want to display error/warning messages.
1373 if (s.result_error_bundle.errorMessageCount() > 0 or
1374 s.result_error_msgs.items.len > 0 or
1375 s.result_stderr.len > 0)
1376 {
1377 const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode);
1378 defer io.unlockStderr();
1379 printErrorMessages(gpa, s, .{}, stderr.terminal(), run.error_style, run.multiline_errors) catch {};
1380 }
1381
1382 if (s.max_rss != 0) {
1383 var dispatch_set: std.ArrayList(*Step) = .empty;
1384 defer dispatch_set.deinit(gpa);
1385
1386 // Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set`
1387 // as a staging buffer to avoid recursing into `makeStep` while `run.max_rss_mutex` is held.
1388 {
1389 try run.max_rss_mutex.lock(io);
1390 defer run.max_rss_mutex.unlock(io);
1391 run.available_rss += s.max_rss;
1392 try dispatch_set.ensureUnusedCapacity(gpa, run.memory_blocked_steps.items.len);
1393 while (run.memory_blocked_steps.getLast()) |candidate| {
1394 if (run.available_rss < candidate.max_rss) break;
1395 assert(run.memory_blocked_steps.pop() == candidate);
1396 dispatch_set.appendAssumeCapacity(candidate);
1397 }
1398 }
1399 for (dispatch_set.items) |candidate| {
1400 group.async(io, makeStep, .{ graph, group, candidate, root_prog_node, run });
1401 }
1402 }
1403
1404 for (s.dependants.items) |dependant| {
1405 // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0.
1406 if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) {
1407 try stepReady(graph, group, dependant, root_prog_node, run);
1408 }
1409 }
1410}
1411
1412fn stepReady(
1413 graph: *Graph,
1414 group: *Io.Group,
1415 s: *Step,
1416 root_prog_node: std.Progress.Node,
1417 run: *Run,
1418) !void {
1419 const io = graph.io;
1420 if (s.max_rss != 0) {
1421 try run.max_rss_mutex.lock(io);
1422 defer run.max_rss_mutex.unlock(io);
1423 if (run.available_rss < s.max_rss) {
1424 // Running this step right now could possibly exceed the allotted RSS.
1425 try run.memory_blocked_steps.append(run.gpa, s);
1426 return;
1427 }
1428 run.available_rss -= s.max_rss;
1429 }
1430 group.async(io, makeStep, .{ graph, group, s, root_prog_node, run });
1431}
1432
1433pub fn printErrorMessages(1486pub fn printErrorMessages(
1434 gpa: Allocator,1487 gpa: Allocator,
1435 failing_step: *Step,1488 c: *const Configuration,
1489 make_steps: []Step,
1490 failing_step_index: Configuration.Step.Index,
1436 options: std.zig.ErrorBundle.RenderOptions,1491 options: std.zig.ErrorBundle.RenderOptions,
1437 stderr: Io.Terminal,1492 stderr: Io.Terminal,
1438 error_style: ErrorStyle,1493 error_style: ErrorStyle,
...@@ -1442,26 +1497,28 @@ pub fn printErrorMessages(...@@ -1442,26 +1497,28 @@ pub fn printErrorMessages(
1442 if (error_style.verboseContext()) {1497 if (error_style.verboseContext()) {
1443 // Provide context for where these error messages are coming from by1498 // Provide context for where these error messages are coming from by
1444 // printing the corresponding Step subtree.1499 // printing the corresponding Step subtree.
1445 var step_stack: std.ArrayList(*Step) = .empty;1500 var step_stack: std.ArrayList(Configuration.Step.Index) = .empty;
1446 defer step_stack.deinit(gpa);1501 defer step_stack.deinit(gpa);
1447 try step_stack.append(gpa, failing_step);1502 try step_stack.append(gpa, failing_step_index);
1448 while (step_stack.items[step_stack.items.len - 1].dependants.items.len != 0) {1503 while (true) {
1449 try step_stack.append(gpa, step_stack.items[step_stack.items.len - 1].dependants.items[0]);1504 const last_step = &make_steps[@intFromEnum(step_stack.items[step_stack.items.len - 1])];
1505 if (last_step.dependants.items.len == 0) break;
1506 try step_stack.append(gpa, last_step.dependants.items[0]);
1450 }1507 }
14511508
1452 // Now, `step_stack` has the subtree that we want to print, in reverse order.1509 // Now, `step_stack` has the subtree that we want to print, in reverse order.
1453 try stderr.setColor(.dim);1510 try stderr.setColor(.dim);
1454 var indent: usize = 0;1511 var indent: usize = 0;
1455 while (step_stack.pop()) |s| : (indent += 1) {1512 while (step_stack.pop()) |step_index| : (indent += 1) {
1456 if (indent > 0) {1513 if (indent > 0) {
1457 try writer.splatByteAll(' ', (indent - 1) * 3);1514 try writer.splatByteAll(' ', (indent - 1) * 3);
1458 try printChildNodePrefix(stderr);1515 try printChildNodePrefix(stderr);
1459 }1516 }
14601517
1461 try writer.writeAll(s.name);1518 try writer.writeAll(step_index.ptr(c).name.slice(c));
14621519
1463 if (s == failing_step) {1520 if (step_index == failing_step_index) {
1464 try printStepFailure(s, stderr, true);1521 try printStepFailure(make_steps, step_index, stderr, true);
1465 } else {1522 } else {
1466 try writer.writeAll("\n");1523 try writer.writeAll("\n");
1467 }1524 }
...@@ -1470,11 +1527,13 @@ pub fn printErrorMessages(...@@ -1470,11 +1527,13 @@ pub fn printErrorMessages(
1470 } else {1527 } else {
1471 // Just print the failing step itself.1528 // Just print the failing step itself.
1472 try stderr.setColor(.dim);1529 try stderr.setColor(.dim);
1473 try writer.writeAll(failing_step.name);1530 try writer.writeAll(failing_step_index.ptr(c).name.slice(c));
1474 try printStepFailure(failing_step, stderr, true);1531 try printStepFailure(make_steps, failing_step_index, stderr, true);
1475 try stderr.setColor(.reset);1532 try stderr.setColor(.reset);
1476 }1533 }
14771534
1535 const failing_step = &make_steps[@intFromEnum(failing_step_index)];
1536
1478 if (failing_step.result_stderr.len > 0) {1537 if (failing_step.result_stderr.len > 0) {
1479 try writer.writeAll(failing_step.result_stderr);1538 try writer.writeAll(failing_step.result_stderr);
1480 if (!mem.endsWith(u8, failing_step.result_stderr, "\n")) {1539 if (!mem.endsWith(u8, failing_step.result_stderr, "\n")) {
...@@ -1567,9 +1626,10 @@ fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn {...@@ -1567,9 +1626,10 @@ fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn {
1567 fatal(f, args);1626 fatal(f, args);
1568}1627}
15691628
1570fn cleanTmpFiles(io: Io, steps: []const *Step) void {1629fn cleanTmpFiles(io: Io, steps: []const Configuration.Step.Index) void {
1571 for (steps) |step| {1630 for (steps) |step_index| {
1572 const wf = step.cast(Step.WriteFile) orelse continue;1631 if (true) @panic("TODO");
1632 const wf = step_index.cast(std.Build.Step.WriteFile) orelse continue;
1573 if (wf.mode != .tmp) continue;1633 if (wf.mode != .tmp) continue;
1574 const path = wf.generated_directory.path orelse continue;1634 const path = wf.generated_directory.path orelse continue;
1575 Io.Dir.cwd().deleteTree(io, path) catch |err| {1635 Io.Dir.cwd().deleteTree(io, path) catch |err| {
lib/compiler/maker/Fuzz.zig+31-24
...@@ -1,16 +1,16 @@...@@ -1,16 +1,16 @@
1const Fuzz = @This();1const Fuzz = @This();
22
3const std = @import("std");3const std = @import("std");
4const Io = std.Io;4const Allocator = std.mem.Allocator;
5const Build = std.Build;5const Build = std.Build;
6const Cache = std.Build.Cache;6const Cache = std.Build.Cache;
7const Step = std.Build.Step;7const Coverage = std.debug.Coverage;
8const Configuration = std.Build.Configuration;
9const Io = std.Io;
10const abi = std.Build.abi.fuzz;
8const assert = std.debug.assert;11const assert = std.debug.assert;
9const fatal = std.process.fatal;12const fatal = std.process.fatal;
10const Allocator = std.mem.Allocator;
11const log = std.log;13const log = std.log;
12const Coverage = std.debug.Coverage;
13const abi = std.Build.abi.fuzz;
1414
15const maker = @import("../maker.zig");15const maker = @import("../maker.zig");
16const WebServer = @import("WebServer.zig");16const WebServer = @import("WebServer.zig");
...@@ -20,7 +20,7 @@ io: Io,...@@ -20,7 +20,7 @@ io: Io,
20mode: Mode,20mode: Mode,
2121
22/// Allocated into `gpa`.22/// Allocated into `gpa`.
23run_steps: []const *Step.Run,23run_steps: []const Configuration.Step.Index,
2424
25group: Io.Group,25group: Io.Group,
26root_prog_node: std.Progress.Node,26root_prog_node: std.Progress.Node,
...@@ -51,7 +51,7 @@ const Msg = union(enum) {...@@ -51,7 +51,7 @@ const Msg = union(enum) {
51 unique: u64,51 unique: u64,
52 coverage: u64,52 coverage: u64,
53 },53 },
54 run: *Step.Run,54 run: Configuration.Step.Index,
55 },55 },
56 entry_point: struct {56 entry_point: struct {
57 coverage_id: u64,57 coverage_id: u64,
...@@ -78,12 +78,12 @@ const CoverageMap = struct {...@@ -78,12 +78,12 @@ const CoverageMap = struct {
78pub fn init(78pub fn init(
79 gpa: Allocator,79 gpa: Allocator,
80 io: Io,80 io: Io,
81 all_steps: []const *Build.Step,81 all_steps: []const Configuration.Step.Index,
82 root_prog_node: std.Progress.Node,82 root_prog_node: std.Progress.Node,
83 mode: Mode,83 mode: Mode,
84) error{ OutOfMemory, Canceled }!Fuzz {84) error{ OutOfMemory, Canceled }!Fuzz {
85 const run_steps: []const *Step.Run = steps: {85 const run_steps: []const Configuration.Step.Index = steps: {
86 var steps: std.ArrayList(*Step.Run) = .empty;86 var steps: std.ArrayList(Configuration.Step.Index) = .empty;
87 defer steps.deinit(gpa);87 defer steps.deinit(gpa);
88 const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0);88 const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0);
89 defer rebuild_node.end();89 defer rebuild_node.end();
...@@ -91,7 +91,8 @@ pub fn init(...@@ -91,7 +91,8 @@ pub fn init(
91 defer rebuild_group.cancel(io);91 defer rebuild_group.cancel(io);
9292
93 for (all_steps) |step| {93 for (all_steps) |step| {
94 const run = step.cast(Step.Run) orelse continue;94 if (true) @panic("TODO");
95 const run = step.cast(std.Build.Step.Run) orelse continue;
95 if (run.producer == null) continue;96 if (run.producer == null) continue;
96 if (run.fuzz_tests.items.len == 0) continue;97 if (run.fuzz_tests.items.len == 0) continue;
97 try steps.append(gpa, run);98 try steps.append(gpa, run);
...@@ -100,15 +101,16 @@ pub fn init(...@@ -100,15 +101,16 @@ pub fn init(
100101
101 if (steps.items.len == 0) fatal("no fuzz tests found", .{});102 if (steps.items.len == 0) fatal("no fuzz tests found", .{});
102 rebuild_node.setEstimatedTotalItems(steps.items.len);103 rebuild_node.setEstimatedTotalItems(steps.items.len);
103 const run_steps = try gpa.dupe(*Step.Run, steps.items);104 const run_steps = try gpa.dupe(Configuration.Step.Index, steps.items);
104 try rebuild_group.await(io);105 try rebuild_group.await(io);
105 break :steps run_steps;106 break :steps run_steps;
106 };107 };
107 errdefer gpa.free(run_steps);108 errdefer gpa.free(run_steps);
108109
109 for (run_steps) |run| {110 for (run_steps) |run_step_index| {
110 assert(run.fuzz_tests.items.len > 0);111 if (true) @panic("TODO");
111 if (run.rebuilt_executable == null)112 assert(run_step_index.fuzz_tests.items.len > 0);
113 if (run_step_index.rebuilt_executable == null)
112 fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{});114 fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{});
113 }115 }
114116
...@@ -138,6 +140,8 @@ pub fn start(fuzz: *Fuzz) void {...@@ -138,6 +140,8 @@ pub fn start(fuzz: *Fuzz) void {
138 fatal("unable to spawn coverage task: {t}", .{err});140 fatal("unable to spawn coverage task: {t}", .{err});
139 }141 }
140142
143 if (true) @panic("TODO");
144
141 for (fuzz.run_steps) |run| {145 for (fuzz.run_steps) |run| {
142 assert(run.rebuilt_executable != null);146 assert(run.rebuilt_executable != null);
143 fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run });147 fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run });
...@@ -151,14 +155,14 @@ pub fn deinit(fuzz: *Fuzz) void {...@@ -151,14 +155,14 @@ pub fn deinit(fuzz: *Fuzz) void {
151 fuzz.gpa.free(fuzz.run_steps);155 fuzz.gpa.free(fuzz.run_steps);
152}156}
153157
154fn rebuildTestsWorkerRun(run: *Step.Run, gpa: Allocator, parent_prog_node: std.Progress.Node) void {158fn rebuildTestsWorkerRun(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) void {
155 rebuildTestsWorkerRunFallible(run, gpa, parent_prog_node) catch |err| {159 rebuildTestsWorkerRunFallible(run, gpa, parent_prog_node) catch |err| {
156 const compile = run.producer.?;160 const compile = run.producer.?;
157 log.err("step '{s}': failed to rebuild in fuzz mode: {t}", .{ compile.step.name, err });161 log.err("step '{s}': failed to rebuild in fuzz mode: {t}", .{ compile.step.name, err });
158 };162 };
159}163}
160164
161fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, parent_prog_node: std.Progress.Node) !void {165fn rebuildTestsWorkerRunFallible(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) !void {
162 const graph = run.step.owner.graph;166 const graph = run.step.owner.graph;
163 const io = graph.io;167 const io = graph.io;
164 const compile = run.producer.?;168 const compile = run.producer.?;
...@@ -185,7 +189,7 @@ fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, parent_prog_nod...@@ -185,7 +189,7 @@ fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, parent_prog_nod
185 run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename);189 run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename);
186}190}
187191
188fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run) void {192fn fuzzWorkerRun(fuzz: *Fuzz, run: Configuration.Step.Index) void {
189 const owner = run.step.owner;193 const owner = run.step.owner;
190 const gpa = owner.allocator;194 const gpa = owner.allocator;
191 const graph = owner.graph;195 const graph = owner.graph;
...@@ -209,6 +213,7 @@ fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run) void {...@@ -209,6 +213,7 @@ fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run) void {
209}213}
210214
211pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void {215pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void {
216 if (true) @panic("TODO");
212 assert(fuzz.mode == .forever);217 assert(fuzz.mode == .forever);
213218
214 var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa);219 var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa);
...@@ -354,7 +359,8 @@ fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void {...@@ -354,7 +359,8 @@ fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void {
354 fuzz.msg_queue.clearRetainingCapacity();359 fuzz.msg_queue.clearRetainingCapacity();
355 }360 }
356}361}
357fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void {362fn prepareTables(fuzz: *Fuzz, run_step_index: Configuration.Step.Index, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void {
363 if (true) @panic("TODO");
358 assert(fuzz.mode == .forever);364 assert(fuzz.mode == .forever);
359 const ws = fuzz.mode.forever.ws;365 const ws = fuzz.mode.forever.ws;
360 const gpa = fuzz.gpa;366 const gpa = fuzz.gpa;
...@@ -384,8 +390,8 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO...@@ -384,8 +390,8 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
384 };390 };
385 errdefer gop.value_ptr.coverage.deinit(gpa);391 errdefer gop.value_ptr.coverage.deinit(gpa);
386392
387 const rebuilt_exe_path = run_step.rebuilt_executable.?;393 const rebuilt_exe_path = run_step_index.rebuilt_executable.?;
388 const target = run_step.producer.?.rootModuleTarget();394 const target = run_step_index.producer.?.rootModuleTarget();
389 var debug_info = std.debug.Info.load(395 var debug_info = std.debug.Info.load(
390 gpa,396 gpa,
391 io,397 io,
...@@ -395,19 +401,19 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO...@@ -395,19 +401,19 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
395 target.cpu.arch,401 target.cpu.arch,
396 ) catch |err| {402 ) catch |err| {
397 log.err("step '{s}': failed to load debug information for '{f}': {t}", .{403 log.err("step '{s}': failed to load debug information for '{f}': {t}", .{
398 run_step.step.name, rebuilt_exe_path, err,404 run_step_index.step.name, rebuilt_exe_path, err,
399 });405 });
400 return error.AlreadyReported;406 return error.AlreadyReported;
401 };407 };
402 defer debug_info.deinit(gpa);408 defer debug_info.deinit(gpa);
403409
404 const coverage_file_path: Build.Cache.Path = .{410 const coverage_file_path: Build.Cache.Path = .{
405 .root_dir = run_step.step.owner.cache_root,411 .root_dir = run_step_index.step.owner.cache_root,
406 .sub_path = "v/" ++ std.fmt.hex(coverage_id),412 .sub_path = "v/" ++ std.fmt.hex(coverage_id),
407 };413 };
408 var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| {414 var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| {
409 log.err("step '{s}': failed to load coverage file '{f}': {t}", .{415 log.err("step '{s}': failed to load coverage file '{f}': {t}", .{
410 run_step.step.name, coverage_file_path, err,416 run_step_index.step.name, coverage_file_path, err,
411 });417 });
412 return error.AlreadyReported;418 return error.AlreadyReported;
413 };419 };
...@@ -514,6 +520,7 @@ fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReporte...@@ -514,6 +520,7 @@ fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReporte
514}520}
515521
516pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void {522pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void {
523 if (true) @panic("TODO");
517 assert(fuzz.mode == .limit);524 assert(fuzz.mode == .limit);
518 const io = fuzz.io;525 const io = fuzz.io;
519526
lib/compiler/maker/Step.zig+5-1
...@@ -10,6 +10,7 @@ const Io = std.Io;...@@ -10,6 +10,7 @@ const Io = std.Io;
10const LazyPath = std.Build.Configuration.LazyPath;10const LazyPath = std.Build.Configuration.LazyPath;
11const Package = std.Build.Configuration.Package;11const Package = std.Build.Configuration.Package;
12const Path = std.Build.Cache.Path;12const Path = std.Build.Cache.Path;
13const Configuration = std.Build.Configuration;
13const assert = std.debug.assert;14const assert = std.debug.assert;
1415
15const WebServer = @import("WebServer.zig");16const WebServer = @import("WebServer.zig");
...@@ -21,7 +22,7 @@ pub const Run = void; // @import("Step/Run.zig");...@@ -21,7 +22,7 @@ pub const Run = void; // @import("Step/Run.zig");
21_: void align(std.atomic.cache_line) = {},22_: void align(std.atomic.cache_line) = {},
2223
23state: State = .precheck_unstarted,24state: State = .precheck_unstarted,
24dependants: std.ArrayList(*Step) = .empty,25dependants: std.ArrayList(Configuration.Step.Index) = .empty,
25/// Collects the set of files that retrigger this step to run.26/// Collects the set of files that retrigger this step to run.
26///27///
27/// This is used by the build system's implementation of `--watch` but it can28/// This is used by the build system's implementation of `--watch` but it can
...@@ -143,6 +144,7 @@ pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void;...@@ -143,6 +144,7 @@ pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void;
143/// have already reported the error. Otherwise, we add a simple error report144/// have already reported the error. Otherwise, we add a simple error report
144/// here.145/// here.
145pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void {146pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void {
147 if (true) @panic("TODO Step.make");
146 const arena = s.owner.allocator;148 const arena = s.owner.allocator;
147 const graph = s.owner.graph;149 const graph = s.owner.graph;
148 const io = graph.io;150 const io = graph.io;
...@@ -182,6 +184,7 @@ pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!voi...@@ -182,6 +184,7 @@ pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!voi
182/// Implementation detail of file watching. Prepares the step for being re-evaluated.184/// Implementation detail of file watching. Prepares the step for being re-evaluated.
183/// Returns `true` if the step was newly invalidated, `false` if it was already invalidated.185/// Returns `true` if the step was newly invalidated, `false` if it was already invalidated.
184pub fn invalidateResult(step: *Step, gpa: Allocator) bool {186pub fn invalidateResult(step: *Step, gpa: Allocator) bool {
187 if (true) @panic("TODO Step.invalidateResult");
185 if (step.state == .precheck_done) return false;188 if (step.state == .precheck_done) return false;
186 assert(step.pending_deps == 0);189 assert(step.pending_deps == 0);
187 step.state = .precheck_done;190 step.state = .precheck_done;
...@@ -544,6 +547,7 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*WebSer...@@ -544,6 +547,7 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*WebSer
544}547}
545548
546pub fn getZigProcess(s: *Step) ?*ZigProcess {549pub fn getZigProcess(s: *Step) ?*ZigProcess {
550 if (true) @panic("TODO getZigProcess");
547 return switch (s.id) {551 return switch (s.id) {
548 .compile => s.cast(Compile).?.zig_process,552 .compile => s.cast(Compile).?.zig_process,
549 else => null,553 else => null,
lib/compiler/maker/Watch.zig+30-22
...@@ -3,11 +3,13 @@ const builtin = @import("builtin");...@@ -3,11 +3,13 @@ const builtin = @import("builtin");
33
4const std = @import("std");4const std = @import("std");
5const Io = std.Io;5const Io = std.Io;
6const Step = std.Build.Step;
7const Allocator = std.mem.Allocator;6const Allocator = std.mem.Allocator;
8const assert = std.debug.assert;7const assert = std.debug.assert;
9const fatal = std.process.fatal;8const fatal = std.process.fatal;
9const Configuration = std.Build.Configuration;
10
10const FsEvents = @import("Watch/FsEvents.zig");11const FsEvents = @import("Watch/FsEvents.zig");
12const Step = @import("Step.zig");
1113
12os: Os,14os: Os,
13/// The number to show as the number of directories being watched.15/// The number to show as the number of directories being watched.
...@@ -16,6 +18,8 @@ dir_count: usize,...@@ -16,6 +18,8 @@ dir_count: usize,
16// They are `undefined` on implementations which do not utilize then.18// They are `undefined` on implementations which do not utilize then.
17dir_table: DirTable,19dir_table: DirTable,
18generation: Generation,20generation: Generation,
21configuration: *const Configuration,
22make_steps: []Step,
1923
20pub const have_impl = Os != void;24pub const have_impl = Os != void;
2125
...@@ -27,7 +31,7 @@ const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, void, Cache.Path.TableAda...@@ -27,7 +31,7 @@ const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, void, Cache.Path.TableAda
2731
28/// Special key of "." means any changes in this directory trigger the steps.32/// Special key of "." means any changes in this directory trigger the steps.
29const ReactionSet = std.StringArrayHashMapUnmanaged(StepSet);33const ReactionSet = std.StringArrayHashMapUnmanaged(StepSet);
30const StepSet = std.AutoArrayHashMapUnmanaged(*Step, Generation);34const StepSet = std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, Generation);
3135
32const Generation = u8;36const Generation = u8;
3337
...@@ -101,7 +105,7 @@ const Os = switch (builtin.os.tag) {...@@ -101,7 +105,7 @@ const Os = switch (builtin.os.tag) {
101 };105 };
102 };106 };
103107
104 fn init(cwd_path: []const u8) !Watch {108 fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch {
105 _ = cwd_path;109 _ = cwd_path;
106 return .{110 return .{
107 .dir_table = .{},111 .dir_table = .{},
...@@ -114,6 +118,8 @@ const Os = switch (builtin.os.tag) {...@@ -114,6 +118,8 @@ const Os = switch (builtin.os.tag) {
114 else => {},118 else => {},
115 },119 },
116 .generation = 0,120 .generation = 0,
121 .make_steps = make_steps,
122 .configuration = configuration,
117 };123 };
118 }124 }
119125
...@@ -161,20 +167,21 @@ const Os = switch (builtin.os.tag) {...@@ -161,20 +167,21 @@ const Os = switch (builtin.os.tag) {
161 const lfh: FileHandle = .{ .handle = file_handle };167 const lfh: FileHandle = .{ .handle = file_handle };
162 if (w.os.handle_table.getPtr(lfh)) |value| {168 if (w.os.handle_table.getPtr(lfh)) |value| {
163 if (value.reaction_set.getPtr(".")) |glob_set|169 if (value.reaction_set.getPtr(".")) |glob_set|
164 any_dirty = markStepSetDirty(gpa, glob_set, any_dirty);170 any_dirty = markStepSetDirty(gpa, w.make_steps, glob_set, any_dirty);
165 if (value.reaction_set.getPtr(file_name)) |step_set|171 if (value.reaction_set.getPtr(file_name)) |step_set|
166 any_dirty = markStepSetDirty(gpa, step_set, any_dirty);172 any_dirty = markStepSetDirty(gpa, w.make_steps, step_set, any_dirty);
167 }173 }
168 },174 },
169 else => |t| std.log.warn("unexpected fanotify event '{s}'", .{@tagName(t)}),175 else => |t| std.log.warn("unexpected fanotify event '{t}'", .{t}),
170 }176 }
171 }177 }
172 }178 }
173 }179 }
174180
175 fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void {181 fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
176 // Add missing marks and note persisted ones.182 // Add missing marks and note persisted ones.
177 for (steps) |step| {183 for (steps) |step_index| {
184 const step = &w.make_steps[@intFromEnum(step_index)];
178 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {185 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
179 const reaction_set = rs: {186 const reaction_set = rs: {
180 const gop = try w.dir_table.getOrPut(gpa, path);187 const gop = try w.dir_table.getOrPut(gpa, path);
...@@ -236,7 +243,7 @@ const Os = switch (builtin.os.tag) {...@@ -236,7 +243,7 @@ const Os = switch (builtin.os.tag) {
236 for (files.items) |basename| {243 for (files.items) |basename| {
237 const gop = try reaction_set.getOrPut(gpa, basename);244 const gop = try reaction_set.getOrPut(gpa, basename);
238 if (!gop.found_existing) gop.value_ptr.* = .{};245 if (!gop.found_existing) gop.value_ptr.* = .{};
239 try gop.value_ptr.put(gpa, step, w.generation);246 try gop.value_ptr.put(gpa, step_index, w.generation);
240 }247 }
241 }248 }
242 }249 }
...@@ -537,7 +544,7 @@ const Os = switch (builtin.os.tag) {...@@ -537,7 +544,7 @@ const Os = switch (builtin.os.tag) {
537 return any_dirty;544 return any_dirty;
538 }545 }
539546
540 fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void {547 fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
541 // Add missing marks and note persisted ones.548 // Add missing marks and note persisted ones.
542 for (steps) |step| {549 for (steps) |step| {
543 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {550 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
...@@ -678,7 +685,7 @@ const Os = switch (builtin.os.tag) {...@@ -678,7 +685,7 @@ const Os = switch (builtin.os.tag) {
678 };685 };
679 }686 }
680687
681 fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void {688 fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
682 const handles = &w.os.handles;689 const handles = &w.os.handles;
683 for (steps) |step| {690 for (steps) |step| {
684 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {691 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
...@@ -856,7 +863,7 @@ const Os = switch (builtin.os.tag) {...@@ -856,7 +863,7 @@ const Os = switch (builtin.os.tag) {
856 .generation = undefined,863 .generation = undefined,
857 };864 };
858 }865 }
859 fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void {866 fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
860 try w.os.fse.setPaths(gpa, steps);867 try w.os.fse.setPaths(gpa, steps);
861 w.dir_count = w.os.fse.watch_roots.len;868 w.dir_count = w.os.fse.watch_roots.len;
862 }869 }
...@@ -871,8 +878,8 @@ const Os = switch (builtin.os.tag) {...@@ -871,8 +878,8 @@ const Os = switch (builtin.os.tag) {
871 else => void,878 else => void,
872};879};
873880
874pub fn init(cwd_path: []const u8) !Watch {881pub fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch {
875 return Os.init(cwd_path);882 return Os.init(cwd_path, configuration, make_steps);
876}883}
877884
878pub const Match = struct {885pub const Match = struct {
...@@ -880,20 +887,19 @@ pub const Match = struct {...@@ -880,20 +887,19 @@ pub const Match = struct {
880 /// match.887 /// match.
881 basename: []const u8,888 basename: []const u8,
882 /// The step to re-run when file corresponding to `basename` is changed.889 /// The step to re-run when file corresponding to `basename` is changed.
883 step: *Step,890 step_index: Configuration.Step.Index,
884891
885 pub const Context = struct {892 pub const Context = struct {
886 pub fn hash(self: Context, a: Match) u32 {893 pub fn hash(self: Context, a: Match) u32 {
887 _ = self;894 _ = self;
888 var hasher = Hash.init(0);895 var hasher = Hash.init(@intFromEnum(a.step_index));
889 std.hash.autoHash(&hasher, a.step);
890 hasher.update(a.basename);896 hasher.update(a.basename);
891 return @truncate(hasher.final());897 return @truncate(hasher.final());
892 }898 }
893 pub fn eql(self: Context, a: Match, b: Match, b_index: usize) bool {899 pub fn eql(self: Context, a: Match, b: Match, b_index: usize) bool {
894 _ = self;900 _ = self;
895 _ = b_index;901 _ = b_index;
896 return a.step == b.step and std.mem.eql(u8, a.basename, b.basename);902 return a.step_index == b.step_index and std.mem.eql(u8, a.basename, b.basename);
897 }903 }
898 };904 };
899};905};
...@@ -908,22 +914,24 @@ fn markAllFilesDirty(w: *Watch, gpa: Allocator) void {...@@ -908,22 +914,24 @@ fn markAllFilesDirty(w: *Watch, gpa: Allocator) void {
908 else => item,914 else => item,
909 };915 };
910 for (reaction_set.values()) |step_set| {916 for (reaction_set.values()) |step_set| {
911 for (step_set.keys()) |step| {917 for (step_set.keys()) |step_index| {
918 const step = &w.make_steps[@intFromEnum(step_index)];
912 _ = step.invalidateResult(gpa);919 _ = step.invalidateResult(gpa);
913 }920 }
914 }921 }
915 }922 }
916}923}
917924
918fn markStepSetDirty(gpa: Allocator, step_set: *StepSet, any_dirty: bool) bool {925fn markStepSetDirty(gpa: Allocator, make_steps: []Step, step_set: *StepSet, any_dirty: bool) bool {
919 var this_any_dirty = false;926 var this_any_dirty = false;
920 for (step_set.keys()) |step| {927 for (step_set.keys()) |step_index| {
928 const step = &make_steps[@intFromEnum(step_index)];
921 if (step.invalidateResult(gpa)) this_any_dirty = true;929 if (step.invalidateResult(gpa)) this_any_dirty = true;
922 }930 }
923 return any_dirty or this_any_dirty;931 return any_dirty or this_any_dirty;
924}932}
925933
926pub fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void {934pub fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void {
927 return Os.update(w, gpa, steps);935 return Os.update(w, gpa, steps);
928}936}
929937
lib/compiler/maker/WebServer.zig+37-24
...@@ -4,8 +4,8 @@ const builtin = @import("builtin");...@@ -4,8 +4,8 @@ const builtin = @import("builtin");
44
5const std = @import("std");5const std = @import("std");
6const Allocator = std.mem.Allocator;6const Allocator = std.mem.Allocator;
7const Build = std.Build;
8const Cache = std.Build.Cache;7const Cache = std.Build.Cache;
8const Configuration = std.Build.Configuration;
9const Io = std.Io;9const Io = std.Io;
10const abi = std.Build.abi;10const abi = std.Build.abi;
11const assert = std.debug.assert;11const assert = std.debug.assert;
...@@ -15,10 +15,12 @@ const mem = std.mem;...@@ -15,10 +15,12 @@ const mem = std.mem;
15const net = std.Io.net;15const net = std.Io.net;
1616
17const Fuzz = @import("Fuzz.zig");17const Fuzz = @import("Fuzz.zig");
18const Graph = @import("Graph.zig");
19const Step = @import("Step.zig");
1820
19gpa: Allocator,21gpa: Allocator,
20graph: *const Build.Graph,22graph: *const Graph,
21all_steps: []const *Build.Step,23all_steps: []const Configuration.Step.Index,
22listen_address: net.IpAddress,24listen_address: net.IpAddress,
23root_prog_node: std.Progress.Node,25root_prog_node: std.Progress.Node,
24watch: bool,26watch: bool,
...@@ -69,12 +71,13 @@ pub fn notifyUpdate(ws: *WebServer) void {...@@ -69,12 +71,13 @@ pub fn notifyUpdate(ws: *WebServer) void {
6971
70pub const Options = struct {72pub const Options = struct {
71 gpa: Allocator,73 gpa: Allocator,
72 graph: *const std.Build.Graph,74 graph: *const Graph,
73 all_steps: []const *Build.Step,75 all_steps: []const Configuration.Step.Index,
74 root_prog_node: std.Progress.Node,76 root_prog_node: std.Progress.Node,
75 watch: bool,77 watch: bool,
76 listen_address: net.IpAddress,78 listen_address: net.IpAddress,
77 base_timestamp: Io.Clock.Timestamp,79 base_timestamp: Io.Clock.Timestamp,
80 configuration: *const Configuration,
78};81};
79pub fn init(opts: Options) WebServer {82pub fn init(opts: Options) WebServer {
80 // The upcoming `Io` interface should allow us to use `Io.async` and `Io.concurrent`83 // The upcoming `Io` interface should allow us to use `Io.async` and `Io.concurrent`
...@@ -83,19 +86,21 @@ pub fn init(opts: Options) WebServer {...@@ -83,19 +86,21 @@ pub fn init(opts: Options) WebServer {
83 assert(opts.base_timestamp.clock == base_clock);86 assert(opts.base_timestamp.clock == base_clock);
8487
85 const all_steps = opts.all_steps;88 const all_steps = opts.all_steps;
89 const c = opts.configuration;
8690
87 const step_names_trailing = opts.gpa.alloc(u8, len: {91 const step_names_trailing = opts.gpa.alloc(u8, len: {
88 var name_bytes: usize = 0;92 var name_bytes: usize = 0;
89 for (all_steps) |step| name_bytes += step.name.len;93 for (all_steps) |step_index| name_bytes += step_index.ptr(c).name.slice(c).len;
90 break :len name_bytes + all_steps.len * 4;94 break :len name_bytes + all_steps.len * 4;
91 }) catch @panic("out of memory");95 }) catch @panic("out of memory");
92 {96 {
93 const step_name_lens: []align(1) u32 = @ptrCast(step_names_trailing[0 .. all_steps.len * 4]);97 const step_name_lens: []align(1) u32 = @ptrCast(step_names_trailing[0 .. all_steps.len * 4]);
94 var idx: usize = all_steps.len * 4;98 var idx: usize = all_steps.len * 4;
95 for (all_steps, step_name_lens) |step, *name_len| {99 for (all_steps, step_name_lens) |step_index, *name_len| {
96 name_len.* = @intCast(step.name.len);100 const step_name = step_index.ptr(c).name.slice(c);
97 @memcpy(step_names_trailing[idx..][0..step.name.len], step.name);101 name_len.* = @intCast(step_name.len);
98 idx += step.name.len;102 @memcpy(step_names_trailing[idx..][0..step_name.len], step_name);
103 idx += step_name.len;
99 }104 }
100 assert(idx == step_names_trailing.len);105 assert(idx == step_names_trailing.len);
101 }106 }
...@@ -213,9 +218,14 @@ pub fn startBuild(ws: *WebServer) void {...@@ -213,9 +218,14 @@ pub fn startBuild(ws: *WebServer) void {
213 ws.notifyUpdate();218 ws.notifyUpdate();
214}219}
215220
216pub fn updateStepStatus(ws: *WebServer, step: *Build.Step, new_status: abi.StepUpdate.Status) void {221pub fn updateStepStatus(
222 ws: *WebServer,
223 step_index: Configuration.Step.Index,
224 new_status: abi.StepUpdate.Status,
225) void {
226 // TODO don't do linear search, especially in a hot loop like this
217 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {227 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
218 if (s == step) break @intCast(i);228 if (s == step_index) break @intCast(i);
219 } else unreachable;229 } else unreachable;
220 const ptr = &ws.step_status_bits[step_idx / 4];230 const ptr = &ws.step_status_bits[step_idx / 4];
221 const bit_offset: u3 = @intCast((step_idx % 4) * 2);231 const bit_offset: u3 = @intCast((step_idx % 4) * 2);
...@@ -687,7 +697,7 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim...@@ -687,7 +697,7 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim
687 if (code != 0) {697 if (code != 0) {
688 log.err(698 log.err(
689 "the following command exited with error code {d}:\n{s}",699 "the following command exited with error code {d}:\n{s}",
690 .{ code, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) },700 .{ code, try Step.allocPrintCmd(arena, .inherit, null, argv.items) },
691 );701 );
692 return error.WasmCompilationFailed;702 return error.WasmCompilationFailed;
693 }703 }
...@@ -695,7 +705,7 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim...@@ -695,7 +705,7 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim
695 .signal => |sig| {705 .signal => |sig| {
696 log.err(706 log.err(
697 "the following command terminated with signal {t}:\n{s}",707 "the following command terminated with signal {t}:\n{s}",
698 .{ sig, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) },708 .{ sig, try Step.allocPrintCmd(arena, .inherit, null, argv.items) },
699 );709 );
700 return error.WasmCompilationFailed;710 return error.WasmCompilationFailed;
701 },711 },
...@@ -709,7 +719,7 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim...@@ -709,7 +719,7 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim
709 .unknown => {719 .unknown => {
710 log.err(720 log.err(
711 "the following command terminated unexpectedly:\n{s}",721 "the following command terminated unexpectedly:\n{s}",
712 .{try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items)},722 .{try Step.allocPrintCmd(arena, .inherit, null, argv.items)},
713 );723 );
714 return error.WasmCompilationFailed;724 return error.WasmCompilationFailed;
715 },725 },
...@@ -719,14 +729,14 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim...@@ -719,14 +729,14 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim
719 try result_error_bundle.renderToStderr(io, .{}, .auto);729 try result_error_bundle.renderToStderr(io, .{}, .auto);
720 log.err("the following command failed with {d} compilation errors:\n{s}", .{730 log.err("the following command failed with {d} compilation errors:\n{s}", .{
721 result_error_bundle.errorMessageCount(),731 result_error_bundle.errorMessageCount(),
722 try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items),732 try Step.allocPrintCmd(arena, .inherit, null, argv.items),
723 });733 });
724 return error.WasmCompilationFailed;734 return error.WasmCompilationFailed;
725 }735 }
726736
727 const base_path = result orelse {737 const base_path = result orelse {
728 log.err("child process failed to report result\n{s}", .{738 log.err("child process failed to report result\n{s}", .{
729 try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items),739 try Step.allocPrintCmd(arena, .inherit, null, argv.items),
730 });740 });
731 return error.WasmCompilationFailed;741 return error.WasmCompilationFailed;
732 };742 };
...@@ -750,7 +760,7 @@ fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8...@@ -750,7 +760,7 @@ fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8
750}760}
751761
752pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {762pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {
753 compile: *Build.Step.Compile,763 compile_step: Configuration.Step.Index,
754764
755 use_llvm: bool,765 use_llvm: bool,
756 stats: abi.time_report.CompileResult.Stats,766 stats: abi.time_report.CompileResult.Stats,
...@@ -766,8 +776,9 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {...@@ -766,8 +776,9 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {
766 const gpa = ws.gpa;776 const gpa = ws.gpa;
767 const io = ws.graph.io;777 const io = ws.graph.io;
768778
779 // TODO don't do linear search
769 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {780 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
770 if (s == &opts.compile.step) break @intCast(i);781 if (s == opts.compile_step) break @intCast(i);
771 } else unreachable;782 } else unreachable;
772783
773 const old_buf = old: {784 const old_buf = old: {
...@@ -803,12 +814,13 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {...@@ -803,12 +814,13 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {
803 ws.notifyUpdate();814 ws.notifyUpdate();
804}815}
805816
806pub fn updateTimeReportGeneric(ws: *WebServer, step: *Build.Step, duration: Io.Duration) void {817pub fn updateTimeReportGeneric(ws: *WebServer, step_index: Configuration.Step.Index, duration: Io.Duration) void {
807 const gpa = ws.gpa;818 const gpa = ws.gpa;
808 const io = ws.graph.io;819 const io = ws.graph.io;
809820
821 // TODO don't do linear search
810 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {822 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
811 if (s == step) break @intCast(i);823 if (s == step_index) break @intCast(i);
812 } else unreachable;824 } else unreachable;
813825
814 const old_buf = old: {826 const old_buf = old: {
...@@ -836,15 +848,16 @@ pub fn updateTimeReportGeneric(ws: *WebServer, step: *Build.Step, duration: Io.D...@@ -836,15 +848,16 @@ pub fn updateTimeReportGeneric(ws: *WebServer, step: *Build.Step, duration: Io.D
836848
837pub fn updateTimeReportRunTest(849pub fn updateTimeReportRunTest(
838 ws: *WebServer,850 ws: *WebServer,
839 run: *Build.Step.Run,851 run_step_index: Configuration.Step.Index,
840 tests: *const Build.Step.Run.CachedTestMetadata,852 tests: *const Step.Run.CachedTestMetadata,
841 ns_per_test: []const u64,853 ns_per_test: []const u64,
842) void {854) void {
843 const gpa = ws.gpa;855 const gpa = ws.gpa;
844 const io = ws.graph.io;856 const io = ws.graph.io;
845857
858 // TODO don't do linear search
846 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {859 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
847 if (s == &run.step) break @intCast(i);860 if (s == run_step_index) break @intCast(i);
848 } else unreachable;861 } else unreachable;
849862
850 assert(tests.names.len == ns_per_test.len);863 assert(tests.names.len == ns_per_test.len);