authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-02-28 23:58:13-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-03-15 10:48:13-07:00
log0e078790feaf49964d7a0da3042117ebd10de13b
treedba66f98f82654a645a1540aea8df7d4f1ed4ca2
parent81376e72057026464e1c17330666d4fb3c5a7ce0

multiplex compiler progress messages into the build runner

21 files changed, 131 insertions(+), 31 deletions(-)

lib/build_runner.zig+1-1
......@@ -571,7 +571,7 @@ fn workerMakeOneStep(
571571 // For example, CompileStep does some sus things with modifying the saved
572572 // *Build object in install header steps that might be able to be removed
573573 // by passing the *Build object through the make() functions.
574 const make_result = s.make();
574 const make_result = s.make(&sub_prog_node);
575575
576576 // No matter the result, we want to display error/warning messages.
577577 if (s.result_error_msgs.items.len > 0) {
lib/std/Build.zig+13-3
......@@ -718,7 +718,8 @@ pub fn getUninstallStep(self: *Build) *Step {
718718 return &self.uninstall_tls.step;
719719}
720720
721fn makeUninstall(uninstall_step: *Step) anyerror!void {
721fn makeUninstall(uninstall_step: *Step, prog_node: *std.Progress.Node) anyerror!void {
722 _ = prog_node;
722723 const uninstall_tls = @fieldParentPtr(TopLevelStep, "step", uninstall_step);
723724 const self = @fieldParentPtr(Build, "uninstall_tls", uninstall_tls);
724725
......@@ -1404,7 +1405,7 @@ pub fn execAllowFail(
14041405
14051406/// This function is used exclusively for spawning and communicating with the zig compiler.
14061407/// TODO: move to build_runner.zig
1407pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]const u8 {
1408pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step, prog_node: *std.Progress.Node) ![]const u8 {
14081409 assert(argv.len != 0);
14091410
14101411 if (b.verbose) {
......@@ -1439,6 +1440,11 @@ pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]const u8 {
14391440 const Header = std.zig.Server.Message.Header;
14401441 var result: ?[]const u8 = null;
14411442
1443 var node_name: std.ArrayListUnmanaged(u8) = .{};
1444 defer node_name.deinit(b.allocator);
1445 var sub_prog_node: ?std.Progress.Node = null;
1446 defer if (sub_prog_node) |*n| n.end();
1447
14421448 while (try poller.poll()) {
14431449 const stdout = poller.fifo(.stdout);
14441450 const buf = stdout.readableSlice(0);
......@@ -1478,7 +1484,11 @@ pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]const u8 {
14781484 };
14791485 },
14801486 .progress => {
1481 @panic("TODO handle progress message");
1487 if (sub_prog_node) |*n| n.end();
1488 node_name.clearRetainingCapacity();
1489 try node_name.appendSlice(b.allocator, body);
1490 sub_prog_node = prog_node.start(node_name.items, 0);
1491 sub_prog_node.?.activate();
14821492 },
14831493 .emit_bin_path => {
14841494 result = try b.allocator.dupe(u8, body);
lib/std/Build/CheckFileStep.zig+2-1
......@@ -33,7 +33,8 @@ pub fn create(
3333 return self;
3434}
3535
36fn make(step: *Step) !void {
36fn make(step: *Step, prog_node: *std.Progress.Node) !void {
37 _ = prog_node;
3738 const self = @fieldParentPtr(CheckFileStep, "step", step);
3839
3940 const src_path = self.source.getPath(self.builder);
lib/std/Build/CheckObjectStep.zig+2-1
......@@ -300,7 +300,8 @@ pub fn checkComputeCompare(
300300 self.checks.append(new_check) catch @panic("OOM");
301301}
302302
303fn make(step: *Step) !void {
303fn make(step: *Step, prog_node: *std.Progress.Node) !void {
304 _ = prog_node;
304305 const self = @fieldParentPtr(CheckObjectStep, "step", step);
305306
306307 const gpa = self.builder.allocator;
lib/std/Build/CompileStep.zig+3-3
......@@ -1160,7 +1160,7 @@ fn constructDepString(
11601160 }
11611161}
11621162
1163fn make(step: *Step) !void {
1163fn make(step: *Step, prog_node: *std.Progress.Node) !void {
11641164 const self = @fieldParentPtr(CompileStep, "step", step);
11651165 const builder = self.builder;
11661166
......@@ -1718,7 +1718,7 @@ fn make(step: *Step) !void {
17181718 }
17191719 if (other.installed_headers.items.len > 0) {
17201720 for (other.installed_headers.items) |install_step| {
1721 try install_step.make();
1721 try install_step.make(prog_node);
17221722 }
17231723 try zig_args.append("-I");
17241724 try zig_args.append(builder.pathJoin(&.{
......@@ -1894,7 +1894,7 @@ fn make(step: *Step) !void {
18941894 try zig_args.append(resolved_args_file);
18951895 }
18961896
1897 const output_bin_path = try builder.execFromStep(zig_args.items, &self.step);
1897 const output_bin_path = try builder.execFromStep(zig_args.items, &self.step, prog_node);
18981898 const build_output_dir = fs.path.dirname(output_bin_path).?;
18991899
19001900 if (self.output_dir) |output_dir| {
lib/std/Build/ConfigHeaderStep.zig+2-1
......@@ -152,7 +152,8 @@ fn putValue(self: *ConfigHeaderStep, field_name: []const u8, comptime T: type, v
152152 }
153153}
154154
155fn make(step: *Step) !void {
155fn make(step: *Step, prog_node: *std.Progress.Node) !void {
156 _ = prog_node;
156157 const self = @fieldParentPtr(ConfigHeaderStep, "step", step);
157158 const gpa = self.builder.allocator;
158159
lib/std/Build/EmulatableRunStep.zig+2-1
......@@ -71,7 +71,8 @@ pub fn create(builder: *std.Build, name: []const u8, artifact: *CompileStep) *Em
7171 return self;
7272}
7373
74fn make(step: *Step) !void {
74fn make(step: *Step, prog_node: *std.Progress.Node) !void {
75 _ = prog_node;
7576 const self = @fieldParentPtr(EmulatableRunStep, "step", step);
7677 const host_info = self.builder.host;
7778
lib/std/Build/FmtStep.zig+2-1
......@@ -29,7 +29,8 @@ pub fn create(builder: *std.Build, paths: []const []const u8) *FmtStep {
2929 return self;
3030}
3131
32fn make(step: *Step) !void {
32fn make(step: *Step, prog_node: *std.Progress.Node) !void {
33 _ = prog_node;
3334 const self = @fieldParentPtr(FmtStep, "step", step);
3435
3536 return self.builder.spawnChild(self.argv);
lib/std/Build/InstallArtifactStep.zig+2-1
......@@ -64,7 +64,8 @@ pub fn create(builder: *std.Build, artifact: *CompileStep) *InstallArtifactStep
6464 return self;
6565}
6666
67fn make(step: *Step) !void {
67fn make(step: *Step, prog_node: *std.Progress.Node) !void {
68 _ = prog_node;
6869 const self = @fieldParentPtr(InstallArtifactStep, "step", step);
6970 const builder = self.builder;
7071
lib/std/Build/InstallDirStep.zig+2-1
......@@ -56,7 +56,8 @@ pub fn init(
5656 };
5757}
5858
59fn make(step: *Step) !void {
59fn make(step: *Step, prog_node: *std.Progress.Node) !void {
60 _ = prog_node;
6061 const self = @fieldParentPtr(InstallDirStep, "step", step);
6162 const dest_prefix = self.builder.getInstallPath(self.options.install_dir, self.options.install_subdir);
6263 const src_builder = self.override_source_builder orelse self.builder;
lib/std/Build/InstallFileStep.zig+2-1
......@@ -35,7 +35,8 @@ pub fn init(
3535 };
3636}
3737
38fn make(step: *Step) !void {
38fn make(step: *Step, prog_node: *std.Progress.Node) !void {
39 _ = prog_node;
3940 const self = @fieldParentPtr(InstallFileStep, "step", step);
4041 const src_builder = self.override_source_builder orelse self.builder;
4142 const full_src_path = self.source.getPath2(src_builder, step);
lib/std/Build/LogStep.zig+2-1
......@@ -21,7 +21,8 @@ pub fn init(builder: *std.Build, data: []const u8) LogStep {
2121 };
2222}
2323
24fn make(step: *Step) anyerror!void {
24fn make(step: *Step, prog_node: *std.Progress.Node) anyerror!void {
25 _ = prog_node;
2526 const self = @fieldParentPtr(LogStep, "step", step);
2627 log.info("{s}", .{self.data});
2728}
lib/std/Build/ObjCopyStep.zig+2-1
......@@ -66,7 +66,8 @@ pub fn getOutputSource(self: *const ObjCopyStep) std.Build.FileSource {
6666 return .{ .generated = &self.output_file };
6767}
6868
69fn make(step: *Step) !void {
69fn make(step: *Step, prog_node: *std.Progress.Node) !void {
70 _ = prog_node;
7071 const self = @fieldParentPtr(ObjCopyStep, "step", step);
7172 const b = self.builder;
7273
lib/std/Build/OptionsStep.zig+2-1
......@@ -219,7 +219,8 @@ pub fn getSource(self: *OptionsStep) FileSource {
219219 return .{ .generated = &self.generated_file };
220220}
221221
222fn make(step: *Step) !void {
222fn make(step: *Step, prog_node: *std.Progress.Node) !void {
223 _ = prog_node;
223224 const self = @fieldParentPtr(OptionsStep, "step", step);
224225
225226 for (self.artifact_args.items) |item| {
lib/std/Build/RemoveDirStep.zig+2-1
......@@ -22,7 +22,8 @@ pub fn init(builder: *std.Build, dir_path: []const u8) RemoveDirStep {
2222 };
2323}
2424
25fn make(step: *Step) !void {
25fn make(step: *Step, prog_node: *std.Progress.Node) !void {
26 _ = prog_node;
2627 const self = @fieldParentPtr(RemoveDirStep, "step", step);
2728
2829 const full_path = self.builder.pathFromRoot(self.dir_path);
lib/std/Build/RunStep.zig+2-1
......@@ -206,7 +206,8 @@ fn needOutputCheck(self: RunStep) bool {
206206 return false;
207207}
208208
209fn make(step: *Step) !void {
209fn make(step: *Step, prog_node: *std.Progress.Node) !void {
210 _ = prog_node;
210211 const self = @fieldParentPtr(RunStep, "step", step);
211212 const need_output_check = self.needOutputCheck();
212213
lib/std/Build/Step.zig+8-5
......@@ -1,6 +1,6 @@
11id: Id,
22name: []const u8,
3makeFn: *const fn (self: *Step) anyerror!void,
3makeFn: MakeFn,
44dependencies: std.ArrayList(*Step),
55/// This field is empty during execution of the user's build script, and
66/// then populated during dependency loop checking in the build runner.
......@@ -13,6 +13,8 @@ debug_stack_trace: [n_debug_stack_frames]usize,
1313result_error_msgs: std.ArrayListUnmanaged([]const u8),
1414result_error_bundle: std.zig.ErrorBundle,
1515
16pub const MakeFn = *const fn (self: *Step, prog_node: *std.Progress.Node) anyerror!void;
17
1618const n_debug_stack_frames = 4;
1719
1820pub const State = enum {
......@@ -72,7 +74,7 @@ pub const Id = enum {
7274pub const Options = struct {
7375 id: Id,
7476 name: []const u8,
75 makeFn: *const fn (self: *Step) anyerror!void = makeNoOp,
77 makeFn: MakeFn = makeNoOp,
7678 first_ret_addr: ?usize = null,
7779};
7880
......@@ -101,8 +103,8 @@ pub fn init(allocator: Allocator, options: Options) Step {
101103/// If the Step's `make` function reports `error.MakeFailed`, it indicates they
102104/// have already reported the error. Otherwise, we add a simple error report
103105/// here.
104pub fn make(s: *Step) error{MakeFailed}!void {
105 return s.makeFn(s) catch |err| {
106pub fn make(s: *Step, prog_node: *std.Progress.Node) error{MakeFailed}!void {
107 return s.makeFn(s, prog_node) catch |err| {
106108 if (err != error.MakeFailed) {
107109 const gpa = s.dependencies.allocator;
108110 s.result_error_msgs.append(gpa, std.fmt.allocPrint(gpa, "{s} failed: {s}", .{
......@@ -129,8 +131,9 @@ pub fn getStackTrace(s: *Step) std.builtin.StackTrace {
129131 };
130132}
131133
132fn makeNoOp(self: *Step) anyerror!void {
134fn makeNoOp(self: *Step, prog_node: *std.Progress.Node) anyerror!void {
133135 _ = self;
136 _ = prog_node;
134137}
135138
136139pub fn cast(step: *Step, comptime T: type) ?*T {
lib/std/Build/TranslateCStep.zig+2-2
......@@ -88,7 +88,7 @@ pub fn defineCMacroRaw(self: *TranslateCStep, name_and_value: []const u8) void {
8888 self.c_macros.append(self.builder.dupe(name_and_value)) catch @panic("OOM");
8989}
9090
91fn make(step: *Step) !void {
91fn make(step: *Step, prog_node: *std.Progress.Node) !void {
9292 const self = @fieldParentPtr(TranslateCStep, "step", step);
9393
9494 var argv_list = std.ArrayList([]const u8).init(self.builder.allocator);
......@@ -120,7 +120,7 @@ fn make(step: *Step) !void {
120120
121121 try argv_list.append(self.source.getPath(self.builder));
122122
123 const output_path_nl = try self.builder.execFromStep(argv_list.items, &self.step);
123 const output_path_nl = try self.builder.execFromStep(argv_list.items, &self.step, prog_node);
124124 const output_path = mem.trimRight(u8, output_path_nl, "\r\n");
125125
126126 self.out_basename = fs.path.basename(output_path);
lib/std/Build/WriteFileStep.zig+2-1
......@@ -99,7 +99,8 @@ pub fn getFileSource(wf: *WriteFileStep, sub_path: []const u8) ?std.Build.FileSo
9999 return null;
100100}
101101
102fn make(step: *Step) !void {
102fn make(step: *Step, prog_node: *std.Progress.Node) !void {
103 _ = prog_node;
103104 const wf = @fieldParentPtr(WriteFileStep, "step", step);
104105
105106 // Writing to source files is kind of an extra capability of this
src/main.zig+72-1
......@@ -3573,7 +3573,21 @@ fn serve(
35733573 if (comp.bin_file.options.output_mode == .Exe) {
35743574 try comp.makeBinFileWritable();
35753575 }
3576 try comp.update(main_progress_node);
3576
3577 {
3578 var reset: std.Thread.ResetEvent = .{};
3579
3580 var progress_thread = try std.Thread.spawn(.{}, progressThread, .{
3581 &progress, out, &reset,
3582 });
3583 defer {
3584 reset.set();
3585 progress_thread.join();
3586 }
3587
3588 try comp.update(main_progress_node);
3589 }
3590
35773591 try comp.makeBinFileExecutable();
35783592 try serveUpdateResults(out, comp);
35793593 },
......@@ -3629,6 +3643,63 @@ fn serve(
36293643 }
36303644}
36313645
3646fn progressThread(progress: *std.Progress, out: fs.File, reset: *std.Thread.ResetEvent) void {
3647 while (true) {
3648 if (reset.timedWait(500 * std.time.ns_per_ms)) |_| {
3649 // The Compilation update has completed.
3650 return;
3651 } else |err| switch (err) {
3652 error.Timeout => {},
3653 }
3654
3655 var buf: std.BoundedArray(u8, 160) = .{};
3656
3657 {
3658 progress.update_mutex.lock();
3659 defer progress.update_mutex.unlock();
3660
3661 var need_ellipse = false;
3662 var maybe_node: ?*std.Progress.Node = &progress.root;
3663 while (maybe_node) |node| {
3664 if (need_ellipse) {
3665 buf.appendSlice("... ") catch {};
3666 }
3667 need_ellipse = false;
3668 const eti = @atomicLoad(usize, &node.unprotected_estimated_total_items, .Monotonic);
3669 const completed_items = @atomicLoad(usize, &node.unprotected_completed_items, .Monotonic);
3670 const current_item = completed_items + 1;
3671 if (node.name.len != 0 or eti > 0) {
3672 if (node.name.len != 0) {
3673 buf.appendSlice(node.name) catch {};
3674 need_ellipse = true;
3675 }
3676 if (eti > 0) {
3677 if (need_ellipse) buf.appendSlice(" ") catch {};
3678 buf.writer().print("[{d}/{d}] ", .{ current_item, eti }) catch {};
3679 need_ellipse = false;
3680 } else if (completed_items != 0) {
3681 if (need_ellipse) buf.appendSlice(" ") catch {};
3682 buf.writer().print("[{d}] ", .{current_item}) catch {};
3683 need_ellipse = false;
3684 }
3685 }
3686 maybe_node = @atomicLoad(?*std.Progress.Node, &node.recently_updated_child, .Acquire);
3687 }
3688 }
3689
3690 const progress_string = buf.slice();
3691
3692 serveMessage(out, .{
3693 .tag = .progress,
3694 .bytes_len = @intCast(u32, progress_string.len),
3695 }, &.{
3696 progress_string,
3697 }) catch |err| {
3698 fatal("unable to write to client: {s}", .{@errorName(err)});
3699 };
3700 }
3701}
3702
36323703fn serveMessage(
36333704 out: fs.File,
36343705 header: std.zig.Server.Message.Header,
test/tests.zig+4-2
......@@ -875,7 +875,8 @@ pub const StackTracesContext = struct {
875875 return ptr;
876876 }
877877
878 fn make(step: *Step) !void {
878 fn make(step: *Step, prog_node: *std.Progress.Node) !void {
879 _ = prog_node;
879880 const self = @fieldParentPtr(RunAndCompareStep, "step", step);
880881 const b = self.context.b;
881882
......@@ -1218,7 +1219,8 @@ pub const GenHContext = struct {
12181219 return ptr;
12191220 }
12201221
1221 fn make(step: *Step) !void {
1222 fn make(step: *Step, prog_node: *std.Progress.Node) !void {
1223 _ = prog_node;
12221224 const self = @fieldParentPtr(GenHCmpOutputStep, "step", step);
12231225 const b = self.context.b;
12241226