authorgravatar for techatrix@mailbox.orgTechatrix <techatrix@mailbox.org> 2026-07-21 16:31:04+02:00
committergravatar for techatrix@mailbox.orgTechatrix <techatrix@mailbox.org> 2026-07-21 16:31:04+02:00
logf58c93b5663beacb71dc7add21a85bd0b4f4dc46
treeb391897bc7b7ce83d7dc09bc13aa45f38315c62b
parentbb17211958a236e592043895bdfba317c6e7cd16
signaturebadge-check Signed by SSH key SHA256:HYC3SjXQcAt6uwv9pu/6OoVQ2rUH8rb5zKiUHSe9uxk

Maker: implement build system protocol foundation

4 files changed, 328 insertions(+), 2 deletions(-)

lib/compiler/Maker.zig+101-2
......@@ -11,6 +11,7 @@ const File = std.Io.File;
1111const Io = std.Io;
1212const Dir = std.Io.Dir;
1313const Path = std.Build.Cache.Path;
14const Reader = std.Io.Reader;
1415const Writer = std.Io.Writer;
1516const assert = std.debug.assert;
1617const fatal = std.process.fatal;
......@@ -19,6 +20,8 @@ const log = std.log;
1920const mem = std.mem;
2021const process = std.process;
2122const Color = std.zig.Color;
23const Client = std.zig.Client;
24const Server = std.zig.Server;
2225const EnvVar = std.zig.EnvVar;
2326const default_local_zig_cache_basename = std.zig.default_local_zig_cache_basename;
2427const stringToEnum = std.meta.stringToEnum;
......@@ -51,6 +54,8 @@ max_rss_mutex: Io.Mutex,
5154skip_oom_steps: bool,
5255unit_test_timeout_ns: ?u64,
5356watch: bool,
57protocol_server: ?*AvoidableServer,
58protocol_server_mutex: Io.Mutex,
5459web_server: ?*AvoidableWebServer,
5560/// Allocated into `gpa`.
5661memory_blocked_steps: std.ArrayList(Configuration.Step.Index),
......@@ -67,6 +72,7 @@ var stdio_buffer_allocation: [256]u8 = undefined;
6772var stdout_writer_allocation: Io.File.Writer = undefined;
6873var debug_maker_leaks: bool = false;
6974
75const AvoidableServer = if (builtin.single_threaded) void else Server;
7076const AvoidableWebServer = if (builtin.single_threaded) void else WebServer;
7177
7278const is_debug_mode = builtin.mode == .debug;
......@@ -216,6 +222,7 @@ pub fn main(init: process.Init.Minimal) !void {
216222 var watch = false;
217223 var fuzz: ?Fuzz.Mode = null;
218224 var debounce_interval_ms: u16 = 50;
225 var listen: bool = false;
219226 var webui_listen: ?Io.net.IpAddress = null;
220227 var debug_pkg_config = false;
221228 var run_args: ?[]const []const u8 = null;
......@@ -416,6 +423,8 @@ pub fn main(init: process.Init.Minimal) !void {
416423 next_arg, err,
417424 });
418425 };
426 } else if (mem.eql(u8, arg, "--listen=-")) {
427 listen = true;
419428 } else if (mem.eql(u8, arg, "--webui")) {
420429 if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) };
421430 } else if (mem.startsWith(u8, arg, "--webui=")) {
......@@ -553,7 +562,7 @@ pub fn main(init: process.Init.Minimal) !void {
553562 }
554563
555564 const early_exit_mode = fetch_only or help_menu or steps_menu or print_configuration != .none;
556 const server_mode = !early_exit_mode and (watch or webui_listen != null or fuzz != null);
565 const server_mode = !early_exit_mode and (watch or webui_listen != null or fuzz != null or listen);
557566
558567 process.raiseFileDescriptorLimit();
559568
......@@ -661,6 +670,25 @@ pub fn main(init: process.Init.Minimal) !void {
661670 break :ws &web_server_allocation;
662671 } else null;
663672
673 var stdin_buffer: [256]u8 = undefined;
674 var stdout_buffer: [256]u8 = undefined;
675 var stdin_reader = Io.File.stdin().reader(io, &stdin_buffer);
676 var stdout_writer = Io.File.stdout().writer(io, &stdout_buffer);
677
678 var protocol_server_allocation: AvoidableServer = undefined;
679 const protocol_server: ?*AvoidableServer = if (listen) s: {
680 if (builtin.single_threaded) fatal("--listen is not yet supported on single-threaded hosts", .{});
681 if (watch) fatal("using '--watch' and '--listen' together is not supported", .{});
682 if (fuzz != null) fatal("using '--fuzz' and '--listen' together is not supported", .{});
683 if (step_names.items.len > 0) fatal("build steps must be provided over the protocol instead of using CLI arguments", .{});
684 protocol_server_allocation = .{
685 .in = &stdin_reader.interface,
686 .out = &stdout_writer.interface,
687 };
688 try serveBSPHandshake(&protocol_server_allocation);
689 break :s &protocol_server_allocation;
690 } else null;
691
664692 while (true) {
665693 // If this fails, we can still start the server and wait for user
666694 // to request a rebuild. If it returns error.FailedButCacheIntact
......@@ -731,13 +759,20 @@ pub fn main(init: process.Init.Minimal) !void {
731759
732760 .watch = watch,
733761 .web_server = web_server,
762 .protocol_server = protocol_server,
763 .protocol_server_mutex = .init,
734764 .memory_blocked_steps = .empty,
735765 .step_stack = .empty,
736766 .pkg_config = .{ .debug = debug_pkg_config },
737767
738768 .error_style = error_style,
739769 .multiline_errors = multiline_errors,
740 .summary = summary orelse if (watch or webui_listen != null) .new else .failures,
770 .summary = summary orelse if (listen)
771 .none
772 else if (watch or webui_listen != null)
773 .new
774 else
775 .failures,
741776 };
742777 defer {
743778 maker.memory_blocked_steps.deinit(gpa);
......@@ -749,6 +784,52 @@ pub fn main(init: process.Init.Minimal) !void {
749784 maker.max_rss_is_default = true;
750785 }
751786
787 if (protocol_server) |s| {
788 try s.serveStringMessage(.bsp_configuration, try arena.print("{f}", .{scanned_config.path}));
789
790 var w: ?Watch = null;
791
792 const Event = union(enum) {
793 message: Reader.Error!Client.Message.Header,
794 fs_event: if (Watch.have_impl) @typeInfo(@TypeOf(Watch.wait)).@"fn".return_type.? else noreturn,
795 };
796
797 var select_buffer: [2]Event = undefined;
798 var select: Io.Select(Event) = .init(io, &select_buffer);
799 defer select.cancelDiscard();
800
801 try select.concurrent(.message, Server.receiveMessage, .{s});
802
803 var in_debounce = false;
804 loop: switch (try select.await()) {
805 .message => |payload| {
806 const header: Client.Message.Header = try payload;
807 switch (header.tag) {
808 .exit => {
809 cleanExit(io, &scanned_config);
810 process.exit(0);
811 },
812 else => fatal("unsupported message: {t}", .{header.tag}),
813 }
814 },
815 .fs_event => |payload| {
816 if (!Watch.have_impl) unreachable;
817 switch (try payload) {
818 .timeout => {
819 assert(in_debounce);
820 markFailedStepsDirty(&maker);
821 if (true) @panic("TODO run steps that were previous specified over the build system protocol");
822 in_debounce = false;
823 },
824 .dirty => in_debounce = true,
825 .clean => {},
826 }
827 try select.concurrent(.fs_event, Watch.wait, .{ &w.?, if (in_debounce) .{ .ms = debounce_interval_ms } else .none });
828 continue :loop try select.await();
829 },
830 }
831 }
832
752833 maker.prepare(step_names.items) catch |err| switch (err) {
753834 error.DependencyLoopDetected, error.InsufficientMemory => {
754835 // TODO handle DependencyLoopDetected as error.FailedButCacheIntact
......@@ -850,6 +931,9 @@ pub fn main(init: process.Init.Minimal) !void {
850931 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
851932 process.exit(1);
852933 }
934 if (protocol_server != null) {
935 fatal("(zig build system) TODO send error messages to client when build.zig compilation fails", .{});
936 }
853937 if (watch and can_fs_watch) {
854938 fatal("(zig build system) TODO set up fs watching even when build.zig compilation fails", .{});
855939 } else {
......@@ -2990,6 +3074,21 @@ fn cleanTmpFiles(maker: *Maker, steps: []const Configuration.Step.Index) void {
29903074 }
29913075}
29923076
3077fn serveBSPHandshake(s: *const std.zig.Server) !void {
3078 const handshake_header: Server.Message.Handshake = .{
3079 .version = Server.build_system_version,
3080 .flags = .{
3081 .file_system_watch_supported = Watch.have_impl,
3082 },
3083 };
3084 try s.serveMessageHeader(.{
3085 .tag = .bsp_handshake,
3086 .bytes_len = @sizeOf(Server.Message.Handshake),
3087 });
3088 try s.out.writeStruct(handshake_header, .little);
3089 try s.out.flush();
3090}
3091
29933092fn initStdoutWriter(io: Io) *Writer {
29943093 stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation);
29953094 return &stdout_writer_allocation.interface;
lib/std/zig/Server.zig+30
......@@ -12,6 +12,14 @@ const Writer = std.Io.Writer;
1212in: *Reader,
1313out: *Writer,
1414
15/// The ABI version of the build system protocol. Will be bumped whenever a
16/// backwards incompatible changes to the protocol is made.
17///
18/// Does not apply to the internal compiler protocol or test runner.
19///
20/// See `version` in `Message.Handshake`.
21pub const build_system_version: u32 = 1;
22
1523pub const Message = struct {
1624 pub const Header = extern struct {
1725 tag: Tag,
......@@ -66,9 +74,31 @@ pub const Message = struct {
6674 /// Body is a TimeReport.
6775 time_report,
6876
77 /// The first message sent by the server over the build system protocol.
78 /// Body is a `Handshake`.
79 /// This message only applies to the build system protocol.
80 bsp_handshake = 0x80000000,
81 /// Notifies that a new configuration file is available.
82 /// Body is a cwd relative path to the configuration file.
83 /// This message only applies to the build system protocol.
84 bsp_configuration,
85
6986 _,
7087 };
7188
89 /// Trailing:
90 /// * base_paths: BasePaths,
91 pub const Handshake = extern struct {
92 /// See `build_system_version`.
93 version: u32,
94 flags: Flags,
95
96 pub const Flags = packed struct(u32) {
97 file_system_watch_supported: bool,
98 _: u31 = 0,
99 };
100 };
101
72102 pub const PathPrefix = enum(u8) {
73103 cwd,
74104 zig_lib,
test/standalone/build.zig+1
......@@ -31,6 +31,7 @@ pub fn build(b: *std.Build) void {
3131 const tools_target = b.resolveTargetQuery(.{});
3232 for ([_][]const u8{
3333 // Alphabetically sorted. No need to build `tools/spirv/grammar.zig`.
34 "../../tools/bsp.zig",
3435 "../../tools/check_mingw.zig",
3536 "../../tools/dump-cov.zig",
3637 "../../tools/fetch_them_macos_headers.zig",
tools/bsp.zig created+196
......@@ -0,0 +1,196 @@
1//! CLI tool to interface with the build system protocol (zig build --listen=-)
2
3const std = @import("std");
4const Io = std.Io;
5const Allocator = std.mem.Allocator;
6const Configuration = std.Build.Configuration;
7const Client = std.zig.Client;
8const Server = std.zig.Server;
9const log = std.log.scoped(.bsp);
10
11pub fn main(init: std.process.Init) !void {
12 const io = init.io;
13 const gpa = init.gpa;
14 const arena = init.arena.allocator();
15
16 var maker_args: std.ArrayList([]const u8) = .empty;
17
18 const args = try init.minimal.args.toSlice(arena);
19 for (args[1..]) |arg| {
20 try maker_args.append(arena, try arena.dupe(u8, arg));
21 }
22 if (maker_args.items.len < 1) try maker_args.append(arena, "zig");
23 if (maker_args.items.len < 2) try maker_args.append(arena, "build");
24 if (!std.mem.eql(u8, maker_args.last().?.*, "--listen=-")) try maker_args.append(arena, "--listen=-");
25
26 log.debug("cmd: {f}", .{std.zig.SubprocessCommand{
27 .argv = maker_args.items,
28 }});
29
30 var child_process = std.process.spawn(io, .{
31 .argv = maker_args.items,
32 .stdin = .pipe,
33 .stdout = .pipe,
34 .stderr = .pipe,
35 }) catch |err| std.debug.panic("failed to spawn process: {}", .{err});
36 errdefer child_process.kill(io);
37
38 var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined;
39 var multi_reader: Io.File.MultiReader = undefined;
40 defer multi_reader.deinit();
41 multi_reader.init(
42 gpa,
43 io,
44 multi_reader_buffer.toStreams(),
45 &.{ child_process.stdout.?, child_process.stderr.? },
46 );
47 const client_stdout = multi_reader.reader(0);
48 const client_stderr = multi_reader.reader(1);
49
50 var client_stdout_buffer: [256]u8 = undefined;
51 var client_stdout_writer = child_process.stdin.?.writerStreaming(io, &client_stdout_buffer);
52
53 var client: Client = .{
54 .in = client_stdout,
55 .out = &client_stdout_writer.interface,
56 };
57
58 const err = blk: {
59 const handshake: Server.Message.Handshake = handshake: {
60 const header = client.receiveMessageWithMultiReader(&multi_reader, .none) catch |err| switch (err) {
61 error.Canceled, error.ConcurrencyUnavailable => |e| return e,
62 error.Timeout => unreachable,
63 else => |e| {
64 log.err("failed to receive message: {t}", .{err});
65 break :blk e;
66 },
67 };
68 const body = client_stdout.take(header.bytes_len) catch unreachable;
69 log.debug("received {f} ({d} bytes)", .{ fmtEnum(header.tag), body.len });
70
71 if (header.tag != .bsp_handshake) {
72 log.err("received unexpected message: {f}", .{fmtEnum(header.tag)});
73 return error.UnexpectedMessage;
74 }
75
76 var r: Io.Reader = .fixed(body);
77 break :handshake try r.takeStruct(Server.Message.Handshake, .little);
78 };
79 _ = handshake;
80
81 var conf_arena_allocator: std.heap.ArenaAllocator = .init(gpa);
82 defer conf_arena_allocator.deinit();
83 const conf_arena = conf_arena_allocator.allocator();
84
85 const configuration = configuration: {
86 const header = client.receiveMessageWithMultiReader(&multi_reader, .none) catch |err| switch (err) {
87 error.Canceled, error.ConcurrencyUnavailable => |e| return e,
88 error.Timeout => unreachable,
89 else => |e| {
90 log.err("failed to receive message: {t}", .{err});
91 break :blk e;
92 },
93 };
94 const body = client_stdout.take(header.bytes_len) catch unreachable;
95 log.debug("received {t} ({d} bytes)", .{ header.tag, body.len });
96
97 if (header.tag != .bsp_configuration) {
98 log.err("received unexpected message: {f}", .{fmtEnum(header.tag)});
99 return error.UnexpectedMessage;
100 }
101
102 const configuration_path = body;
103 var file = Io.Dir.cwd().openFile(io, configuration_path, .{}) catch |err|
104 std.debug.panic("failed to open configuration file {q}: {t}", .{ configuration_path, err });
105 defer file.close(io);
106 break :configuration Configuration.loadFile(conf_arena, io, file) catch |err|
107 std.debug.panic("failed to load configuration file {q}: {t}", .{ configuration_path, err });
108 };
109 const c = &configuration;
110
111 var top_level_steps: std.array_hash_map.String(Configuration.Step.Index) = .empty;
112 defer top_level_steps.deinit(gpa);
113
114 for (c.steps, 0..) |*conf_step, step_index_usize| {
115 if (conf_step.owner != .root) continue;
116 const step_index: Configuration.Step.Index = @fromBackingInt(@intCast(step_index_usize));
117 const flags = conf_step.flags(c);
118 if (flags.tag != .top_level) continue;
119 const name = step_index.ptr(c).name.slice(c);
120 try top_level_steps.putNoClobber(gpa, name, step_index);
121 }
122
123 std.debug.print("Steps:\n", .{});
124 for (top_level_steps.keys()) |name| {
125 std.debug.print(" - {q}\n", .{name});
126 }
127 std.debug.print(
128 \\Available Commands:
129 \\ - build [step names / step indices]
130 \\ - watch [step names / step indices]
131 \\ - exit
132 \\
133 , .{});
134
135 var stdin_reader_buffer: [256]u8 = undefined;
136 var stdin_reader = Io.File.stdin().reader(io, &stdin_reader_buffer);
137 const stdin = &stdin_reader.interface;
138
139 while (true) {
140 try Io.File.stdout().writeStreamingAll(io, "> ");
141 const command = try stdin.takeDelimiterExclusive('\n');
142 stdin.toss(1);
143 if (std.mem.startsWith(u8, command, "build") or
144 std.mem.startsWith(u8, command, "watch"))
145 {
146 @panic("TODO");
147 } else if (std.mem.eql(u8, command, "exit")) {
148 try client.serveBodylessMessage(.exit);
149 break;
150 } else {
151 log.err("unknown command: {q}", .{command});
152 continue;
153 }
154 }
155 };
156
157 try multi_reader.fillRemaining(.none);
158
159 if (client_stderr.bufferedLen() > 0) {
160 log.err("stderr:\n{s}\n", .{client_stderr.buffered()});
161 }
162
163 try err;
164
165 const term = try child_process.wait(io);
166
167 if (!term.success()) {
168 log.err("maker {f}", .{term});
169 }
170}
171
172const FormatEnum = union(enum) {
173 named: []const u8,
174 unnamed: usize,
175
176 pub fn format(
177 e: FormatEnum,
178 writer: *std.Io.Writer,
179 ) std.Io.Writer.Error!void {
180 switch (e) {
181 .named => |name| {
182 try writer.writeByte('.');
183 try writer.writeAll(name);
184 },
185 .unnamed => |number| try writer.print("0x{x}", .{number}),
186 }
187 }
188};
189
190fn fmtEnum(e: anytype) FormatEnum {
191 if (std.enums.tagName(@TypeOf(e), e)) |name| {
192 return .{ .named = name };
193 } else {
194 return .{ .unnamed = @backingInt(e) };
195 }
196}