Commit 5bf19f61ff5fe80a8f9aad5ce55812bd7e673fd0 | zig

author	Andrew Kelley <andrew@ziglang.org> 2026-08-05 01:42:28+02:00
committer	Andrew Kelley <andrew@ziglang.org> 2026-08-05 01:42:28+02:00
log	5bf19f61ff5fe80a8f9aad5ce55812bd7e673fd0
tree	5cf57fe99e767cfbfa827263d308f23616271232
parent	5f74e4f3f8b909835ef794253a98a77868b3880e
parent	d697d97a95688e873d2677367e94010cdaa3ac73

Merge pull request 'Implement foundation of the build system protocol' (#36147) from Techatrix/zig:build-system-protocol into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/36147
Reviewed-by: Andrew Kelley <andrew@ziglang.org>

16 files changed, 958 insertions(+), 375 deletions(-)

lib
- compiler
  - Maker.zig
  - Maker
    Step.zig
    Step
    Run.zig
  - objcopy.zig
  - std-docs.zig
  - test_runner.zig
- std
  - Io
    Reader.zig
  - mem.zig
  - zig.zig
  - zig
    Client.zig
    Server.zig
src
- Compilation.zig
- main.zig
test
- standalone
  - build.zig
tools
- bsp.zig
- incr-check.zig

lib/compiler/Maker.zig+272-44

...	...	@@ -11,6 +11,7 @@ const File = std.Io.File;
11	11	const Io = std.Io;
12	12	const Dir = std.Io.Dir;
13	13	const Path = std.Build.Cache.Path;
	14	const Reader = std.Io.Reader;
14	15	const Writer = std.Io.Writer;
15	16	const assert = std.debug.assert;
16	17	const fatal = std.process.fatal;
...	...	@@ -19,6 +20,8 @@ const log = std.log;
19	20	const mem = std.mem;
20	21	const process = std.process;
21	22	const Color = std.zig.Color;
	23	const Client = std.zig.Client;
	24	const Server = std.zig.Server;
22	25	const EnvVar = std.zig.EnvVar;
23	26	const default_local_zig_cache_basename = std.zig.default_local_zig_cache_basename;
24	27	const stringToEnum = std.meta.stringToEnum;
...	...	@@ -51,10 +54,14 @@ max_rss_mutex: Io.Mutex,
51	54	skip_oom_steps: bool,
52	55	unit_test_timeout_ns: ?u64,
53	56	watch: bool,
	57	protocol_server: ?*AvoidableServer,
	58	protocol_server_mutex: Io.Mutex,
54	59	web_server: ?*AvoidableWebServer,
55	60	/// Allocated into `gpa`.
56	61	memory_blocked_steps: std.ArrayList(Configuration.Step.Index),
57	62	/// Allocated into `gpa`.
	63	initial_steps: std.array_hash_map.Auto(Configuration.Step.Index, void),
	64	/// Allocated into `gpa`.
58	65	step_stack: std.array_hash_map.Auto(Configuration.Step.Index, void),
59	66	pkg_config: PkgConfig,
60	67
...	...	@@ -67,6 +74,7 @@ var stdio_buffer_allocation: [256]u8 = undefined;
67	74	var stdout_writer_allocation: Io.File.Writer = undefined;
68	75	var debug_maker_leaks: bool = false;
69	76
	77	const AvoidableServer = if (builtin.single_threaded) void else Server;
70	78	const AvoidableWebServer = if (builtin.single_threaded) void else WebServer;
71	79
72	80	const is_debug_mode = builtin.mode == .debug;
...	...	@@ -216,6 +224,7 @@ pub fn main(init: process.Init.Minimal) !void {
216	224	var watch = false;
217	225	var fuzz: ?Fuzz.Mode = null;
218	226	var debounce_interval_ms: u16 = 50;
	227	var listen: bool = false;
219	228	var webui_listen: ?Io.net.IpAddress = null;
220	229	var debug_pkg_config = false;
221	230	var run_args: ?[]const []const u8 = null;
...	...	@@ -422,6 +431,8 @@ pub fn main(init: process.Init.Minimal) !void {
422	431	next_arg, err,
423	432	});
424	433	};
	434	} else if (mem.eql(u8, arg, "--listen=-")) {
	435	listen = true;
425	436	} else if (mem.eql(u8, arg, "--webui")) {
426	437	if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) };
427	438	} else if (mem.startsWith(u8, arg, "--webui=")) {
...	...	@@ -559,7 +570,7 @@ pub fn main(init: process.Init.Minimal) !void {
559	570	}
560	571
561	572	const early_exit_mode = fetch_only or help_menu or steps_menu or print_configuration != .none;
562		const server_mode = !early_exit_mode and (watch or webui_listen != null or fuzz != null);
	573	const server_mode = !early_exit_mode and (watch or webui_listen != null or fuzz != null or listen);
563	574
564	575	process.raiseFileDescriptorLimit();
565	576
...	...	@@ -667,6 +678,25 @@ pub fn main(init: process.Init.Minimal) !void {
667	678	break :ws &web_server_allocation;
668	679	} else null;
669	680
	681	var stdin_buffer: [256]u8 = undefined;
	682	var stdout_buffer: [256]u8 = undefined;
	683	var stdin_reader = Io.File.stdin().reader(io, &stdin_buffer);
	684	var stdout_writer = Io.File.stdout().writer(io, &stdout_buffer);
	685
	686	var protocol_server_allocation: AvoidableServer = undefined;
	687	const protocol_server: ?*AvoidableServer = if (listen) s: {
	688	if (builtin.single_threaded) fatal("--listen is not yet supported on single-threaded hosts", .{});
	689	if (watch) fatal("using '--watch' and '--listen' together is not supported", .{});
	690	if (fuzz != null) fatal("using '--fuzz' and '--listen' together is not supported", .{});
	691	if (step_names.items.len > 0) fatal("build steps must be provided over the protocol instead of using CLI arguments", .{});
	692	protocol_server_allocation = .{
	693	.in = &stdin_reader.interface,
	694	.out = &stdout_writer.interface,
	695	};
	696	try serveBSPHandshake(&protocol_server_allocation);
	697	break :s &protocol_server_allocation;
	698	} else null;
	699
670	700	while (true) {
671	701	// If this fails, we can still start the server and wait for user
672	702	// to request a rebuild. If it returns error.FailedButCacheIntact
...	...	@@ -737,16 +767,25 @@ pub fn main(init: process.Init.Minimal) !void {
737	767
738	768	.watch = watch,
739	769	.web_server = web_server,
	770	.protocol_server = protocol_server,
	771	.protocol_server_mutex = .init,
740	772	.memory_blocked_steps = .empty,
	773	.initial_steps = .empty,
741	774	.step_stack = .empty,
742	775	.pkg_config = .{ .debug = debug_pkg_config },
743	776
744	777	.error_style = error_style,
745	778	.multiline_errors = multiline_errors,
746		.summary = summary orelse if (watch or webui_listen != null) .new else .failures,
	779	.summary = summary orelse if (listen)
	780	.none
	781	else if (watch or webui_listen != null)
	782	.new
	783	else
	784	.failures,
747	785	};
748	786	defer {
749	787	maker.memory_blocked_steps.deinit(gpa);
	788	maker.initial_steps.deinit(gpa);
750	789	maker.step_stack.deinit(gpa);
751	790	}
752	791
...	...	@@ -755,7 +794,91 @@ pub fn main(init: process.Init.Minimal) !void {
755	794	maker.max_rss_is_default = true;
756	795	}
757	796
758		maker.prepare(step_names.items) catch \|err\| switch (err) {
	797	if (protocol_server) \|s\| {
	798	try s.serveStringMessage(.bsp_configuration, try arena.print("{f}", .{scanned_config.path}));
	799
	800	var w: ?Watch = null;
	801
	802	const Event = union(enum) {
	803	message: Reader.Error!Client.Message.Header,
	804	fs_event: if (Watch.have_impl) @typeInfo(@TypeOf(Watch.wait)).@"fn".return_type.? else noreturn,
	805	};
	806
	807	var select_buffer: [2]Event = undefined;
	808	var select: Io.Select(Event) = .init(io, &select_buffer);
	809	defer select.cancelDiscard();
	810
	811	try select.concurrent(.message, Server.receiveMessage, .{s});
	812
	813	var in_debounce = false;
	814	loop: switch (try select.await()) {
	815	.message => \|payload\| {
	816	const header: Client.Message.Header = try payload;
	817	switch (header.tag) {
	818	.exit => {
	819	cleanExit(io, &scanned_config);
	820	process.exit(0);
	821	},
	822	.bsp_build_steps => {
	823	// Cancel existing file watching
	824	select.cancelDiscard();
	825	in_debounce = false;
	826
	827	const body = try s.in.takeStruct(Client.Message.BuildSteps, .little);
	828	const steps = try s.in.readSliceEndianAlloc(gpa, Configuration.Step.Index, body.step_count, .little);
	829	defer gpa.free(steps);
	830	if (body.flags.watch and !Watch.have_impl) fatal("file watching is unavailable", .{});
	831
	832	try select.concurrent(.message, Server.receiveMessage, .{s});
	833
	834	maker.watch = body.flags.watch;
	835	maker.prepare(steps) catch \|err\| switch (err) {
	836	error.DependencyLoopDetected, error.InsufficientMemory => {
	837	// TODO handle DependencyLoopDetected as error.FailedButCacheIntact
	838	// and handle InsufficientMemory as error.AlreadyReported
	839	_ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
	840	process.exit(1);
	841	},
	842	else => \|e\| return e,
	843	};
	844
	845	try maker.makeSteps(main_progress_node, null);
	846
	847	if (body.flags.watch) {
	848	if (!Watch.have_impl) unreachable;
	849	if (w == null) w = try .init(&maker);
	850
	851	try w.?.update(maker.step_stack.keys());
	852	try select.concurrent(.fs_event, Watch.wait, .{ &w.?, if (in_debounce) .{ .ms = debounce_interval_ms } else .none });
	853	}
	854
	855	continue :loop try select.await();
	856	},
	857	else => fatal("unsupported message: {t}", .{header.tag}),
	858	}
	859	},
	860	.fs_event => \|payload\| {
	861	if (!Watch.have_impl) unreachable;
	862	switch (try payload) {
	863	.timeout => {
	864	assert(in_debounce);
	865	markFailedStepsDirty(&maker);
	866	try maker.makeSteps(main_progress_node, null);
	867	in_debounce = false;
	868	},
	869	.dirty => in_debounce = true,
	870	.clean => {},
	871	}
	872	try select.concurrent(.fs_event, Watch.wait, .{ &w.?, if (in_debounce) .{ .ms = debounce_interval_ms } else .none });
	873	continue :loop try select.await();
	874	},
	875	}
	876	}
	877
	878	const initial_steps = try maker.resolveTopLevelSteps(step_names.items);
	879	defer gpa.free(initial_steps);
	880
	881	maker.prepare(initial_steps) catch \|err\| switch (err) {
759	882	error.DependencyLoopDetected, error.InsufficientMemory => {
760	883	// TODO handle DependencyLoopDetected as error.FailedButCacheIntact
761	884	// and handle InsufficientMemory as error.AlreadyReported
...	...	@@ -780,18 +903,7 @@ pub fn main(init: process.Init.Minimal) !void {
780	903	error.WriteFailed => return stderr.file_writer.err.?,
781	904	};
782	905	}) {
783		if (web_server) \|ws\| ws.startBuild();
784
785		try maker.makeStepNames(step_names.items, main_progress_node, fuzz);
786
787		if (web_server) \|ws\| {
788		if (fuzz) \|mode\| if (mode != .forever) fatal(
789		"error: limited fuzzing is not implemented yet for --webui",
790		.{},
791		);
792
793		ws.finishBuild(.{ .fuzz = fuzz != null });
794		}
	906	try maker.makeSteps(main_progress_node, fuzz);
795	907
796	908	if (web_server) \|ws\| {
797	909	const c = &scanned_config.configuration;
...	...	@@ -856,6 +968,9 @@ pub fn main(init: process.Init.Minimal) !void {
856	968	_ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
857	969	process.exit(1);
858	970	}
	971	if (protocol_server != null) {
	972	fatal("(zig build system) TODO send error messages to client when build.zig compilation fails", .{});
	973	}
859	974	if (watch and can_fs_watch) {
860	975	fatal("(zig build system) TODO set up fs watching even when build.zig compilation fails", .{});
861	976	} else {
...	...	@@ -2022,11 +2137,37 @@ pub fn stepByIndex(maker: const Maker, i: Configuration.Step.Index) Step {
2022	2137	return &maker.steps[@backingInt(i)];
2023	2138	}
2024	2139
2025		fn prepare(maker: *Maker, step_names: []const []const u8) !void {
	2140	fn resolveTopLevelSteps(maker: *Maker, step_names: []const []const u8) ![]const Configuration.Step.Index {
	2141	const gpa = maker.gpa;
	2142	const c = &maker.scanned_config.configuration;
	2143
	2144	if (step_names.len == 0) {
	2145	return try gpa.dupe(Configuration.Step.Index, &.{c.default_step});
	2146	}
	2147
	2148	var result: std.array_hash_map.Auto(Configuration.Step.Index, void) = .empty;
	2149	defer result.deinit(gpa);
	2150
	2151	try result.ensureTotalCapacity(gpa, step_names.len);
	2152
	2153	for (0..step_names.len) \|i\| {
	2154	const step_name = step_names[step_names.len - i - 1];
	2155	const s = maker.scanned_config.top_level_steps.get(step_name) orelse {
	2156	log.info("to list available steps: zig build -l", .{});
	2157	fatal("no such step: {s}", .{step_name});
	2158	};
	2159	result.putAssumeCapacity(s, {});
	2160	}
	2161
	2162	return try gpa.dupe(Configuration.Step.Index, result.keys());
	2163	}
	2164
	2165	fn prepare(maker: *Maker, step_indices: []const Configuration.Step.Index) !void {
2026	2166	const gpa = maker.gpa;
2027	2167	const graph = maker.graph;
2028	2168	const arena = graph.arena;
2029	2169	const seed: u32 = graph.random_seed;
	2170	const initial_steps = &maker.initial_steps;
2030	2171	const step_stack = &maker.step_stack;
2031	2172	const c = &maker.scanned_config.configuration;
2032	2173
...	...	@@ -2035,18 +2176,15 @@ fn prepare(maker: *Maker, step_names: []const []const u8) !void {
2035	2176	step.* = .{ .extended = .init(step_index.ptr(c).flags(c).tag) };
2036	2177	}
2037	2178
2038		if (step_names.len == 0) {
2039		try step_stack.put(gpa, c.default_step, {});
2040		} else {
2041		try step_stack.ensureUnusedCapacity(gpa, step_names.len);
2042		for (0..step_names.len) \|i\| {
2043		const step_name = step_names[step_names.len - i - 1];
2044		const s = maker.scanned_config.top_level_steps.get(step_name) orelse {
2045		log.info("to list available steps: zig build -l", .{});
2046		fatal("no such step: {s}", .{step_name});
2047		};
2048		step_stack.putAssumeCapacity(s, {});
2049		}
	2179	try initial_steps.ensureUnusedCapacity(gpa, step_indices.len);
	2180	try step_stack.ensureUnusedCapacity(gpa, step_indices.len);
	2181
	2182	initial_steps.clearRetainingCapacity();
	2183	step_stack.clearRetainingCapacity();
	2184
	2185	for (step_indices) \|step\| {
	2186	initial_steps.putAssumeCapacity(step, {});
	2187	step_stack.putAssumeCapacity(step, {});
2050	2188	}
2051	2189
2052	2190	const starting_steps = try arena.dupe(Configuration.Step.Index, step_stack.keys());
...	...	@@ -2095,9 +2233,8 @@ fn prepare(maker: *Maker, step_names: []const []const u8) !void {
2095	2233	}
2096	2234	}
2097	2235
2098		fn makeStepNames(
	2236	fn makeSteps(
2099	2237	maker: *Maker,
2100		step_names: []const []const u8,
2101	2238	parent_progress_node: std.Progress.Node,
2102	2239	fuzz: ?Fuzz.Mode,
2103	2240	) !void {
...	...	@@ -2108,6 +2245,12 @@ fn makeStepNames(
2108	2245	const top_level_steps = &maker.scanned_config.top_level_steps;
2109	2246	const c = &maker.scanned_config.configuration;
2110	2247
	2248	if (maker.web_server) \|ws\| ws.startBuild();
	2249
	2250	if (maker.protocol_server) \|s\| {
	2251	try s.serveBodylessMessage(.bsp_build_started);
	2252	}
	2253
2111	2254	{
2112	2255	// Collect the initial set of tasks (those with no outstanding dependencies) into a buffer,
2113	2256	// then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking
...	...	@@ -2133,6 +2276,19 @@ fn makeStepNames(
2133	2276	try group.await(io);
2134	2277	}
2135	2278
	2279	if (maker.web_server) \|ws\| {
	2280	if (fuzz) \|mode\| if (mode != .forever) fatal(
	2281	"error: limited fuzzing is not implemented yet for --webui",
	2282	.{},
	2283	);
	2284
	2285	ws.finishBuild(.{ .fuzz = fuzz != null });
	2286	}
	2287
	2288	if (maker.protocol_server) \|s\| {
	2289	try s.serveBodylessMessage(.bsp_build_completed);
	2290	}
	2291
2136	2292	assert(maker.memory_blocked_steps.items.len == 0);
2137	2293
2138	2294	var test_pass_count: usize = 0;
...	...	@@ -2285,7 +2441,7 @@ fn makeStepNames(
2285	2441	defer step_stack_copy.deinit(gpa);
2286	2442
2287	2443	var print_node: PrintNode = .{ .parent = null };
2288		if (step_names.len == 0) {
	2444	if (maker.initial_steps.count() == 0) {
2289	2445	print_node.last = true;
2290	2446	printTreeStep(maker, c.default_step, t, &print_node, &step_stack_copy) catch \|err\| switch (err) {
2291	2447	error.Canceled => \|e\| return e,
...	...	@@ -2293,10 +2449,10 @@ fn makeStepNames(
2293	2449	};
2294	2450	} else {
2295	2451	const last_index = if (maker.summary == .all) top_level_steps.count() else blk: {
2296		var i: usize = step_names.len;
	2452	var i: usize = maker.initial_steps.count();
2297	2453	while (i > 0) {
2298	2454	i -= 1;
2299		const step_index = top_level_steps.get(step_names[i]).?;
	2455	const step_index = maker.initial_steps.keys()[i];
2300	2456	const step = maker.stepByIndex(step_index);
2301	2457	const found = switch (maker.summary) {
2302	2458	.all, .line, .none => unreachable,
...	...	@@ -2307,8 +2463,7 @@ fn makeStepNames(
2307	2463	}
2308	2464	break :blk top_level_steps.count();
2309	2465	};
2310		for (step_names, 0..) \|step_name, i\| {
2311		const step_index = top_level_steps.get(step_name).?;
	2466	for (maker.initial_steps.keys(), 0..) \|step_index, i\| {
2312	2467	print_node.last = i + 1 == last_index;
2313	2468	printTreeStep(maker, step_index, t, &print_node, &step_stack_copy) catch \|err\| switch (err) {
2314	2469	error.Canceled => \|e\| return e,
...	...	@@ -2319,7 +2474,7 @@ fn makeStepNames(
2319	2474	w.writeByte('\n') catch {};
2320	2475	}
2321	2476
2322		if (maker.watch or maker.web_server != null) return;
	2477	if (maker.watch or maker.web_server != null or maker.protocol_server != null) return;
2323	2478
2324	2479	const code: u8 = code: {
2325	2480	if (failure_count == 0) break :code 0; // success
...	...	@@ -2394,6 +2549,15 @@ fn makeStep(
2394	2549	defer step_prog_node.end();
2395	2550
2396	2551	if (maker.web_server) \|ws\| ws.updateStepStatus(step_index, .wip);
	2552	if (maker.protocol_server) \|s\| {
	2553	maker.protocol_server_mutex.lockUncancelable(io);
	2554	defer maker.protocol_server_mutex.unlock(io);
	2555
	2556	s.serveU32Message(
	2557	.bsp_step_started,
	2558	@backingInt(step_index),
	2559	) catch @panic("TODO propagate error when failing to send protocol message");
	2560	}
2397	2561
2398	2562	const new_state: Step.State = for (deps) \|dep_index\| {
2399	2563	const dep_make_step = maker.stepByIndex(dep_index);
...	...	@@ -2419,7 +2583,7 @@ fn makeStep(
2419	2583
2420	2584	@atomicStore(Step.State, &make_step.state, new_state, .monotonic);
2421	2585
2422		switch (new_state) {
	2586	const success = switch (new_state) {
2423	2587	.precheck_unstarted => unreachable,
2424	2588	.precheck_started => unreachable,
2425	2589	.precheck_done => unreachable,
...	...	@@ -2427,17 +2591,37 @@ fn makeStep(
2427	2591	.failure,
2428	2592	.dependency_failure,
2429	2593	.skipped_oom,
2430		=> {
2431		if (maker.web_server) \|ws\| ws.updateStepStatus(step_index, .failure);
2432		std.Progress.setStatus(.failure_working);
2433		},
	2594	=> false,
2434	2595
2435	2596	.success,
2436	2597	.skipped,
2437		=> {
2438		if (maker.web_server) \|ws\| ws.updateStepStatus(step_index, .success);
2439		},
	2598	=> true,
	2599	};
	2600
	2601	if (maker.web_server) \|ws\| {
	2602	ws.updateStepStatus(step_index, if (success) .success else .failure);
2440	2603	}
	2604	if (maker.protocol_server != null) {
	2605	maker.protocol_server_mutex.lockUncancelable(io);
	2606	defer maker.protocol_server_mutex.unlock(io);
	2607
	2608	const status: Server.Message.BuildStepCompleted.Status = switch (new_state) {
	2609	.precheck_unstarted => unreachable,
	2610	.precheck_started => unreachable,
	2611	.precheck_done => unreachable,
	2612	.success => .success,
	2613	.failure, .dependency_failure => .failure,
	2614	.skipped => .skipped,
	2615	.skipped_oom => .skipped_oom,
	2616	};
	2617	serveBuildStepCompleted(
	2618	maker,
	2619	step_index,
	2620	status,
	2621	) catch \|err\| std.debug.panic("TODO propagate error when failing to send protocol message: {t}", .{err});
	2622	}
	2623
	2624	if (!success) std.Progress.setStatus(.failure_working);
2441	2625	}
2442	2626
2443	2627	// No matter the result, we want to display error/warning messages.
...	...	@@ -2992,6 +3176,50 @@ fn cleanTmpFiles(maker: *Maker, steps: []const Configuration.Step.Index) void {
2992	3176	}
2993	3177	}
2994	3178
	3179	fn serveBSPHandshake(s: *const std.zig.Server) !void {
	3180	const handshake_header: Server.Message.Handshake = .{
	3181	.version = Server.build_system_version,
	3182	.flags = .{
	3183	.file_system_watch_supported = Watch.have_impl,
	3184	},
	3185	};
	3186	try s.serveMessageHeader(.{
	3187	.tag = .bsp_handshake,
	3188	.bytes_len = @sizeOf(Server.Message.Handshake),
	3189	});
	3190	try s.out.writeStruct(handshake_header, .little);
	3191	try s.out.flush();
	3192	}
	3193
	3194	fn serveBuildStepCompleted(
	3195	maker: *Maker,
	3196	step_index: Configuration.Step.Index,
	3197	status: Server.Message.BuildStepCompleted.Status,
	3198	) !void {
	3199	const s: *Server = maker.protocol_server.?;
	3200	const step = maker.stepByIndex(step_index);
	3201	const error_bundle = step.result_error_bundle;
	3202
	3203	const body: Server.Message.BuildStepCompleted = .{
	3204	.step_index = step_index,
	3205	.status = status,
	3206	.error_bundle = .{
	3207	.extra_len = @intCast(error_bundle.extra.len),
	3208	.string_bytes_len = @intCast(error_bundle.string_bytes.len),
	3209	},
	3210	};
	3211	const eb_bytes_len = @sizeOf(u32) * error_bundle.extra.len + error_bundle.string_bytes.len;
	3212	const bytes_len = @sizeOf(Server.Message.BuildStepCompleted) + eb_bytes_len;
	3213	try s.serveMessageHeader(.{
	3214	.tag = .bsp_step_completed,
	3215	.bytes_len = @intCast(bytes_len),
	3216	});
	3217	try s.out.writeStruct(body, .little);
	3218	try s.out.writeSliceEndian(u32, error_bundle.extra, .little);
	3219	try s.out.writeAll(error_bundle.string_bytes);
	3220	try s.out.flush();
	3221	}
	3222
2995	3223	fn initStdoutWriter(io: Io) *Writer {
2996	3224	stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation);
2997	3225	return &stdout_writer_allocation.interface;

lib/compiler/Maker/Step.zig+10-8

...	...	@@ -561,24 +561,26 @@ fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: Maker, zp: Zi
561	561	var result: ?Path = null;
562	562	var eos_err: error{EndOfStream}!void = {};
563	563
564		const stdout = zp.multi_reader.fileReader(0);
	564	var client: std.zig.Client = .{
	565	.in = zp.multi_reader.reader(0),
	566	.out = undefined,
	567	};
565	568
566	569	while (true) {
567		const Header = std.zig.Server.Message.Header;
568		const header = stdout.interface.takeStruct(Header, .little) catch \|err\| switch (err) {
569		error.EndOfStream => break,
570		error.ReadFailed => return stdout.err.?,
571		};
572		const body = stdout.interface.take(header.bytes_len) catch \|err\| switch (err) {
	570	const header = client.receiveMessageWithMultiReader(&zp.multi_reader, .none) catch \|err\| switch (err) {
	571	error.Timeout => unreachable,
573	572	error.EndOfStream => \|e\| {
	573	if (client.in.bufferedLen() == 0) break;
574	574	// Better to report the crash with stderr below, but we set
575	575	// this in case the child exits successfully while violating
576	576	// this protocol.
577	577	eos_err = e;
578	578	break;
579	579	},
580		error.ReadFailed => return stdout.err.?,
	580	else => \|e\| return e,
581	581	};
	582	const body = client.in.take(header.bytes_len) catch unreachable;
	583
582	584	switch (header.tag) {
583	585	.zig_version => {
584	586	if (!std.mem.eql(u8, builtin.zig_version_string, body)) {

lib/compiler/Maker/Step/Run.zig+51-125

...	...	@@ -384,13 +384,23 @@ fn waitZigTest(
384	384	var sub_prog_node: ?std.Progress.Node = null;
385	385	defer if (sub_prog_node) \|n\| n.end();
386	386
	387	const stdout = multi_reader.reader(0);
	388	const stderr = multi_reader.reader(1);
	389
	390	var stdin_writer = child.stdin.?.writerStreaming(io, &.{});
	391
	392	var client: std.zig.Client = .{
	393	.in = stdout,
	394	.out = &stdin_writer.interface,
	395	};
	396
387	397	if (opt_metadata.) \|md\| {
388	398	// Previous unit test process died or was killed; we're continuing where it left off
389		requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch \|err\| return .{ .write_failed = err };
	399	requestNextTest(&client, md, &sub_prog_node) catch \|err\| return .{ .write_failed = err };
390	400	} else {
391	401	// Running unit tests normally
392	402	run.fuzz_tests.clearRetainingCapacity();
393		sendMessage(io, child.stdin.?, .query_test_metadata) catch \|err\| return .{ .write_failed = err };
	403	client.serveBodylessMessage(.query_test_metadata) catch \|err\| return .{ .write_failed = err };
394	404	}
395	405
396	406	var active_test_index: ?u32 = null;
...	...	@@ -410,10 +420,6 @@ fn waitZigTest(
410	420	.raw = .fromNanoseconds(ns),
411	421	} else null;
412	422
413		const stdout = multi_reader.reader(0);
414		const stderr = multi_reader.reader(1);
415		const Header = std.zig.Server.Message.Header;
416
417	423	while (true) {
418	424	const timeout: Io.Timeout = t: {
419	425	const opt_duration = if (active_test_index == null) response_timeout else test_timeout;
...	...	@@ -421,46 +427,20 @@ fn waitZigTest(
421	427	break :t .{ .deadline = last_update.addDuration(duration) };
422	428	};
423	429
424		// This block is exited when `stdout` contains enough bytes for a `Header`.
425		header_ready: {
426		if (stdout.buffered().len >= @sizeOf(Header)) {
427		// We already have one, no need to poll!
428		break :header_ready;
429		}
430
431		multi_reader.fill(64, timeout) catch \|err\| switch (err) {
432		error.Timeout => return .{ .timeout = .{
433		.active_test_index = active_test_index,
434		.ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds),
435		} },
436		error.EndOfStream => return .{ .no_poll = .{
437		.active_test_index = active_test_index,
438		.ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds),
439		} },
440		else => \|e\| return e,
441		};
442
443		continue;
444		}
445		// There is definitely a header available now -- read it.
446		const header = stdout.takeStruct(Header, .little) catch unreachable;
447
448		while (stdout.buffered().len < header.bytes_len) {
449		multi_reader.fill(64, timeout) catch \|err\| switch (err) {
450		error.Timeout => return .{ .timeout = .{
451		.active_test_index = active_test_index,
452		.ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds),
453		} },
454		error.EndOfStream => return .{ .no_poll = .{
455		.active_test_index = active_test_index,
456		.ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds),
457		} },
458		else => \|e\| return e,
459		};
460		}
461
462		const body = stdout.take(header.bytes_len) catch unreachable;
	430	const header = client.receiveMessageWithMultiReader(multi_reader, timeout) catch \|err\| switch (err) {
	431	error.Timeout => return .{ .timeout = .{
	432	.active_test_index = active_test_index,
	433	.ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds),
	434	} },
	435	error.EndOfStream => return .{ .no_poll = .{
	436	.active_test_index = active_test_index,
	437	.ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds),
	438	} },
	439	else => \|e\| return e,
	440	};
	441	const body = client.in.take(header.bytes_len) catch unreachable;
463	442	var body_r: std.Io.Reader = .fixed(body);
	443
464	444	switch (header.tag) {
465	445	.zig_version => {
466	446	if (!std.mem.eql(u8, builtin.zig_version_string, body)) return step.fail(
...	...	@@ -500,7 +480,7 @@ fn waitZigTest(
500	480	active_test_index = null;
501	481	last_update = .now(io, .awake);
502	482
503		requestNextTest(io, child.stdin.?, &opt_metadata.*.?, &sub_prog_node) catch \|err\| return .{ .write_failed = err };
	483	requestNextTest(&client, &opt_metadata.*.?, &sub_prog_node) catch \|err\| return .{ .write_failed = err };
504	484	},
505	485	.test_started => {
506	486	active_test_index = opt_metadata.*.?.next_index - 1;
...	...	@@ -551,7 +531,7 @@ fn waitZigTest(
551	531	md.ns_per_test[tr_hdr.index] = @intCast(last_update.durationTo(now).raw.nanoseconds);
552	532	last_update = now;
553	533
554		requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch \|err\| return .{ .write_failed = err };
	534	requestNextTest(&client, md, &sub_prog_node) catch \|err\| return .{ .write_failed = err };
555	535	},
556	536	else => {}, // ignore other messages
557	537	}
...	...	@@ -697,17 +677,18 @@ const FuzzTestRunner = struct {
697	677
698	678	for (0.., f.instances) \|id, *instance\| {
699	679	const id32: u32 = @intCast(id);
	680	var writer = instance.child.stdin.?.writerStreaming(io, &.{});
	681	const client: std.zig.Client = .{
	682	.in = undefined,
	683	.out = &writer.interface,
	684	};
700	685	(switch (f.ctx.fuzz.mode) {
701		.forever => sendRunFuzzTestMessage(
702		io,
703		instance.child.stdin.?,
	686	.forever => client.serveRunFuzzTestMessage(
704	687	run.fuzz_tests.items,
705	688	.forever,
706	689	id32,
707	690	),
708		.limit => \|limit\| sendRunFuzzTestMessage(
709		io,
710		instance.child.stdin.?,
	691	.limit => \|limit\| client.serveRunFuzzTestMessage(
711	692	run.fuzz_tests.items,
712	693	.iterations,
713	694	limit.amount,
...	...	@@ -1315,7 +1296,7 @@ pub const CachedTestMetadata = struct {
1315	1296	}
1316	1297	};
1317	1298
1318		fn requestNextTest(io: Io, in: Io.File, metadata: TestMetadata, sub_prog_node: ?std.Progress.Node) !void {
	1299	fn requestNextTest(client: std.zig.Client, metadata: TestMetadata, sub_prog_node: *?std.Progress.Node) !void {
1319	1300	while (metadata.next_index < metadata.names.len) {
1320	1301	const i = metadata.next_index;
1321	1302	metadata.next_index += 1;
...	...	@@ -1326,76 +1307,11 @@ fn requestNextTest(io: Io, in: Io.File, metadata: *TestMetadata, sub_prog_node:
1326	1307	if (sub_prog_node.*) \|n\| n.end();
1327	1308	sub_prog_node.* = metadata.prog_node.start(name, 0);
1328	1309
1329		try sendRunTestMessage(io, in, .run_test, i);
	1310	try client.serveRunTest(i);
1330	1311	return;
1331	1312	} else {
1332	1313	metadata.next_index = std.math.maxInt(u32); // indicate that all tests are done
1333		try sendMessage(io, in, .exit);
1334		}
1335		}
1336
1337		fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void {
1338		const header: std.zig.Client.Message.Header = .{
1339		.tag = tag,
1340		.bytes_len = 0,
1341		};
1342		var w = file.writerStreaming(io, &.{});
1343		w.interface.writeStruct(header, .little) catch \|err\| switch (err) {
1344		error.WriteFailed => return w.err.?,
1345		};
1346		}
1347
1348		fn sendRunTestMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag, index: u32) !void {
1349		const header: std.zig.Client.Message.Header = .{
1350		.tag = tag,
1351		.bytes_len = 4,
1352		};
1353		var w = file.writerStreaming(io, &.{});
1354		w.interface.writeStruct(header, .little) catch \|err\| switch (err) {
1355		error.WriteFailed => return w.err.?,
1356		};
1357		w.interface.writeInt(u32, index, .little) catch \|err\| switch (err) {
1358		error.WriteFailed => return w.err.?,
1359		};
1360		}
1361
1362		fn sendRunFuzzTestMessage(
1363		io: Io,
1364		file: Io.File,
1365		test_names: []const []const u8,
1366		kind: std.Build.abi.fuzz.LimitKind,
1367		amount_or_instance: u64,
1368		) !void {
1369		const header: std.zig.Client.Message.Header = .{
1370		.tag = .start_fuzzing,
1371		.bytes_len = 1 + 8 + 4 + count: {
1372		var c: u32 = @intCast(test_names.len * 4);
1373		for (test_names) \|name\| {
1374		c += @intCast(name.len);
1375		}
1376		break :count c;
1377		},
1378		};
1379		var w = file.writerStreaming(io, &.{});
1380		w.interface.writeStruct(header, .little) catch \|err\| switch (err) {
1381		error.WriteFailed => return w.err.?,
1382		};
1383		w.interface.writeByte(@backingInt(kind)) catch \|err\| switch (err) {
1384		error.WriteFailed => return w.err.?,
1385		};
1386		w.interface.writeInt(u64, amount_or_instance, .little) catch \|err\| switch (err) {
1387		error.WriteFailed => return w.err.?,
1388		};
1389		w.interface.writeInt(u32, @intCast(test_names.len), .little) catch \|err\| switch (err) {
1390		error.WriteFailed => return w.err.?,
1391		};
1392		for (test_names) \|test_name\| {
1393		w.interface.writeInt(u32, @intCast(test_name.len), .little) catch \|err\| switch (err) {
1394		error.WriteFailed => return w.err.?,
1395		};
1396		w.interface.writeAll(test_name) catch \|err\| switch (err) {
1397		error.WriteFailed => return w.err.?,
1398		};
	1314	try client.serveBodylessMessage(.exit);
1399	1315	}
1400	1316	}
1401	1317
...	...	@@ -2285,25 +2201,35 @@ fn spawnChildAndCollect(
2285	2201	assert(conf_run.flags.stdio != .inherit);
2286	2202	break :s .pipe;
2287	2203	} else switch (conf_run.flags.stdio) {
2288		.infer_from_args => if (has_side_effects) .inherit else .ignore,
	2204	.infer_from_args => if (maker.protocol_server == null and has_side_effects) .inherit else .ignore,
2289	2205	.inherit => .inherit,
2290	2206	.check => .ignore,
2291	2207	.zig_test => .pipe,
2292	2208	},
2293	2209	.stdout = if (conf_run.captured_stdout.value != null) .pipe else switch (conf_run.flags.stdio) {
2294		.infer_from_args => if (has_side_effects) .inherit else .ignore,
	2210	.infer_from_args => if (maker.protocol_server == null and has_side_effects) .inherit else .ignore,
2295	2211	.inherit => .inherit,
2296	2212	.check => if (checksContainStdout(&conf_run)) .pipe else .ignore,
2297	2213	.zig_test => .pipe,
2298	2214	},
2299	2215	.stderr = if (conf_run.captured_stderr.value != null) .pipe else switch (conf_run.flags.stdio) {
2300		.infer_from_args => if (has_side_effects) .inherit else .pipe,
2301		.inherit => .inherit,
	2216	.infer_from_args => if (maker.protocol_server == null and has_side_effects) .inherit else .pipe,
	2217	.inherit => if (maker.protocol_server == null) .inherit else .pipe,
2302	2218	.check => .pipe,
2303	2219	.zig_test => .pipe,
2304	2220	},
2305	2221	};
2306	2222
	2223	if (maker.protocol_server != null) {
	2224	if (spawn_options.stdin == .inherit) {
	2225	return step.fail(maker, "Cannot inherit stdin when running through over the build system protocol", .{});
	2226	}
	2227	if (spawn_options.stdout == .inherit) {
	2228	return step.fail(maker, "Cannot inherit stdout when running through over the build system protocol", .{});
	2229	}
	2230	assert(spawn_options.stderr != .inherit);
	2231	}
	2232
2307	2233	if (conf_run.flags.stdio == .zig_test) {
2308	2234	try setColorEnvironmentVariables(&conf_run, environ_map, graph.stderr_mode.?);
2309	2235	const started: Io.Clock.Timestamp = .now(io, .awake);

lib/compiler/objcopy.zig+3-3

...	...	@@ -214,11 +214,11 @@ fn cmdObjCopy(arena: Allocator, io: Io, args: []const []const u8) !void {
214	214	if (listen) {
215	215	var stdin_reader = Io.File.stdin().reader(io, &stdin_buffer);
216	216	var stdout_writer = Io.File.stdout().writer(io, &stdout_buffer);
217		var server = try Server.init(.{
	217	var server: Server = .{
218	218	.in = &stdin_reader.interface,
219	219	.out = &stdout_writer.interface,
220		.zig_version = builtin.zig_version_string,
221		});
	220	};
	221	try server.serveStringMessage(.zig_version, builtin.zig_version_string);
222	222
223	223	var seen_update = false;
224	224	while (true) {

lib/compiler/std-docs.zig+21-22

...	...	@@ -346,29 +346,39 @@ fn buildWasmBinary(
346	346	multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ child.stdout.?, child.stderr.? });
347	347	defer multi_reader.deinit();
348	348
349		try sendMessage(io, child.stdin.?, .update);
350		try sendMessage(io, child.stdin.?, .exit);
	349	const stdout = multi_reader.reader(0);
	350
	351	var stdin_buffer: [256]u8 = undefined;
	352	var stdin_writer = child.stdin.?.writerStreaming(io, &stdin_buffer);
	353
	354	var client: std.zig.Client = .{
	355	.in = stdout,
	356	.out = &stdin_writer.interface,
	357	};
	358
	359	try client.serveMessageHeader(.{ .tag = .update, .bytes_len = 0 });
	360	try client.serveMessageHeader(.{ .tag = .exit, .bytes_len = 0 });
	361	try client.out.flush();
351	362
352	363	var result: ?Cache.Path = null;
353	364	var result_error_bundle = std.zig.ErrorBundle.empty;
354	365
355		const stdout = multi_reader.fileReader(0);
356		const MessageHeader = std.zig.Server.Message.Header;
357
358	366	var eos_err: error{EndOfStream}!void = {};
359	367
360	368	while (true) {
361		const header = stdout.interface.takeStruct(MessageHeader, .little) catch \|err\| switch (err) {
362		error.EndOfStream => break,
363		error.ReadFailed => return stdout.err.?,
364		};
365		const body = stdout.interface.take(header.bytes_len) catch \|err\| switch (err) {
	369	const header = client.receiveMessageWithMultiReader(&multi_reader, .none) catch \|err\| switch (err) {
	370	error.Timeout => unreachable,
366	371	error.EndOfStream => \|e\| {
	372	if (client.in.bufferedLen() == 0) break;
	373	// Better to report the crash with stderr below, but we set
	374	// this in case the child exits successfully while violating
	375	// this protocol.
367	376	eos_err = e;
368	377	break;
369	378	},
370		error.ReadFailed => return stdout.err.?,
	379	else => \|e\| return e,
371	380	};
	381	const body = client.in.take(header.bytes_len) catch unreachable;
372	382
373	383	switch (header.tag) {
374	384	.zig_version => {
...	...	@@ -435,17 +445,6 @@ fn buildWasmBinary(
435	445	};
436	446	}
437	447
438		fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void {
439		const header: std.zig.Client.Message.Header = .{
440		.tag = tag,
441		.bytes_len = 0,
442		};
443		var w = file.writer(io, &.{});
444		w.interface.writeStruct(header, .little) catch \|err\| switch (err) {
445		error.WriteFailed => return w.err.?,
446		};
447		}
448
449	448	fn openBrowserTab(io: Io, url: []const u8) !void {
450	449	// Until https://github.com/ziglang/zig/issues/19205 is implemented, we
451	450	// spawn and then leak a concurrent task for this child process.

lib/compiler/test_runner.zig+3-3

...	...	@@ -78,11 +78,11 @@ fn mainServer(init: std.process.Init.Minimal) !void {
78	78	@disableInstrumentation();
79	79	stdin_reader = .initStreaming(.stdin(), runner_threaded_io, &stdin_buffer);
80	80	stdout_writer = .initStreaming(.stdout(), runner_threaded_io, &stdout_buffer);
81		var server = try std.zig.Server.init(.{
	81	var server: std.zig.Server = .{
82	82	.in = &stdin_reader.interface,
83	83	.out = &stdout_writer.interface,
84		.zig_version = builtin.zig_version_string,
85		});
	84	};
	85	try server.serveStringMessage(.zig_version, builtin.zig_version_string);
86	86
87	87	while (true) {
88	88	const hdr = try server.receiveMessage();

lib/std/Io/Reader.zig+3-5

...	...	@@ -718,7 +718,7 @@ pub inline fn readSliceEndian(
718	718	endian: std.builtin.Endian,
719	719	) Error!void {
720	720	try readSliceAll(r, @ptrCast(buffer));
721		if (native_endian != endian) for (buffer) \|*elem\| std.mem.byteSwapAllFields(Elem, elem);
	721	if (native_endian != endian) std.mem.byteSwapAllElements(Elem, buffer);
722	722	}
723	723
724	724	pub const ReadAllocError = Error \|\| Allocator.Error;
...	...	@@ -734,8 +734,7 @@ pub inline fn readSliceEndianAlloc(
734	734	) ReadAllocError![]Elem {
735	735	const dest = try allocator.alloc(Elem, len);
736	736	errdefer allocator.free(dest);
737		try readSliceAll(r, @ptrCast(dest));
738		if (native_endian != endian) for (dest) \|*elem\| std.mem.byteSwapAllFields(Elem, elem);
	737	try r.readSliceEndian(Elem, dest, endian);
739	738	return dest;
740	739	}
741	740
...	...	@@ -1227,8 +1226,7 @@ pub inline fn takeStruct(r: *Reader, comptime T: type, endian: std.builtin.Endia
1227	1226	.auto => @compileError("ill-defined memory layout"),
1228	1227	.@"extern" => {
1229	1228	var res: T = undefined;
1230		try r.readSliceAll(std.mem.asBytes(&res));
1231		if (native_endian != endian) std.mem.byteSwapAllFields(T, &res);
	1229	try r.readSliceEndian(T, (&res)[0..1], endian);
1232	1230	return res;
1233	1231	},
1234	1232	.@"packed" => {

lib/std/mem.zig+66-43

...	...	@@ -2215,33 +2215,54 @@ test writeVarPackedInt {
2215	2215	try testing.expectEqual(T{ .a = 1, .b = value, .c = 4 }, st);
2216	2216	}
2217	2217
2218		/// Swap the byte order of all the members of the fields of a struct
2219		/// (Changing their endianness)
2220		pub fn byteSwapAllFields(comptime S: type, ptr: *S) void {
2221		byteSwapAllFieldsAligned(S, .of(S), ptr);
	2218	/// Deprecated: use `byteSwap` instead.
	2219	pub const byteSwapAllFields = byteSwap;
	2220
	2221	/// Deprecated: use `byteSwapAligned` instead.
	2222	pub const byteSwapAllFieldsAligned = byteSwapAligned;
	2223
	2224	/// Reverses the byte order.
	2225	/// Handles structs, unions, arrays, enums, floats, and integers recursively.
	2226	/// The order of extern struct fields and array elements remains unchanged and
	2227	/// will be byte swapped recursively.
	2228	/// Useful for converting between little-endian and big-endian representations.
	2229	pub fn byteSwap(comptime S: type, ptr: *S) void {
	2230	byteSwapAligned(S, .of(S), ptr);
2222	2231	}
2223	2232
2224		/// Swap the byte order of all the members of the fields of a struct
2225		/// (Changing their endianness)
2226		pub fn byteSwapAllFieldsAligned(comptime S: type, comptime a: Alignment, ptr: *align(a.toByteUnits()) S) void {
	2233	/// Reverses the byte order.
	2234	/// Handles structs, unions, arrays, enums, floats, and integers recursively.
	2235	/// The order of extern struct fields and array elements remains unchanged and
	2236	/// will be byte swapped recursively.
	2237	/// Useful for converting between little-endian and big-endian representations.
	2238	pub fn byteSwapAligned(
	2239	comptime S: type,
	2240	comptime a: Alignment,
	2241	ptr: *align(a.toByteUnits()) S,
	2242	) void {
2227	2243	switch (@typeInfo(S)) {
2228	2244	.@"struct" => \|@"struct"\| {
2229	2245	if (@"struct".backing_integer) \|Int\| {
2230	2246	ptr.* = @bitCast(@byteSwap(@as(Int, @bitCast(ptr.*))));
2231		} else inline for (@"struct".field_types, @"struct".field_names, @"struct".field_attrs) \|f_type, f_name, f_attr\| {
2232		switch (@typeInfo(f_type)) {
2233		.@"struct" => byteSwapAllFieldsAligned(f_type, .fromByteUnits(f_attr.@"align" orelse @alignOf(f_type)), &@field(ptr, f_name)),
2234		.@"union", .array => byteSwapAllFieldsAligned(f_type, .fromByteUnits(f_attr.@"align" orelse @alignOf(f_type)), &@field(ptr, f_name)),
2235		.@"enum" => {
2236		@field(ptr, f_name) = @fromBackingInt(@intCast(@byteSwap(@backingInt(@field(ptr, f_name)))));
2237		},
2238		.bool => {},
2239		.float => \|float\| {
2240		@field(ptr, f_name) = @bitCast(@byteSwap(@as(@Int(.unsigned, float.bits), @bitCast(@field(ptr, f_name)))));
2241		},
2242		else => {
2243		@field(ptr, f_name) = @byteSwap(@field(ptr, f_name));
2244		},
	2247	} else {
	2248	if (@"struct".layout != .@"extern") {
	2249	@compileError("byteSwapAligned expects a packed or extern struct");
	2250	}
	2251	inline for (@"struct".field_types, @"struct".field_names, @"struct".field_attrs) \|f_type, f_name, f_attr\| {
	2252	switch (@typeInfo(f_type)) {
	2253	.@"struct" => byteSwapAligned(f_type, .fromByteUnits(f_attr.@"align" orelse @alignOf(f_type)), &@field(ptr, f_name)),
	2254	.@"union", .array => byteSwapAligned(f_type, .fromByteUnits(f_attr.@"align" orelse @alignOf(f_type)), &@field(ptr, f_name)),
	2255	.@"enum" => {
	2256	@field(ptr, f_name) = @fromBackingInt(@byteSwap(@backingInt(@field(ptr, f_name))));
	2257	},
	2258	.bool => {},
	2259	.float => \|float\| {
	2260	@field(ptr, f_name) = @bitCast(@byteSwap(@as(@Int(.unsigned, float.bits), @bitCast(@field(ptr, f_name)))));
	2261	},
	2262	else => {
	2263	@field(ptr, f_name) = @byteSwap(@field(ptr, f_name));
	2264	},
	2265	}
2245	2266	}
2246	2267	}
2247	2268	},
...	...	@@ -2249,7 +2270,7 @@ pub fn byteSwapAllFieldsAligned(comptime S: type, comptime a: Alignment, ptr: *a
2249	2270	ptr.* = @bitCast(@byteSwap(@as(Int, @bitCast(ptr.*))));
2250	2271	} else {
2251	2272	if (@"union".layout != .@"extern") {
2252		@compileError("byteSwapAllFields expects a packed or extern union");
	2273	@compileError("byteSwapAligned expects a packed or extern union");
2253	2274	}
2254	2275
2255	2276	const first_size = @bitSizeOf(@"union".field_types[0]);
...	...	@@ -2266,13 +2287,21 @@ pub fn byteSwapAllFieldsAligned(comptime S: type, comptime a: Alignment, ptr: *a
2266	2287	.array => \|array\| {
2267	2288	byteSwapAllElements(array.child, ptr);
2268	2289	},
	2290	.@"enum" => {
	2291	ptr.* = @fromBackingInt(@byteSwap(@backingInt(ptr.*)));
	2292	},
	2293	.bool => {},
	2294	.float => \|float\| {
	2295	const int_repr: @Int(.unsigned, float.bits) = @bitCast(ptr.*);
	2296	ptr.* = @bitCast(@byteSwap(int_repr));
	2297	},
2269	2298	else => {
2270	2299	ptr.* = @byteSwap(ptr.*);
2271	2300	},
2272	2301	}
2273	2302	}
2274	2303
2275		test byteSwapAllFields {
	2304	test byteSwap {
2276	2305	const T = extern struct {
2277	2306	f0: u8,
2278	2307	f1: u16,
...	...	@@ -2304,6 +2333,9 @@ test byteSwapAllFields {
2304	2333	} align(4),
2305	2334	f2: u32,
2306	2335	};
	2336	const E = enum(u32) {
	2337	_,
	2338	};
2307	2339	var s = T{
2308	2340	.f0 = 0x12,
2309	2341	.f1 = 0x1234,
...	...	@@ -2327,10 +2359,14 @@ test byteSwapAllFields {
2327	2359	.f1 = .{ .f0 = 0x123456789ABCDEF0 },
2328	2360	.f2 = 0x87654321,
2329	2361	};
2330		byteSwapAllFields(T, &s);
2331		byteSwapAllFields(K, &k);
2332		byteSwapAllFields(P, &p);
2333		byteSwapAllFields(A, &a);
	2362	var e: E = @fromBackingInt(0x12345678);
	2363	var f: f32 = @bitCast(@as(u32, 0x4640e400));
	2364	byteSwap(T, &s);
	2365	byteSwap(K, &k);
	2366	byteSwap(P, &p);
	2367	byteSwap(A, &a);
	2368	byteSwap(E, &e);
	2369	byteSwap(f32, &f);
2334	2370	try std.testing.expectEqual(T{
2335	2371	.f0 = 0x12,
2336	2372	.f1 = 0x3412,
...	...	@@ -2354,28 +2390,15 @@ test byteSwapAllFields {
2354	2390	.f1 = .{ .f0 = 0xF0DEBC9A78563412 },
2355	2391	.f2 = 0x21436587,
2356	2392	}, a);
	2393	try std.testing.expectEqual(@as(E, @fromBackingInt(0x78563412)), e);
	2394	try std.testing.expectEqual(@as(f32, @bitCast(@as(u32, 0x00e44046))), f);
2357	2395	}
2358	2396
2359	2397	/// Reverses the byte order of all elements in a slice.
2360	2398	/// Handles structs, unions, arrays, enums, floats, and integers recursively.
2361	2399	/// Useful for converting between little-endian and big-endian representations.
2362	2400	pub fn byteSwapAllElements(comptime Elem: type, slice: []Elem) void {
2363		for (slice) \|*elem\| {
2364		switch (@typeInfo(@TypeOf(elem.*))) {
2365		.@"struct", .@"union", .array => byteSwapAllFields(@TypeOf(elem.*), elem),
2366		.@"enum" => {
2367		elem.* = @fromBackingInt(@intCast(@byteSwap(@backingInt(elem.*))));
2368		},
2369		.bool => {},
2370		.float => \|float\| {
2371		const int_repr: @Int(.unsigned, float.bits) = @bitCast(elem.*);
2372		elem.* = @bitCast(@byteSwap(int_repr));
2373		},
2374		else => {
2375		elem.* = @byteSwap(elem.*);
2376		},
2377		}
2378		}
	2401	for (slice) \|*elem\| byteSwap(Elem, elem);
2379	2402	}
2380	2403
2381	2404	/// Returns an iterator that iterates over the slices of `buffer` that are not

lib/std/zig.zig+46-58

...	...	@@ -1658,31 +1658,32 @@ pub fn buildExeSubprocess(
1658	1658	};
1659	1659	defer child.kill(io);
1660	1660
1661		var stderr_task = io.concurrent(readStreamAlloc, .{ gpa, io, child.stderr.?, .unlimited }) catch
1662		@panic("TODO use multireader instead");
1663		defer if (stderr_task.cancel(io)) \|slice\| gpa.free(slice) else \|_\| {};
1664
1665		var stdout_buffer: [512]u8 = undefined;
1666		var stdout_reader: Io.File.Reader = .initStreaming(child.stdout.?, io, &stdout_buffer);
1667		const stdout = &stdout_reader.interface;
1668
1669		{
1670		var w = child.stdin.?.writer(io, &.{});
1671		w.interface.writeStruct(Client.Message.Header{ .tag = .update, .bytes_len = 0 }, .little) catch \|err\| switch (err) {
1672		error.WriteFailed => {
1673		log.err("{t} writing to command: {f}", .{ w.err.?, cmd });
1674		return error.AlreadyReported;
1675		},
1676		};
1677		w.interface.writeStruct(Client.Message.Header{ .tag = .exit, .bytes_len = 0 }, .little) catch \|err\| switch (err) {
1678		error.WriteFailed => {
1679		log.err("{t} writing to command: {f}", .{ w.err.?, cmd });
1680		return error.AlreadyReported;
1681		},
1682		};
1683		}
	1661	var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined;
	1662	var multi_reader: Io.File.MultiReader = undefined;
	1663	multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ child.stdout.?, child.stderr.? });
	1664	defer multi_reader.deinit();
	1665	const stdout = multi_reader.reader(0);
	1666	const stderr = multi_reader.reader(1);
	1667
	1668	var stdin_buffer: [8]u8 = undefined;
	1669	var stdin_writer = child.stdin.?.writerStreaming(io, &stdin_buffer);
	1670
	1671	var client: Client = .{
	1672	.in = stdout,
	1673	.out = &stdin_writer.interface,
	1674	};
1684	1675
1685		const Header = Server.Message.Header;
	1676	(blk: {
	1677	client.serveMessageHeader(.{ .tag = .update, .bytes_len = 0 }) catch \|err\| break :blk err;
	1678	client.serveMessageHeader(.{ .tag = .exit, .bytes_len = 0 }) catch \|err\| break :blk err;
	1679	client.out.flush() catch \|err\| break :blk err;
	1680	}) catch \|err\| switch (err) {
	1681	error.WriteFailed => {
	1682	if (stdin_writer.err.? == error.Canceled) return error.Canceled;
	1683	log.err("{t} writing to command: {f}", .{ stdin_writer.err.?, cmd });
	1684	return error.AlreadyReported;
	1685	},
	1686	};
1686	1687
1687	1688	var result: ?Cache.Path = null;
1688	1689	defer if (result) \|r\| gpa.free(r.sub_path);
...	...	@@ -1690,33 +1691,29 @@ pub fn buildExeSubprocess(
1690	1691	var result_error_bundle: ErrorBundle = .empty;
1691	1692	defer result_error_bundle.deinit(gpa);
1692	1693
1693		var body_buffer: std.ArrayList(u8) = .empty;
1694		defer body_buffer.deinit(gpa);
1695
1696	1694	var received_fs_inputs = false;
1697	1695	var cache_hit = false;
1698	1696
	1697	var eos_err: error{EndOfStream}!void = {};
	1698
1699	1699	while (true) {
1700		const header = stdout.takeStruct(Header, .little) catch \|err\| switch (err) {
1701		error.ReadFailed => {
1702		log.err("{t} reading from command: {f}", .{ stdout_reader.err.?, cmd });
1703		return error.AlreadyReported;
1704		},
1705		error.EndOfStream => break,
1706		};
1707		body_buffer.clearRetainingCapacity();
1708		stdout.appendExact(gpa, &body_buffer, header.bytes_len) catch \|err\| switch (err) {
1709		error.ReadFailed => {
1710		log.err("{t} reading from command: {f}", .{ stdout_reader.err.?, cmd });
1711		return error.AlreadyReported;
	1700	const header = client.receiveMessageWithMultiReader(&multi_reader, .none) catch \|err\| switch (err) {
	1701	error.Timeout => unreachable,
	1702	error.EndOfStream => \|e\| {
	1703	if (client.in.bufferedLen() == 0) break;
	1704	// Better to report the crash with stderr below, but we set
	1705	// this in case the child exits successfully while violating
	1706	// this protocol.
	1707	eos_err = e;
	1708	break;
1712	1709	},
1713		error.OutOfMemory => \|e\| return e,
1714		error.EndOfStream => {
1715		log.err("unexpected end of stream from command: {f}", .{cmd});
	1710	error.Canceled, error.OutOfMemory => \|e\| return e,
	1711	else => \|e\| {
	1712	log.err("{t} reading from command: {f}", .{ e, cmd });
1716	1713	return error.AlreadyReported;
1717	1714	},
1718	1715	};
1719		const body = body_buffer.items;
	1716	const body = stdout.take(header.bytes_len) catch unreachable;
1720	1717
1721	1718	switch (header.tag) {
1722	1719	.zig_version => {
...	...	@@ -1767,16 +1764,15 @@ pub fn buildExeSubprocess(
1767	1764	}
1768	1765	}
1769	1766
1770		const stderr_contents = stderr_task.await(io) catch \|err\| switch (err) {
1771		error.Canceled, error.OutOfMemory => \|e\| return e,
1772		else => \|e\| c: {
1773		log.warn("{t} reading stderr from command: {f}", .{ e, cmd });
1774		break :c "";
1775		},
1776		};
	1767	const stderr_contents = stderr.buffered();
1777	1768	if (stderr_contents.len > 0)
1778	1769	log.warn("unexpected stderr from {s} command:\n{s}", .{ options.argv[0], stderr_contents });
1779	1770
	1771	eos_err catch {
	1772	log.err("unexpected end of stream from command: {f}", .{cmd});
	1773	return error.AlreadyReported;
	1774	};
	1775
1780	1776	// Send EOF to stdin.
1781	1777	child.stdin.?.close(io);
1782	1778	child.stdin = null;
...	...	@@ -1834,14 +1830,6 @@ pub fn buildExeSubprocess(
1834	1830	};
1835	1831	}
1836	1832
1837		fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8 {
1838		var file_reader: Io.File.Reader = .initStreaming(file, io, &.{});
1839		return file_reader.interface.allocRemaining(gpa, limit) catch \|err\| switch (err) {
1840		error.ReadFailed => return file_reader.err.?,
1841		else => \|e\| return e,
1842		};
1843		}
1844
1845	1833	test {
1846	1834	_ = Ast;
1847	1835	_ = AstRlAnnotate;

lib/std/zig/Client.zig+126-2

...	...	@@ -1,3 +1,18 @@
	1	const Client = @This();
	2
	3	const std = @import("std");
	4	const Io = std.Io;
	5	const Allocator = std.mem.Allocator;
	6	const assert = std.debug.assert;
	7	const Configuration = std.Build.Configuration;
	8	const OutMessage = std.zig.Client.Message;
	9	const InMessage = std.zig.Server.Message;
	10	const Reader = Io.Reader;
	11	const Writer = Io.Writer;
	12
	13	in: *Reader,
	14	out: *Writer,
	15
1	16	pub const Message = struct {
2	17	pub const Header = extern struct {
3	18	tag: Tag,
...	...	@@ -46,11 +61,120 @@ pub const Message = struct {
46	61	/// The message body has the same format as in Server.
47	62	new_fuzz_input,
48	63
	64	/// Asks the server to run a list of steps.
	65	/// Body is a `BuildSteps`.
	66	/// This message only applies to the build system protocol.
	67	bsp_build_steps = 0x80000000,
	68
49	69	_,
50	70	};
51	71
	72	/// Trailing:
	73	/// * step_indices: [step_count]std.Build.Configuration.Step.Index,
	74	pub const BuildSteps = extern struct {
	75	step_count: u32,
	76	flags: Flags,
	77
	78	pub const Flags = packed struct(u32) {
	79	/// Can only be enabled when the server declared support for file
	80	/// watching.
	81	watch: bool,
	82	reserved: u31 = 0,
	83	};
	84	};
	85
52	86	comptime {
53		const std = @import("std");
54		std.debug.assert(@sizeOf(std.Build.abi.fuzz.LimitKind) == 1);
	87	assert(@sizeOf(std.Build.abi.fuzz.LimitKind) == 1);
55	88	}
56	89	};
	90
	91	pub fn receiveMessage(c: *const Client) Reader.Error!InMessage.Header {
	92	return c.in.takeStruct(InMessage.Header, .little);
	93	}
	94
	95	/// Assumes that `c.in` is a reader in `multi_reader`.
	96	/// Guarantees that the response body will be buffered in `c.in` on success.
	97	pub fn receiveMessageWithMultiReader(
	98	c: *Client,
	99	multi_reader: *Io.File.MultiReader,
	100	timeout: Io.Timeout,
	101	) (Io.File.MultiReader.Error \|\| Io.Timeout.Error)!InMessage.Header {
	102	while (c.in.bufferedLen() < @sizeOf(InMessage.Header)) {
	103	multi_reader.fill(64, timeout) catch \|err\| switch (err) {
	104	error.Canceled,
	105	error.Timeout,
	106	error.ConcurrencyUnavailable,
	107	error.EndOfStream,
	108	=> \|e\| return e,
	109	};
	110	}
	111	const header = c.in.takeStruct(InMessage.Header, .little) catch unreachable;
	112	while (c.in.bufferedLen() < header.bytes_len) {
	113	try multi_reader.fill(header.bytes_len - c.in.bufferedLen(), timeout);
	114	}
	115	try multi_reader.checkAnyError();
	116	return header;
	117	}
	118
	119	/// Don't forget to flush!
	120	pub fn serveMessageHeader(c: *const Client, header: OutMessage.Header) Writer.Error!void {
	121	try c.out.writeStruct(header, .little);
	122	}
	123
	124	pub fn serveBodylessMessage(c: *const Client, tag: OutMessage.Tag) Writer.Error!void {
	125	try c.serveMessageHeader(.{ .tag = tag, .bytes_len = 0 });
	126	try c.out.flush();
	127	}
	128
	129	pub fn serveRunTest(c: *const Client, index: u32) !void {
	130	try c.serveMessageHeader(.{
	131	.tag = .run_test,
	132	.bytes_len = @sizeOf(u32),
	133	});
	134	try c.out.writeInt(u32, index, .little);
	135	try c.out.flush();
	136	}
	137
	138	pub fn serveRunFuzzTestMessage(
	139	c: *const Client,
	140	test_names: []const []const u8,
	141	kind: std.Build.abi.fuzz.LimitKind,
	142	amount_or_instance: u64,
	143	) !void {
	144	try c.serveMessageHeader(.{
	145	.tag = .start_fuzzing,
	146	.bytes_len = 1 + 8 + 4 + count: {
	147	var bytes_len: u32 = @intCast(test_names.len * 4);
	148	for (test_names) \|name\| {
	149	bytes_len += @intCast(name.len);
	150	}
	151	break :count bytes_len;
	152	},
	153	});
	154	try c.out.writeByte(@backingInt(kind));
	155	try c.out.writeInt(u64, amount_or_instance, .little);
	156	try c.out.writeInt(u32, @intCast(test_names.len), .little);
	157	for (test_names) \|test_name\| {
	158	try c.out.writeInt(u32, @intCast(test_name.len), .little);
	159	try c.out.writeAll(test_name);
	160	}
	161	try c.out.flush();
	162	}
	163
	164	pub fn serveBuildSteps(
	165	c: *const Client,
	166	steps: []const Configuration.Step.Index,
	167	flags: OutMessage.BuildSteps.Flags,
	168	) !void {
	169	try c.serveMessageHeader(.{
	170	.tag = .bsp_build_steps,
	171	.bytes_len = @intCast(@sizeOf(OutMessage.BuildSteps) + steps.len * @sizeOf(Configuration.Step.Index)),
	172	});
	173	const body: OutMessage.BuildSteps = .{
	174	.step_count = @intCast(steps.len),
	175	.flags = flags,
	176	};
	177	try c.out.writeStruct(body, .little);
	178	try c.out.writeSliceEndian(Configuration.Step.Index, steps, .little);
	179	try c.out.flush();
	180	}

lib/std/zig/Server.zig+66-19

...	...	@@ -1,12 +1,8 @@
1	1	const Server = @This();
2	2
3		const builtin = @import("builtin");
4
5	3	const std = @import("std");
6	4	const Allocator = std.mem.Allocator;
7	5	const assert = std.debug.assert;
8		const native_endian = builtin.target.cpu.arch.endian();
9		const need_bswap = native_endian != .little;
10	6	const Cache = std.Build.Cache;
11	7	const OutMessage = std.zig.Server.Message;
12	8	const InMessage = std.zig.Client.Message;
...	...	@@ -16,6 +12,14 @@ const Writer = std.Io.Writer;
16	12	in: *Reader,
17	13	out: *Writer,
18	14
	15	/// The ABI version of the build system protocol. Will be bumped whenever a
	16	/// backwards incompatible changes to the protocol is made.
	17	///
	18	/// Does not apply to the internal compiler protocol or test runner.
	19	///
	20	/// See `version` in `Message.Handshake`.
	21	pub const build_system_version: u32 = 1;
	22
19	23	pub const Message = struct {
20	24	pub const Header = extern struct {
21	25	tag: Tag,
...	...	@@ -70,9 +74,62 @@ pub const Message = struct {
70	74	/// Body is a TimeReport.
71	75	time_report,
72	76
	77	/// The first message sent by the server over the build system protocol.
	78	/// Body is a `Handshake`.
	79	/// This message only applies to the build system protocol.
	80	bsp_handshake = 0x80000000,
	81	/// Notifies that a new configuration file is available.
	82	/// Body is a cwd relative path to the configuration file.
	83	/// This message only applies to the build system protocol.
	84	bsp_configuration,
	85	/// Does not have a body.
	86	/// This message only applies to the build system protocol.
	87	bsp_build_started,
	88	/// Does not have a body.
	89	/// This message only applies to the build system protocol.
	90	bsp_build_completed,
	91	/// Body is a `Configuration.Step.Index`.
	92	/// This message only applies to the build system protocol.
	93	bsp_step_started,
	94	/// Body is a `BuildStepCompleted`.
	95	/// This message only applies to the build system protocol.
	96	bsp_step_completed,
	97
73	98	_,
74	99	};
75	100
	101	/// Trailing:
	102	/// * base_paths: BasePaths,
	103	pub const Handshake = extern struct {
	104	/// See `build_system_version`.
	105	version: u32,
	106	flags: Flags,
	107
	108	pub const Flags = packed struct(u32) {
	109	file_system_watch_supported: bool,
	110	_: u31 = 0,
	111	};
	112	};
	113
	114	/// Trailing:
	115	/// * error_bundle: ErrorBundle,
	116	pub const BuildStepCompleted = extern struct {
	117	step_index: std.Build.Configuration.Step.Index,
	118	status: Status,
	119	error_bundle: ErrorBundle,
	120	// TODO result_error_msgs
	121	// TODO result_stderr
	122	// TODO result_peak_rss
	123	// TODO result_duration_ns
	124
	125	pub const Status = enum(u32) {
	126	success,
	127	failure,
	128	skipped,
	129	skipped_oom,
	130	};
	131	};
	132
76	133	pub const PathPrefix = enum(u8) {
77	134	cwd,
78	135	zig_lib,
...	...	@@ -140,21 +197,6 @@ pub const Message = struct {
140	197	};
141	198	};
142	199
143		pub const Options = struct {
144		in: *Reader,
145		out: *Writer,
146		zig_version: []const u8,
147		};
148
149		pub fn init(options: Options) !Server {
150		var s: Server = .{
151		.in = options.in,
152		.out = options.out,
153		};
154		try s.serveStringMessage(.zig_version, options.zig_version);
155		return s;
156		}
157
158	200	pub fn receiveMessage(s: *Server) !InMessage.Header {
159	201	return s.in.takeStruct(InMessage.Header, .little);
160	202	}
...	...	@@ -183,6 +225,11 @@ pub fn serveMessageHeader(s: *const Server, header: OutMessage.Header) !void {
183	225	try s.out.writeStruct(header, .little);
184	226	}
185	227
	228	pub fn serveBodylessMessage(s: *const Server, tag: OutMessage.Tag) Writer.Error!void {
	229	try s.serveMessageHeader(.{ .tag = tag, .bytes_len = 0 });
	230	try s.out.flush();
	231	}
	232
186	233	pub fn serveU32Message(s: *const Server, tag: OutMessage.Tag, int: u32) !void {
187	234	try serveMessageHeader(s, .{
188	235	.tag = tag,

src/Compilation.zig+12-8

...	...	@@ -6028,26 +6028,30 @@ fn spawnZigRc(
6028	6028	multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ child.stdout.?, child.stderr.? });
6029	6029	defer multi_reader.deinit();
6030	6030
6031		const stdout = multi_reader.fileReader(0);
6032		const MessageHeader = std.zig.Server.Message.Header;
	6031	const stdout = multi_reader.reader(0);
6033	6032
6034	6033	var eos_err: error{EndOfStream}!void = {};
6035	6034
	6035	var client: std.zig.Client = .{
	6036	.in = stdout,
	6037	.out = undefined,
	6038	};
	6039
6036	6040	while (true) {
6037		const header = stdout.interface.takeStruct(MessageHeader, .little) catch \|err\| switch (err) {
6038		error.EndOfStream => break,
6039		error.ReadFailed => return stdout.err.?,
6040		};
6041		const body = stdout.interface.take(header.bytes_len) catch \|err\| switch (err) {
	6041	const header = client.receiveMessageWithMultiReader(&multi_reader, .none) catch \|err\| switch (err) {
	6042	error.Timeout => unreachable,
6042	6043	error.EndOfStream => \|e\| {
	6044	if (client.in.bufferedLen() == 0) break;
6043	6045	// Better to report the crash with stderr below, but we set
6044	6046	// this in case the child exits successfully while violating
6045	6047	// this protocol.
6046	6048	eos_err = e;
6047	6049	break;
6048	6050	},
6049		error.ReadFailed => return stdout.err.?,
	6051	else => \|e\| return e,
6050	6052	};
	6053	const body = client.in.take(header.bytes_len) catch unreachable;
	6054
6051	6055	switch (header.tag) {
6052	6056	// We expect exactly one ErrorBundle, and if any error_bundle header is
6053	6057	// sent then it's a fatal error.

src/main.zig+2-5

...	...	@@ -4297,11 +4297,8 @@ fn serve(
4297	4297	const gpa = comp.gpa;
4298	4298	const io = comp.io;
4299	4299
4300		var server = try Server.init(.{
4301		.in = in,
4302		.out = out,
4303		.zig_version = build_options.version,
4304		});
	4300	var server: Server = .{ .in = in, .out = out };
	4301	try server.serveStringMessage(.zig_version, build_options.version);
4305	4302
4306	4303	var child_pid: ?std.process.Child.Id = null;
4307	4304

test/standalone/build.zig+1

...	...	@@ -31,6 +31,7 @@ pub fn build(b: *std.Build) void {
31	31	const tools_target = b.resolveTargetQuery(.{});
32	32	for ([_][]const u8{
33	33	// Alphabetically sorted. No need to build `tools/spirv/grammar.zig`.
	34	"../../tools/bsp.zig",
34	35	"../../tools/check_mingw.zig",
35	36	"../../tools/dump-cov.zig",
36	37	"../../tools/fetch_them_macos_headers.zig",

tools/bsp.zig created+242

...	...	@@ -0,0 +1,242 @@
	1	//! CLI tool to interface with the build system protocol (zig build --listen=-)
	2
	3	const std = @import("std");
	4	const Io = std.Io;
	5	const Allocator = std.mem.Allocator;
	6	const Configuration = std.Build.Configuration;
	7	const Client = std.zig.Client;
	8	const Server = std.zig.Server;
	9	const log = std.log.scoped(.bsp);
	10
	11	pub fn main(init: std.process.Init) !void {
	12	const io = init.io;
	13	const gpa = init.gpa;
	14	const arena = init.arena.allocator();
	15
	16	var maker_args: std.ArrayList([]const u8) = .empty;
	17
	18	const args = try init.minimal.args.toSlice(arena);
	19	for (args[1..]) \|arg\| {
	20	try maker_args.append(arena, try arena.dupe(u8, arg));
	21	}
	22	if (maker_args.items.len < 1) try maker_args.append(arena, "zig");
	23	if (maker_args.items.len < 2) try maker_args.append(arena, "build");
	24	if (!std.mem.eql(u8, maker_args.last().?.*, "--listen=-")) try maker_args.append(arena, "--listen=-");
	25
	26	log.debug("cmd: {f}", .{std.zig.SubprocessCommand{
	27	.argv = maker_args.items,
	28	}});
	29
	30	var child_process = std.process.spawn(io, .{
	31	.argv = maker_args.items,
	32	.stdin = .pipe,
	33	.stdout = .pipe,
	34	.stderr = .pipe,
	35	}) catch \|err\| std.debug.panic("failed to spawn process: {}", .{err});
	36	errdefer child_process.kill(io);
	37
	38	var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined;
	39	var multi_reader: Io.File.MultiReader = undefined;
	40	defer multi_reader.deinit();
	41	multi_reader.init(
	42	gpa,
	43	io,
	44	multi_reader_buffer.toStreams(),
	45	&.{ child_process.stdout.?, child_process.stderr.? },
	46	);
	47	const client_stdout = multi_reader.reader(0);
	48	const client_stderr = multi_reader.reader(1);
	49
	50	var client_stdout_buffer: [256]u8 = undefined;
	51	var client_stdout_writer = child_process.stdin.?.writerStreaming(io, &client_stdout_buffer);
	52
	53	var client: Client = .{
	54	.in = client_stdout,
	55	.out = &client_stdout_writer.interface,
	56	};
	57
	58	const err = blk: {
	59	const handshake: Server.Message.Handshake = handshake: {
	60	const header = client.receiveMessageWithMultiReader(&multi_reader, .none) catch \|err\| switch (err) {
	61	error.Canceled, error.ConcurrencyUnavailable => \|e\| return e,
	62	error.Timeout => unreachable,
	63	else => \|e\| {
	64	log.err("failed to receive message: {t}", .{err});
	65	break :blk e;
	66	},
	67	};
	68	const body = client_stdout.take(header.bytes_len) catch unreachable;
	69	log.debug("received {f} ({d} bytes)", .{ fmtEnum(header.tag), body.len });
	70
	71	if (header.tag != .bsp_handshake) {
	72	log.err("received unexpected message: {f}", .{fmtEnum(header.tag)});
	73	return error.UnexpectedMessage;
	74	}
	75
	76	var r: Io.Reader = .fixed(body);
	77	break :handshake try r.takeStruct(Server.Message.Handshake, .little);
	78	};
	79	_ = handshake;
	80
	81	var conf_arena_allocator: std.heap.ArenaAllocator = .init(gpa);
	82	defer conf_arena_allocator.deinit();
	83	const conf_arena = conf_arena_allocator.allocator();
	84
	85	const configuration = configuration: {
	86	const header = client.receiveMessageWithMultiReader(&multi_reader, .none) catch \|err\| switch (err) {
	87	error.Canceled, error.ConcurrencyUnavailable => \|e\| return e,
	88	error.Timeout => unreachable,
	89	else => \|e\| {
	90	log.err("failed to receive message: {t}", .{err});
	91	break :blk e;
	92	},
	93	};
	94	const body = client_stdout.take(header.bytes_len) catch unreachable;
	95	log.debug("received {t} ({d} bytes)", .{ header.tag, body.len });
	96
	97	if (header.tag != .bsp_configuration) {
	98	log.err("received unexpected message: {f}", .{fmtEnum(header.tag)});
	99	return error.UnexpectedMessage;
	100	}
	101
	102	const configuration_path = body;
	103	var file = Io.Dir.cwd().openFile(io, configuration_path, .{}) catch \|err\|
	104	std.debug.panic("failed to open configuration file {q}: {t}", .{ configuration_path, err });
	105	defer file.close(io);
	106	break :configuration Configuration.loadFile(conf_arena, io, file) catch \|err\|
	107	std.debug.panic("failed to load configuration file {q}: {t}", .{ configuration_path, err });
	108	};
	109	const c = &configuration;
	110
	111	var top_level_steps: std.array_hash_map.String(Configuration.Step.Index) = .empty;
	112	defer top_level_steps.deinit(gpa);
	113
	114	for (c.steps, 0..) \|*conf_step, step_index_usize\| {
	115	if (conf_step.owner != .root) continue;
	116	const step_index: Configuration.Step.Index = @fromBackingInt(@intCast(step_index_usize));
	117	const flags = conf_step.flags(c);
	118	if (flags.tag != .top_level) continue;
	119	const name = step_index.ptr(c).name.slice(c);
	120	try top_level_steps.putNoClobber(gpa, name, step_index);
	121	}
	122
	123	std.debug.print("Steps:\n", .{});
	124	for (top_level_steps.keys()) \|name\| {
	125	std.debug.print(" - {q}\n", .{name});
	126	}
	127	std.debug.print(
	128	\\Available Commands:
	129	\\ - build [step names / step indices]
	130	\\ - watch [step names / step indices]
	131	\\ - exit
	132	\\
	133	, .{});
	134
	135	var stdin_reader_buffer: [256]u8 = undefined;
	136	var stdin_reader = Io.File.stdin().reader(io, &stdin_reader_buffer);
	137	const stdin = &stdin_reader.interface;
	138
	139	while (true) {
	140	try Io.File.stdout().writeStreamingAll(io, "> ");
	141	const command = try stdin.takeDelimiterExclusive('\n');
	142	stdin.toss(1);
	143	if (std.mem.startsWith(u8, command, "build") or
	144	std.mem.startsWith(u8, command, "watch"))
	145	{
	146	var steps: std.ArrayList(Configuration.Step.Index) = .empty;
	147	defer steps.deinit(gpa);
	148
	149	const watch = std.mem.startsWith(u8, command, "watch");
	150
	151	if (std.mem.cutPrefix(u8, command, "build ") orelse
	152	std.mem.cutPrefix(u8, command, "watch ")) \|command_args\|
	153	{
	154	var it = std.mem.tokenizeScalar(u8, command_args, ' ');
	155	while (it.next()) \|arg\| {
	156	const step: Configuration.Step.Index =
	157	if (std.fmt.parseInt(u32, arg, 10)) \|i\|
	158	@fromBackingInt(i)
	159	else \|_\|
	160	top_level_steps.get(arg) orelse std.debug.panic("unexpected step name or index", .{});
	161	try steps.append(gpa, step);
	162	}
	163	}
	164
	165	if (steps.items.len < 1) {
	166	try steps.append(gpa, c.default_step);
	167	}
	168
	169	try client.serveBuildSteps(steps.items, .{ .watch = watch });
	170
	171	while (true) {
	172	const header: Server.Message.Header = client.receiveMessageWithMultiReader(&multi_reader, .none) catch \|err\| switch (err) {
	173	error.Canceled, error.ConcurrencyUnavailable => \|e\| return e,
	174	error.Timeout => unreachable,
	175	else => \|e\| {
	176	log.err("failed to receive message: {t}", .{err});
	177	break :blk e;
	178	},
	179	};
	180	const body = client_stdout.take(header.bytes_len) catch unreachable;
	181	log.debug("received {f} ({d} bytes)", .{ fmtEnum(header.tag), body.len });
	182
	183	switch (header.tag) {
	184	.bsp_build_started => {},
	185	.bsp_build_completed => if (!watch) break,
	186	.bsp_step_started => {},
	187	.bsp_step_completed => {},
	188	.bsp_configuration => @panic("TODO"),
	189	else => std.debug.panic("received unexpected message: {f}", .{fmtEnum(header.tag)}),
	190	}
	191	}
	192	continue;
	193	} else if (std.mem.eql(u8, command, "exit")) {
	194	try client.serveBodylessMessage(.exit);
	195	break;
	196	} else {
	197	log.err("unknown command: {q}", .{command});
	198	continue;
	199	}
	200	}
	201	};
	202
	203	try multi_reader.fillRemaining(.none);
	204
	205	if (client_stderr.bufferedLen() > 0) {
	206	log.err("stderr:\n{s}\n", .{client_stderr.buffered()});
	207	}
	208
	209	try err;
	210
	211	const term = try child_process.wait(io);
	212
	213	if (!term.success()) {
	214	log.err("maker {f}", .{term});
	215	}
	216	}
	217
	218	const FormatEnum = union(enum) {
	219	named: []const u8,
	220	unnamed: usize,
	221
	222	pub fn format(
	223	e: FormatEnum,
	224	writer: *std.Io.Writer,
	225	) std.Io.Writer.Error!void {
	226	switch (e) {
	227	.named => \|name\| {
	228	try writer.writeByte('.');
	229	try writer.writeAll(name);
	230	},
	231	.unnamed => \|number\| try writer.print("0x{x}", .{number}),
	232	}
	233	}
	234	};
	235
	236	fn fmtEnum(e: anytype) FormatEnum {
	237	if (std.enums.tagName(@TypeOf(e), e)) \|name\| {
	238	return .{ .named = name };
	239	} else {
	240	return .{ .unnamed = @backingInt(e) };
	241	}
	242	}

tools/incr-check.zig+34-30

...	...	@@ -305,21 +305,23 @@ const Eval = struct {
305	305
306	306	fn check(eval: Eval, mr: Io.File.MultiReader, update: Case.Update, prog_node: std.Progress.Node) !void {
307	307	const arena = eval.arena;
308		const stdout = mr.fileReader(0);
309		const stderr = &mr.fileReader(1).interface;
310		const Header = std.zig.Server.Message.Header;
	308	const stdout = mr.reader(0);
	309	const stderr = mr.reader(1);
	310
	311	var client: std.zig.Client = .{
	312	.in = stdout,
	313	.out = undefined,
	314	};
311	315
312	316	while (true) {
313		const header = stdout.interface.takeStruct(Header, .little) catch \|err\| switch (err) {
314		error.EndOfStream => break,
315		error.ReadFailed => return stdout.err.?,
316		};
317		const body = stdout.interface.take(header.bytes_len) catch \|err\| switch (err) {
	317	const header = client.receiveMessageWithMultiReader(mr, .none) catch \|err\| switch (err) {
	318	error.Timeout => unreachable,
318	319	// If this panic triggers it might be helpful to rework this
319	320	// code to print the stderr from the abnormally terminated child.
320	321	error.EndOfStream => @panic("unexpected mid-message end of stream"),
321		error.ReadFailed => return stdout.err.?,
	322	else => \|e\| return e,
322	323	};
	324	const body = client.in.take(header.bytes_len) catch unreachable;
323	325
324	326	switch (header.tag) {
325	327	.error_bundle => {
...	...	@@ -605,12 +607,13 @@ const Eval = struct {
605	607
606	608	fn requestUpdate(eval: *Eval) !void {
607	609	const io = eval.io;
608		const header: std.zig.Client.Message.Header = .{
609		.tag = .update,
610		.bytes_len = 0,
	610
	611	var w = eval.child.stdin.?.writerStreaming(io, &.{});
	612	var client: std.zig.Client = .{
	613	.in = undefined,
	614	.out = &w.interface,
611	615	};
612		var w = eval.child.stdin.?.writer(io, &.{});
613		w.interface.writeStruct(header, .little) catch \|err\| switch (err) {
	616	client.serveBodylessMessage(.update) catch \|err\| switch (err) {
614	617	error.WriteFailed => return w.err.?,
615	618	};
616	619	}
...	...	@@ -618,22 +621,23 @@ const Eval = struct {
618	621	fn end(eval: Eval, mr: Io.File.MultiReader) !void {
619	622	requestExit(eval.child, eval);
620	623
621		const stdout = mr.fileReader(0);
622		const Header = std.zig.Server.Message.Header;
	624	var client: std.zig.Client = .{
	625	.in = mr.reader(0),
	626	.out = undefined,
	627	};
623	628
624	629	while (true) {
625		const header = stdout.interface.takeStruct(Header, .little) catch \|err\| switch (err) {
626		error.EndOfStream => break,
627		error.ReadFailed => return stdout.err.?,
628		};
629		stdout.interface.discardAll(header.bytes_len) catch \|err\| switch (err) {
630		error.ReadFailed => return stdout.err.?,
631		error.EndOfStream => \|e\| return e,
	630	const header = client.receiveMessageWithMultiReader(mr, .none) catch \|err\| switch (err) {
	631	error.Timeout => unreachable,
	632	error.EndOfStream => \|e\| {
	633	if (client.in.bufferedLen() == 0) break;
	634	return e;
	635	},
	636	else => \|e\| return e,
632	637	};
	638	try client.in.discardAll(header.bytes_len);
633	639	}
634	640
635		try mr.fillRemaining(.none);
636
637	641	const stderr = mr.reader(1).buffered();
638	642	if (stderr.len > 0) eval.fatal("unexpected stderr:\n{s}", .{stderr});
639	643	}
...	...	@@ -899,12 +903,12 @@ fn requestExit(child: std.process.Child, eval: Eval) void {
899	903	if (child.stdin == null) return;
900	904	const io = eval.io;
901	905
902		const header: std.zig.Client.Message.Header = .{
903		.tag = .exit,
904		.bytes_len = 0,
	906	var w = eval.child.stdin.?.writerStreaming(io, &.{});
	907	var client: std.zig.Client = .{
	908	.in = undefined,
	909	.out = &w.interface,
905	910	};
906		var w = eval.child.stdin.?.writer(io, &.{});
907		w.interface.writeStruct(header, .little) catch \|err\| switch (err) {
	911	client.serveBodylessMessage(.exit) catch \|err\| switch (err) {
908	912	error.WriteFailed => switch (w.err.?) {
909	913	error.BrokenPipe => {},
910	914	else => \|e\| eval.fatal("failed to send exit: {t}", .{e}),