Commit 7e7d7875b9af97bd04ca03a98b2e4188d57e3c13 | zig

author	Matthew Lugg <mlugg@mlugg.co.uk> 2025-08-21 13:44:43+01:00
committer	Matthew Lugg <mlugg@mlugg.co.uk> 2025-10-18 09:28:39+01:00
log	7e7d7875b9af97bd04ca03a98b2e4188d57e3c13
tree	59d1f3ebe4d23a4c92fee31a5a2c763de7a9012e
parent	337762114f575824a1ab793dca41a3d073aa17cd
signature	Commit is signed but in an unrecognized format.

std.Build: implement unit test timeouts

For now, there is a flag to `zig build` called `--test-timeout-ms` which
accepts a value in milliseconds. If the execution time of any individual
unit test exceeds that number of milliseconds, the test is terminated
and marked as timed out.

In the future, we may want to increase the granularity of this feature
by allowing timeouts to be specified per-step or even per-test. However,
a global option is actually very useful. In particular, it can be used
in CI scripts to ensure that no individual unit test exceeds some
reasonable limit (e.g. 60 seconds) without having to assign limits to
every individual test step in the build script.

Also, individual unit test durations are now shown in the time report
web interface -- this was fairly trivial to add since we're timing tests
(to check for timeouts) anyway.

This commit makes progress on #19821, but does not close it, because
that proposal includes a more sophisticated mechanism for setting
timeouts.

Co-Authored-By: David Rubin <david@vortan.dev>

11 files changed, 370 insertions(+), 36 deletions(-)

lib
- build-web
- compiler
  - build_runner.zig
  - test_runner.zig
- std
  - Build
    Step.zig
    Step
    Run.zig
    WebServer.zig
    abi.zig
  - zig
    Server.zig

lib/build-web/index.html+13

...	...	@@ -139,6 +139,19 @@
139	139	<div><slot name="llvm-pass-timings"></slot></div>
140	140	</details>
141	141	</div>
	142	<div id="runTestReport">
	143	<table class="time-stats">
	144	<thead>
	145	<tr>
	146	<th scope="col">Test Name</th>
	147	<th scope="col">Duration</th>
	148	</tr>
	149	</thead>
	150	<!-- HTML does not allow placing a 'slot' inside of a 'tbody' for backwards-compatibility
	151	reasons, so we unfortunately must template on the `id` here. -->
	152	<tbody id="runTestTableBody"></tbody>
	153	</div>
	154	</div>
142	155	</details>
143	156	</template>
144	157

lib/build-web/main.js+21-1

...	...	@@ -46,8 +46,9 @@ WebAssembly.instantiateStreaming(wasm_promise, {
46	46	updateCoverage: fuzzUpdateCoverage,
47	47	},
48	48	time_report: {
49		updateCompile: timeReportUpdateCompile,
50	49	updateGeneric: timeReportUpdateGeneric,
	50	updateCompile: timeReportUpdateCompile,
	51	updateRunTest: timeReportUpdateRunTest,
51	52	},
52	53	}).then(function(obj) {
53	54	setConnectionStatus("Connecting to WebSocket...", true);
...	...	@@ -248,6 +249,7 @@ function timeReportUpdateCompile(
248	249
249	250	shadow.getElementById("genericReport").classList.add("hidden");
250	251	shadow.getElementById("compileReport").classList.remove("hidden");
	252	shadow.getElementById("runTestReport").classList.add("hidden");
251	253
252	254	if (!use_llvm) shadow.querySelector(":host > details").classList.add("no-llvm");
253	255	host.innerHTML = inner_html;
...	...	@@ -265,8 +267,26 @@ function timeReportUpdateGeneric(
265	267	shadow.querySelector(":host > details").classList.remove("pending", "no-llvm");
266	268	shadow.getElementById("genericReport").classList.remove("hidden");
267	269	shadow.getElementById("compileReport").classList.add("hidden");
	270	shadow.getElementById("runTestReport").classList.add("hidden");
268	271	host.innerHTML = inner_html;
269	272	}
	273	function timeReportUpdateRunTest(
	274	step_idx,
	275	table_html_ptr,
	276	table_html_len,
	277	) {
	278	const table_html = decodeString(table_html_ptr, table_html_len);
	279	const host = domTimeReportList.children.item(step_idx);
	280	const shadow = host.shadowRoot;
	281
	282	shadow.querySelector(":host > details").classList.remove("pending", "no-llvm");
	283
	284	shadow.getElementById("genericReport").classList.add("hidden");
	285	shadow.getElementById("compileReport").classList.add("hidden");
	286	shadow.getElementById("runTestReport").classList.remove("hidden");
	287
	288	shadow.getElementById("runTestTableBody").innerHTML = table_html;
	289	}
270	290
271	291	const fuzz_entry_template = document.getElementById("fuzzEntryTemplate").content;
272	292	const domFuzz = document.getElementById("fuzz");

lib/build-web/main.zig+1

...	...	@@ -94,6 +94,7 @@ export fn message_end() void {
94	94
95	95	.time_report_generic_result => return time_report.genericResultMessage(msg_bytes) catch @panic("OOM"),
96	96	.time_report_compile_result => return time_report.compileResultMessage(msg_bytes) catch @panic("OOM"),
	97	.time_report_run_test_result => return time_report.runTestResultMessage(msg_bytes) catch @panic("OOM"),
97	98	}
98	99	}
99	100

lib/build-web/time_report.zig+41

...	...	@@ -27,6 +27,13 @@ const js = struct {
27	27	/// Whether the LLVM backend was used. If not, LLVM-specific statistics are hidden.
28	28	use_llvm: bool,
29	29	) void;
	30	extern "time_report" fn updateRunTest(
	31	/// The index of the step.
	32	step_idx: u32,
	33	// The HTML which will populate the <tbody> of the test table.
	34	table_html_ptr: [*]const u8,
	35	table_html_len: usize,
	36	) void;
30	37	};
31	38
32	39	pub fn genericResultMessage(msg_bytes: []u8) error{OutOfMemory}!void {
...	...	@@ -237,3 +244,37 @@ pub fn compileResultMessage(msg_bytes: []u8) error{ OutOfMemory, WriteFailed }!v
237	244	hdr.flags.use_llvm,
238	245	);
239	246	}
	247
	248	pub fn runTestResultMessage(msg_bytes: []u8) error{OutOfMemory}!void {
	249	if (msg_bytes.len < @sizeOf(abi.RunTestResult)) @panic("malformed RunTestResult message");
	250	const hdr: *const abi.RunTestResult = @ptrCast(msg_bytes[0..@sizeOf(abi.RunTestResult)]);
	251	if (hdr.step_idx >= step_list.*.len) @panic("malformed RunTestResult message");
	252	const trailing = msg_bytes[@sizeOf(abi.RunTestResult)..];
	253
	254	const durations: []align(1) const u64 = @ptrCast(trailing[0 .. hdr.tests_len * 8]);
	255	var offset: usize = hdr.tests_len * 8;
	256
	257	var table_html: std.ArrayListUnmanaged(u8) = .empty;
	258	defer table_html.deinit(gpa);
	259
	260	for (durations) \|test_ns\| {
	261	const test_name_len = std.mem.indexOfScalar(u8, trailing[offset..], 0) orelse @panic("malformed RunTestResult message");
	262	const test_name = trailing[offset..][0..test_name_len];
	263	offset += test_name_len + 1;
	264	try table_html.print(gpa, "<tr><th scope=\"row\"><code>{f}</code></th>", .{fmtEscapeHtml(test_name)});
	265	if (test_ns == std.math.maxInt(u64)) {
	266	try table_html.appendSlice(gpa, "<td class=\"empty-cell\"></td>"); // didn't run
	267	} else {
	268	try table_html.print(gpa, "<td>{D}</td>", .{test_ns});
	269	}
	270	try table_html.appendSlice(gpa, "</tr>\n");
	271	}
	272
	273	if (offset != trailing.len) @panic("malformed RunTestResult message");
	274
	275	js.updateRunTest(
	276	hdr.step_idx,
	277	table_html.items.ptr,
	278	table_html.items.len,
	279	);
	280	}

lib/compiler/build_runner.zig+35-7

...	...	@@ -106,6 +106,7 @@ pub fn main() !void {
106	106	var summary: ?Summary = null;
107	107	var max_rss: u64 = 0;
108	108	var skip_oom_steps = false;
	109	var test_timeout_ms: ?u64 = null;
109	110	var color: Color = .auto;
110	111	var prominent_compile_errors = false;
111	112	var help_menu = false;
...	...	@@ -175,6 +176,14 @@ pub fn main() !void {
175	176	};
176	177	} else if (mem.eql(u8, arg, "--skip-oom-steps")) {
177	178	skip_oom_steps = true;
	179	} else if (mem.eql(u8, arg, "--test-timeout-ms")) {
	180	const millis_str = nextArgOrFatal(args, &arg_idx);
	181	test_timeout_ms = std.fmt.parseInt(u64, millis_str, 10) catch \|err\| {
	182	std.debug.print("invalid millisecond count: '{s}': {s}\n", .{
	183	millis_str, @errorName(err),
	184	});
	185	process.exit(1);
	186	};
178	187	} else if (mem.eql(u8, arg, "--search-prefix")) {
179	188	const search_prefix = nextArgOrFatal(args, &arg_idx);
180	189	builder.addSearchPrefix(search_prefix);
...	...	@@ -448,6 +457,11 @@ pub fn main() !void {
448	457	.max_rss_is_default = false,
449	458	.max_rss_mutex = .{},
450	459	.skip_oom_steps = skip_oom_steps,
	460	.unit_test_timeout_ns = ns: {
	461	const ms = test_timeout_ms orelse break :ns null;
	462	break :ns std.math.mul(u64, ms, std.time.ns_per_ms) catch null;
	463	},
	464
451	465	.watch = watch,
452	466	.web_server = undefined, // set after `prepare`
453	467	.memory_blocked_steps = .empty,
...	...	@@ -605,6 +619,7 @@ const Run = struct {
605	619	max_rss_is_default: bool,
606	620	max_rss_mutex: std.Thread.Mutex,
607	621	skip_oom_steps: bool,
	622	unit_test_timeout_ns: ?u64,
608	623	watch: bool,
609	624	web_server: if (!builtin.single_threaded) ?WebServer else ?noreturn,
610	625	/// Allocated into `gpa`.
...	...	@@ -724,6 +739,7 @@ fn runStepNames(
724	739	var test_fail_count: usize = 0;
725	740	var test_pass_count: usize = 0;
726	741	var test_leak_count: usize = 0;
	742	var test_timeout_count: usize = 0;
727	743	var test_count: usize = 0;
728	744
729	745	var success_count: usize = 0;
...	...	@@ -736,6 +752,7 @@ fn runStepNames(
736	752	test_fail_count += s.test_results.fail_count;
737	753	test_skip_count += s.test_results.skip_count;
738	754	test_leak_count += s.test_results.leak_count;
	755	test_timeout_count += s.test_results.timeout_count;
739	756	test_pass_count += s.test_results.passCount();
740	757	test_count += s.test_results.test_count;
741	758
...	...	@@ -834,6 +851,7 @@ fn runStepNames(
834	851	if (test_skip_count > 0) w.print("; {d} skipped", .{test_skip_count}) catch {};
835	852	if (test_fail_count > 0) w.print("; {d} failed", .{test_fail_count}) catch {};
836	853	if (test_leak_count > 0) w.print("; {d} leaked", .{test_leak_count}) catch {};
	854	if (test_timeout_count > 0) w.print("; {d} timed out", .{test_timeout_count}) catch {};
837	855
838	856	w.writeAll("\n") catch {};
839	857
...	...	@@ -995,7 +1013,10 @@ fn printStepStatus(
995	1013	try stderr.writeAll("\n");
996	1014	try ttyconf.setColor(stderr, .reset);
997	1015	},
998		.failure => try printStepFailure(s, stderr, ttyconf),
	1016	.failure => {
	1017	try printStepFailure(s, stderr, ttyconf);
	1018	try ttyconf.setColor(stderr, .reset);
	1019	},
999	1020	}
1000	1021	}
1001	1022
...	...	@@ -1009,7 +1030,6 @@ fn printStepFailure(
1009	1030	try stderr.print(" {d} errors\n", .{
1010	1031	s.result_error_bundle.errorMessageCount(),
1011	1032	});
1012		try ttyconf.setColor(stderr, .reset);
1013	1033	} else if (!s.test_results.isSuccess()) {
1014	1034	try stderr.print(" {d}/{d} passed", .{
1015	1035	s.test_results.passCount(), s.test_results.test_count,
...	...	@@ -1020,7 +1040,7 @@ fn printStepFailure(
1020	1040	try stderr.print("{d} failed", .{
1021	1041	s.test_results.fail_count,
1022	1042	});
1023		try ttyconf.setColor(stderr, .reset);
	1043	try ttyconf.setColor(stderr, .white);
1024	1044	}
1025	1045	if (s.test_results.skip_count > 0) {
1026	1046	try stderr.writeAll(", ");
...	...	@@ -1028,7 +1048,7 @@ fn printStepFailure(
1028	1048	try stderr.print("{d} skipped", .{
1029	1049	s.test_results.skip_count,
1030	1050	});
1031		try ttyconf.setColor(stderr, .reset);
	1051	try ttyconf.setColor(stderr, .white);
1032	1052	}
1033	1053	if (s.test_results.leak_count > 0) {
1034	1054	try stderr.writeAll(", ");
...	...	@@ -1036,18 +1056,24 @@ fn printStepFailure(
1036	1056	try stderr.print("{d} leaked", .{
1037	1057	s.test_results.leak_count,
1038	1058	});
1039		try ttyconf.setColor(stderr, .reset);
	1059	try ttyconf.setColor(stderr, .white);
	1060	}
	1061	if (s.test_results.timeout_count > 0) {
	1062	try stderr.writeAll(", ");
	1063	try ttyconf.setColor(stderr, .red);
	1064	try stderr.print("{d} timed out", .{
	1065	s.test_results.timeout_count,
	1066	});
	1067	try ttyconf.setColor(stderr, .white);
1040	1068	}
1041	1069	try stderr.writeAll("\n");
1042	1070	} else if (s.result_error_msgs.items.len > 0) {
1043	1071	try ttyconf.setColor(stderr, .red);
1044	1072	try stderr.writeAll(" failure\n");
1045		try ttyconf.setColor(stderr, .reset);
1046	1073	} else {
1047	1074	assert(s.result_stderr.len > 0);
1048	1075	try ttyconf.setColor(stderr, .red);
1049	1076	try stderr.writeAll(" stderr\n");
1050		try ttyconf.setColor(stderr, .reset);
1051	1077	}
1052	1078	}
1053	1079
...	...	@@ -1250,6 +1276,7 @@ fn workerMakeOneStep(
1250	1276	.thread_pool = thread_pool,
1251	1277	.watch = run.watch,
1252	1278	.web_server = if (run.web_server) \|*ws\| ws else null,
	1279	.unit_test_timeout_ns = run.unit_test_timeout_ns,
1253	1280	.gpa = run.gpa,
1254	1281	});
1255	1282
...	...	@@ -1439,6 +1466,7 @@ fn printUsage(b: std.Build, w: Writer) !void {
1439	1466	\\ -j<N> Limit concurrent jobs (default is to use all CPU cores)
1440	1467	\\ --maxrss <bytes> Limit memory usage (default is to use available memory)
1441	1468	\\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss
	1469	\\ --test-timeout-ms <ms> Limit execution time of unit tests, terminating if exceeded
1442	1470	\\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit
1443	1471	\\ needed (Default) Lazy dependencies are fetched as needed
1444	1472	\\ all Lazy dependencies are always fetched

lib/compiler/test_runner.zig+4

...	...	@@ -135,6 +135,10 @@ fn mainServer() !void {
135	135	var fail = false;
136	136	var skip = false;
137	137	is_fuzz_test = false;
	138
	139	// let the build server know we're starting the test now
	140	try server.serveStringMessage(.test_started, &.{});
	141
138	142	test_fn.func() catch \|err\| switch (err) {
139	143	error.SkipZigTest => skip = true,
140	144	else => {

lib/std/Build/Step.zig+6-3

...	...	@@ -66,15 +66,16 @@ pub const TestResults = struct {
66	66	fail_count: u32 = 0,
67	67	skip_count: u32 = 0,
68	68	leak_count: u32 = 0,
	69	timeout_count: u32 = 0,
69	70	log_err_count: u32 = 0,
70	71	test_count: u32 = 0,
71	72
72	73	pub fn isSuccess(tr: TestResults) bool {
73		return tr.fail_count == 0 and tr.leak_count == 0 and tr.log_err_count == 0;
	74	return tr.fail_count == 0 and tr.leak_count == 0 and tr.log_err_count == 0 and tr.timeout_count == 0;
74	75	}
75	76
76	77	pub fn passCount(tr: TestResults) u32 {
77		return tr.test_count - tr.fail_count - tr.skip_count;
	78	return tr.test_count - tr.fail_count - tr.skip_count - tr.timeout_count;
78	79	}
79	80	};
80	81
...	...	@@ -88,6 +89,8 @@ pub const MakeOptions = struct {
88	89	// it currently breaks because `std.net.Address` doesn't work there. Work around for now.
89	90	.wasm32 => void,
90	91	},
	92	/// If set, this is a timeout to enforce on all individual unit tests, in nanoseconds.
	93	unit_test_timeout_ns: ?u64,
91	94	/// Not to be confused with `Build.allocator`, which is an alias of `Build.graph.arena`.
92	95	gpa: Allocator,
93	96	};
...	...	@@ -243,6 +246,7 @@ pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!voi
243	246	var timer: ?std.time.Timer = t: {
244	247	if (!s.owner.graph.time_report) break :t null;
245	248	if (s.id == .compile) break :t null;
	249	if (s.id == .run and s.cast(Run).?.stdio == .zig_test) break :t null;
246	250	break :t std.time.Timer.start() catch @panic("--time-report not supported on this host");
247	251	};
248	252	const make_result = s.makeFn(s, options);
...	...	@@ -513,7 +517,6 @@ fn zigProcessUpdate(s: Step, zp: ZigProcess, watch: bool, web_server: ?*Build.
513	517	const header = stdout.takeStruct(Header, .little) catch unreachable;
514	518	while (stdout.buffered().len < header.bytes_len) if (!try zp.poller.poll()) break :poll;
515	519	const body = stdout.take(header.bytes_len) catch unreachable;
516
517	520	switch (header.tag) {
518	521	.zig_version => {
519	522	if (!std.mem.eql(u8, builtin.zig_version_string, body)) {

lib/std/Build/Step/Run.zig+169-25

...	...	@@ -756,7 +756,6 @@ const IndexedOutput = struct {
756	756	output: *Output,
757	757	};
758	758	fn make(step: *Step, options: Step.MakeOptions) !void {
759		const prog_node = options.progress_node;
760	759	const b = step.owner;
761	760	const arena = b.allocator;
762	761	const run: *Run = @fieldParentPtr("step", step);
...	...	@@ -964,7 +963,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
964	963	b.fmt("{s}{s}", .{ placeholder.output.prefix, arg_output_path });
965	964	}
966	965
967		try runCommand(run, argv_list.items, has_side_effects, output_dir_path, prog_node, null);
	966	try runCommand(run, argv_list.items, has_side_effects, output_dir_path, options, null);
968	967	if (!has_side_effects) try step.writeManifestAndWatch(&man);
969	968	return;
970	969	};
...	...	@@ -997,7 +996,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
997	996	});
998	997	}
999	998
1000		try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node, null);
	999	try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, options, null);
1001	1000
1002	1001	const dep_file_dir = std.fs.cwd();
1003	1002	const dep_file_basename = dep_output_file.generated_file.getPath2(b, step);
...	...	@@ -1115,7 +1114,14 @@ pub fn rerunInFuzzMode(
1115	1114	const has_side_effects = false;
1116	1115	const rand_int = std.crypto.random.int(u64);
1117	1116	const tmp_dir_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int);
1118		try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node, .{
	1117	try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, .{
	1118	.progress_node = prog_node,
	1119	.thread_pool = undefined, // not used by `runCommand`
	1120	.watch = undefined, // not used by `runCommand`
	1121	.web_server = null, // only needed for time reports
	1122	.unit_test_timeout_ns = null, // don't time out fuzz tests for now
	1123	.gpa = undefined, // not used by `runCommand`
	1124	}, .{
1119	1125	.unit_test_index = unit_test_index,
1120	1126	.fuzz = fuzz,
1121	1127	});
...	...	@@ -1196,7 +1202,7 @@ fn runCommand(
1196	1202	argv: []const []const u8,
1197	1203	has_side_effects: bool,
1198	1204	output_dir_path: []const u8,
1199		prog_node: std.Progress.Node,
	1205	options: Step.MakeOptions,
1200	1206	fuzz_context: ?FuzzContext,
1201	1207	) !void {
1202	1208	const step = &run.step;
...	...	@@ -1218,7 +1224,7 @@ fn runCommand(
1218	1224
1219	1225	var env_map = run.env_map orelse &b.graph.env_map;
1220	1226
1221		const result = spawnChildAndCollect(run, argv, env_map, has_side_effects, prog_node, fuzz_context) catch \|err\| term: {
	1227	const result = spawnChildAndCollect(run, argv, env_map, has_side_effects, options, fuzz_context) catch \|err\| term: {
1222	1228	// InvalidExe: cpu arch mismatch
1223	1229	// FileNotFound: can happen with a wrong dynamic linker path
1224	1230	if (err == error.InvalidExe or err == error.FileNotFound) interpret: {
...	...	@@ -1357,7 +1363,7 @@ fn runCommand(
1357	1363
1358	1364	try Step.handleVerbose2(step.owner, cwd, run.env_map, interp_argv.items);
1359	1365
1360		break :term spawnChildAndCollect(run, interp_argv.items, env_map, has_side_effects, prog_node, fuzz_context) catch \|e\| {
	1366	break :term spawnChildAndCollect(run, interp_argv.items, env_map, has_side_effects, options, fuzz_context) catch \|e\| {
1361	1367	if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped;
1362	1368
1363	1369	return step.fail("unable to spawn interpreter {s}: {s}", .{
...	...	@@ -1372,8 +1378,14 @@ fn runCommand(
1372	1378	step.result_duration_ns = result.elapsed_ns;
1373	1379	step.result_peak_rss = result.peak_rss;
1374	1380	step.test_results = result.stdio.test_results;
1375		if (result.stdio.test_metadata) \|tm\|
	1381	if (result.stdio.test_metadata) \|tm\| {
1376	1382	run.cached_test_metadata = tm.toCachedTestMetadata();
	1383	if (options.web_server) \|ws\| ws.updateTimeReportRunTest(
	1384	run,
	1385	&run.cached_test_metadata.?,
	1386	tm.ns_per_test,
	1387	);
	1388	}
1377	1389
1378	1390	const final_argv = if (interp_argv.items.len == 0) argv else interp_argv.items;
1379	1391
...	...	@@ -1558,7 +1570,7 @@ fn spawnChildAndCollect(
1558	1570	argv: []const []const u8,
1559	1571	env_map: *EnvMap,
1560	1572	has_side_effects: bool,
1561		prog_node: std.Progress.Node,
	1573	options: Step.MakeOptions,
1562	1574	fuzz_context: ?FuzzContext,
1563	1575	) !ChildProcResult {
1564	1576	const b = run.step.owner;
...	...	@@ -1604,7 +1616,7 @@ fn spawnChildAndCollect(
1604	1616	const inherit = child.stdout_behavior == .Inherit or child.stderr_behavior == .Inherit;
1605	1617
1606	1618	if (run.stdio != .zig_test and !run.disable_zig_progress and !inherit) {
1607		child.progress_node = prog_node;
	1619	child.progress_node = options.progress_node;
1608	1620	}
1609	1621
1610	1622	const term, const result, const elapsed_ns = t: {
...	...	@@ -1622,7 +1634,7 @@ fn spawnChildAndCollect(
1622	1634	var timer = try std.time.Timer.start();
1623	1635
1624	1636	const result = if (run.stdio == .zig_test)
1625		try evalZigTest(run, &child, prog_node, fuzz_context)
	1637	try evalZigTest(run, &child, options, fuzz_context)
1626	1638	else
1627	1639	try evalGeneric(run, &child);
1628	1640
...	...	@@ -1647,13 +1659,15 @@ const StdIoResult = struct {
1647	1659	fn evalZigTest(
1648	1660	run: *Run,
1649	1661	child: *std.process.Child,
1650		prog_node: std.Progress.Node,
	1662	options: Step.MakeOptions,
1651	1663	fuzz_context: ?FuzzContext,
1652	1664	) !StdIoResult {
1653	1665	const gpa = run.step.owner.allocator;
1654	1666	const arena = run.step.owner.allocator;
1655	1667
1656		var poller = std.Io.poll(gpa, enum { stdout, stderr }, .{
	1668	const PollEnum = enum { stdout, stderr };
	1669
	1670	var poller = std.Io.poll(gpa, PollEnum, .{
1657	1671	.stdout = child.stdout.?,
1658	1672	.stderr = child.stderr.?,
1659	1673	});
...	...	@@ -1692,21 +1706,126 @@ fn evalZigTest(
1692	1706	var fail_count: u32 = 0;
1693	1707	var skip_count: u32 = 0;
1694	1708	var leak_count: u32 = 0;
	1709	var timeout_count: u32 = 0;
1695	1710	var test_count: u32 = 0;
1696	1711	var log_err_count: u32 = 0;
1697	1712
1698	1713	var metadata: ?TestMetadata = null;
1699	1714	var coverage_id: ?u64 = null;
1700	1715
	1716	var test_is_running = false;
	1717
	1718	// String allocated into `gpa`. Owned by this function while it runs, then moved to the `Step`.
	1719	var result_stderr: []u8 = &.{};
	1720	defer run.step.result_stderr = result_stderr;
	1721
	1722	// `null` means this host does not support `std.time.Timer`. This timer is `reset()` whenever we
	1723	// toggle `test_is_running`, i.e. whenever a test starts or finishes.
	1724	var timer: ?std.time.Timer = std.time.Timer.start() catch t: {
	1725	std.log.warn("std.time.Timer not supported on host; test timeouts will be ignored", .{});
	1726	break :t null;
	1727	};
	1728
1701	1729	var sub_prog_node: ?std.Progress.Node = null;
1702	1730	defer if (sub_prog_node) \|n\| n.end();
1703	1731
1704		const stdout = poller.reader(.stdout);
1705		const stderr = poller.reader(.stderr);
	1732	// This timeout is used when we're waiting on the test runner itself rather than a user-specified
	1733	// test. For instance, if the test runner leaves this much time between us requesting a test to
	1734	// start and it acknowledging the test starting, we terminate the child and raise an error. This
	1735	// should never happen, but could in theory be caused by some very unlucky IB in a test.
	1736	const response_timeout_ns = 30 * std.time.ns_per_s;
	1737
1706	1738	const any_write_failed = first_write_failed or poll: while (true) {
	1739	// These are scoped inside the loop because we sometimes respawn the child and recreate
	1740	// `poller` which invaldiates these readers.
	1741	const stdout = poller.reader(.stdout);
	1742	const stderr = poller.reader(.stderr);
	1743
1707	1744	const Header = std.zig.Server.Message.Header;
1708		while (stdout.buffered().len < @sizeOf(Header)) if (!try poller.poll()) break :poll false;
	1745
	1746	// This block is exited when `stdout` contains enough bytes for a `Header`.
	1747	header_ready: {
	1748	if (stdout.buffered().len >= @sizeOf(Header)) {
	1749	// We already have one, no need to poll!
	1750	break :header_ready;
	1751	}
	1752
	1753	// Always `null` if `timer` is `null`.
	1754	const opt_timeout_ns: ?u64 = ns: {
	1755	if (timer == null) break :ns null;
	1756	if (!test_is_running) break :ns response_timeout_ns;
	1757	break :ns options.unit_test_timeout_ns;
	1758	};
	1759
	1760	if (opt_timeout_ns) \|timeout_ns\| {
	1761	const remaining_ns = timeout_ns -\| timer.?.read();
	1762	if (!try poller.pollTimeout(remaining_ns)) break :poll false;
	1763	} else {
	1764	if (!try poller.poll()) break :poll false;
	1765	}
	1766
	1767	if (stdout.buffered().len >= @sizeOf(Header)) {
	1768	// There wasn't a header before, but there is one after the `poll`.
	1769	break :header_ready;
	1770	}
	1771
	1772	const timeout_ns = opt_timeout_ns orelse continue;
	1773	const cur_ns = timer.?.read();
	1774	if (cur_ns < timeout_ns) continue;
	1775
	1776	// There was a timeout.
	1777
	1778	if (!test_is_running) {
	1779	// The child stopped responding while not running a test. To avoid getting into
	1780	// a loop if something's broken, don't retry; just report an error and stop.
	1781	try run.step.addError("test runner failed to respond for {D}", .{cur_ns});
	1782	break :poll false;
	1783	}
	1784
	1785	// A test has probably just gotten stuck. We'll report an error, then just kill the
	1786	// child and continue with the next test in the list.
	1787
	1788	const md = &metadata.?;
	1789	const test_index = md.next_index - 1;
	1790
	1791	timeout_count += 1;
	1792	try run.step.addError(
	1793	"'{s}' timed out after {D}",
	1794	.{ md.testName(test_index), cur_ns },
	1795	);
	1796	if (stderr.buffered().len > 0) {
	1797	const new_bytes = stderr.buffered();
	1798	const old_len = result_stderr.len;
	1799	result_stderr = try gpa.realloc(result_stderr, old_len + new_bytes.len);
	1800	@memcpy(result_stderr[old_len..], new_bytes);
	1801	}
	1802
	1803	_ = try child.kill();
	1804	// Respawn the test runner. There's a double-cleanup if this fails, but that's
	1805	// fine because our caller's `kill` will just return `error.AlreadyTerminated`.
	1806	try child.spawn();
	1807	try child.waitForSpawn();
	1808
	1809	// After respawning the child, we must update the poller's streams.
	1810	poller.deinit();
	1811	poller = std.Io.poll(gpa, PollEnum, .{
	1812	.stdout = child.stdout.?,
	1813	.stderr = child.stderr.?,
	1814	});
	1815
	1816	test_is_running = false;
	1817	md.ns_per_test[test_index] = timer.?.lap();
	1818
	1819	requestNextTest(child.stdin.?, md, &sub_prog_node) catch \|err\| {
	1820	try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});
	1821	break :poll true;
	1822	};
	1823
	1824	continue :poll; // continue work with the new (respawned) child
	1825	}
	1826	// There is definitely a header available now -- read it.
1709	1827	const header = stdout.takeStruct(Header, .little) catch unreachable;
	1828
1710	1829	while (stdout.buffered().len < header.bytes_len) if (!try poller.poll()) break :poll false;
1711	1830	const body = stdout.take(header.bytes_len) catch unreachable;
1712	1831	switch (header.tag) {
...	...	@@ -1720,6 +1839,12 @@ fn evalZigTest(
1720	1839	},
1721	1840	.test_metadata => {
1722	1841	assert(fuzz_context == null);
	1842
	1843	// `metadata` would only be populated if we'd already seen a `test_metadata`, but we
	1844	// only request it once (and importantly, we don't re-request it if we kill and
	1845	// restart the test runner).
	1846	assert(metadata == null);
	1847
1723	1848	const TmHdr = std.zig.Server.Message.TestMetadata;
1724	1849	const tm_hdr = @as(*align(1) const TmHdr, @ptrCast(body));
1725	1850	test_count = tm_hdr.tests_len;
...	...	@@ -1730,32 +1855,42 @@ fn evalZigTest(
1730	1855
1731	1856	const names = std.mem.bytesAsSlice(u32, names_bytes);
1732	1857	const expected_panic_msgs = std.mem.bytesAsSlice(u32, expected_panic_msgs_bytes);
	1858
1733	1859	const names_aligned = try arena.alloc(u32, names.len);
1734	1860	for (names_aligned, names) \|dest, src\| dest. = src;
1735	1861
1736	1862	const expected_panic_msgs_aligned = try arena.alloc(u32, expected_panic_msgs.len);
1737	1863	for (expected_panic_msgs_aligned, expected_panic_msgs) \|dest, src\| dest. = src;
1738	1864
1739		prog_node.setEstimatedTotalItems(names.len);
	1865	options.progress_node.setEstimatedTotalItems(names.len);
1740	1866	metadata = .{
1741	1867	.string_bytes = try arena.dupe(u8, string_bytes),
	1868	.ns_per_test = try arena.alloc(u64, test_count),
1742	1869	.names = names_aligned,
1743	1870	.expected_panic_msgs = expected_panic_msgs_aligned,
1744	1871	.next_index = 0,
1745		.prog_node = prog_node,
	1872	.prog_node = options.progress_node,
1746	1873	};
	1874	@memset(metadata.?.ns_per_test, std.math.maxInt(u64));
	1875
	1876	test_is_running = false;
	1877	if (timer) \|*t\| t.reset();
1747	1878
1748	1879	requestNextTest(child.stdin.?, &metadata.?, &sub_prog_node) catch \|err\| {
1749	1880	try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});
1750	1881	break :poll true;
1751	1882	};
1752	1883	},
	1884	.test_started => {
	1885	test_is_running = true;
	1886	if (timer) \|*t\| t.reset();
	1887	},
1753	1888	.test_results => {
1754	1889	assert(fuzz_context == null);
1755		const md = metadata.?;
	1890	const md = &metadata.?;
1756	1891
1757	1892	const TrHdr = std.zig.Server.Message.TestResults;
1758		const tr_hdr = @as(*align(1) const TrHdr, @ptrCast(body));
	1893	const tr_hdr: *align(1) const TrHdr = @ptrCast(body);
1759	1894	fail_count +\|= @intFromBool(tr_hdr.flags.fail);
1760	1895	skip_count +\|= @intFromBool(tr_hdr.flags.skip);
1761	1896	leak_count +\|= @intFromBool(tr_hdr.flags.leak);
...	...	@@ -1764,7 +1899,7 @@ fn evalZigTest(
1764	1899	if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, tr_hdr.index);
1765	1900
1766	1901	if (tr_hdr.flags.fail or tr_hdr.flags.leak or tr_hdr.flags.log_err_count > 0) {
1767		const name = std.mem.sliceTo(md.string_bytes[md.names[tr_hdr.index]..], 0);
	1902	const name = std.mem.sliceTo(md.testName(tr_hdr.index), 0);
1768	1903	const stderr_contents = stderr.buffered();
1769	1904	stderr.toss(stderr_contents.len);
1770	1905	const msg = std.mem.trim(u8, stderr_contents, "\n");
...	...	@@ -1783,7 +1918,10 @@ fn evalZigTest(
1783	1918	}
1784	1919	}
1785	1920
1786		requestNextTest(child.stdin.?, &metadata.?, &sub_prog_node) catch \|err\| {
	1921	test_is_running = false;
	1922	if (timer) \|*t\| md.ns_per_test[tr_hdr.index] = t.lap();
	1923
	1924	requestNextTest(child.stdin.?, md, &sub_prog_node) catch \|err\| {
1787	1925	try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});
1788	1926	break :poll true;
1789	1927	};
...	...	@@ -1831,9 +1969,12 @@ fn evalZigTest(
1831	1969	while (try poller.poll()) {}
1832	1970	}
1833	1971
1834		const stderr_contents = std.mem.trim(u8, stderr.buffered(), "\n");
1835		if (stderr_contents.len > 0) {
1836		run.step.result_stderr = try arena.dupe(u8, stderr_contents);
	1972	const stderr = poller.reader(.stderr);
	1973	if (stderr.buffered().len > 0) {
	1974	const new_bytes = stderr.buffered();
	1975	const old_len = result_stderr.len;
	1976	result_stderr = try gpa.realloc(result_stderr, old_len + new_bytes.len);
	1977	@memcpy(result_stderr[old_len..], new_bytes);
1837	1978	}
1838	1979
1839	1980	// Send EOF to stdin.
...	...	@@ -1848,6 +1989,7 @@ fn evalZigTest(
1848	1989	.fail_count = fail_count,
1849	1990	.skip_count = skip_count,
1850	1991	.leak_count = leak_count,
	1992	.timeout_count = timeout_count,
1851	1993	.log_err_count = log_err_count,
1852	1994	},
1853	1995	.test_metadata = metadata,
...	...	@@ -1856,6 +1998,7 @@ fn evalZigTest(
1856	1998
1857	1999	const TestMetadata = struct {
1858	2000	names: []const u32,
	2001	ns_per_test: []u64,
1859	2002	expected_panic_msgs: []const u32,
1860	2003	string_bytes: []const u8,
1861	2004	next_index: u32,
...	...	@@ -1896,6 +2039,7 @@ fn requestNextTest(in: fs.File, metadata: TestMetadata, sub_prog_node: ?std.Pr
1896	2039	try sendRunTestMessage(in, .run_test, i);
1897	2040	return;
1898	2041	} else {
	2042	metadata.next_index = std.math.maxInt(u32); // indicate that all tests are done
1899	2043	try sendMessage(in, .exit);
1900	2044	}
1901	2045	}

lib/std/Build/WebServer.zig+58

...	...	@@ -751,6 +751,64 @@ pub fn updateTimeReportGeneric(ws: WebServer, step: Build.Step, ns_total: u64)
751	751	ws.notifyUpdate();
752	752	}
753	753
	754	pub fn updateTimeReportRunTest(
	755	ws: *WebServer,
	756	run: *Build.Step.Run,
	757	tests: *const Build.Step.Run.CachedTestMetadata,
	758	ns_per_test: []const u64,
	759	) void {
	760	const gpa = ws.gpa;
	761
	762	const step_idx: u32 = for (ws.all_steps, 0..) \|s, i\| {
	763	if (s == &run.step) break @intCast(i);
	764	} else unreachable;
	765
	766	assert(tests.names.len == ns_per_test.len);
	767	const tests_len: u32 = @intCast(tests.names.len);
	768
	769	const new_len: u64 = len: {
	770	var names_len: u64 = 0;
	771	for (0..tests_len) \|i\| {
	772	names_len += tests.testName(@intCast(i)).len + 1;
	773	}
	774	break :len @sizeOf(abi.time_report.RunTestResult) + names_len + 8 * tests_len;
	775	};
	776	const old_buf = old: {
	777	ws.time_report_mutex.lock();
	778	defer ws.time_report_mutex.unlock();
	779	const old = ws.time_report_msgs[step_idx];
	780	ws.time_report_msgs[step_idx] = &.{};
	781	break :old old;
	782	};
	783	const buf = gpa.realloc(old_buf, new_len) catch @panic("out of memory");
	784
	785	const out_header: *align(1) abi.time_report.RunTestResult = @ptrCast(buf[0..@sizeOf(abi.time_report.RunTestResult)]);
	786	out_header.* = .{
	787	.step_idx = step_idx,
	788	.tests_len = tests_len,
	789	};
	790	var offset: usize = @sizeOf(abi.time_report.RunTestResult);
	791	const ns_per_test_out: []align(1) u64 = @ptrCast(buf[offset..][0 .. tests_len * 8]);
	792	@memcpy(ns_per_test_out, ns_per_test);
	793	offset += tests_len * 8;
	794	for (0..tests_len) \|i\| {
	795	const name = tests.testName(@intCast(i));
	796	@memcpy(buf[offset..][0..name.len], name);
	797	buf[offset..][name.len] = 0;
	798	offset += name.len + 1;
	799	}
	800	assert(offset == buf.len);
	801
	802	{
	803	ws.time_report_mutex.lock();
	804	defer ws.time_report_mutex.unlock();
	805	assert(ws.time_report_msgs[step_idx].len == 0);
	806	ws.time_report_msgs[step_idx] = buf;
	807	ws.time_report_update_times[step_idx] = ws.now();
	808	}
	809	ws.notifyUpdate();
	810	}
	811
754	812	const RunnerRequest = union(enum) {
755	813	rebuild,
756	814	};

lib/std/Build/abi.zig+16

...	...	@@ -56,6 +56,7 @@ pub const ToClientTag = enum(u8) {
56	56	// `--time-report`
57	57	time_report_generic_result,
58	58	time_report_compile_result,
	59	time_report_run_test_result,
59	60
60	61	_,
61	62	};
...	...	@@ -342,4 +343,19 @@ pub const time_report = struct {
342	343	};
343	344	};
344	345	};
	346
	347	/// WebSocket server->client.
	348	///
	349	/// Sent after a `Step.Run` for a Zig test executable finishes, providing the test's time report.
	350	///
	351	/// Trailing:
	352	/// * for each `tests_len`:
	353	/// * `test_ns: u64` (nanoseconds spent running this test)
	354	/// * for each `tests_len`:
	355	/// * `name` (null-terminated UTF-8 string)
	356	pub const RunTestResult = extern struct {
	357	tag: ToClientTag = .time_report_run_test_result,
	358	step_idx: u32 align(1),
	359	tests_len: u32 align(1),
	360	};
345	361	};

lib/std/zig/Server.zig+6

...	...	@@ -34,6 +34,12 @@ pub const Message = struct {
34	34	test_metadata,
35	35	/// Body is a TestResults
36	36	test_results,
	37	/// Does not have a body.
	38	/// Notifies the build runner that the next test (requested by `Client.Message.Tag.run_test`)
	39	/// is starting execution. This message helps to ensure that the timestamp used by the build
	40	/// runner to enforce unit test time limits is relatively accurate under extreme system load
	41	/// (where there may be a non-trivial delay before the test process is scheduled).
	42	test_started,
37	43	/// Body is a series of strings, delimited by null bytes.
38	44	/// Each string is a prefixed file path.
39	45	/// The first byte indicates the file prefix path (see prefixes fields