Commit a242292644a12b8ca0485759ba45c550265da5bd | zig

lib/compiler/build_runner.zig+13-37

...	...	@@ -107,7 +107,6 @@ pub fn main() !void {
107	107
108	108	var targets = std.array_list.Managed([]const u8).init(arena);
109	109	var debug_log_scopes = std.array_list.Managed([]const u8).init(arena);
110		var thread_pool_options: std.Thread.Pool.Options = .{ .allocator = arena };
111	110
112	111	var install_prefix: ?[]const u8 = null;
113	112	var dir_list = std.Build.DirList{};
...	...	@@ -413,19 +412,11 @@ pub fn main() !void {
413	412	};
414	413	} else if (mem.eql(u8, arg, "-fno-reference-trace")) {
415	414	builder.reference_trace = null;
416		} else if (mem.startsWith(u8, arg, "-j")) {
417		const num = arg["-j".len..];
418		const n_jobs = std.fmt.parseUnsigned(u32, num, 10) catch \|err\| {
419		std.debug.print("unable to parse jobs count '{s}': {s}", .{
420		num, @errorName(err),
421		});
422		process.exit(1);
423		};
424		if (n_jobs < 1) {
425		std.debug.print("number of jobs must be at least 1\n", .{});
426		process.exit(1);
427		}
428		thread_pool_options.n_jobs = n_jobs;
	415	} else if (mem.cutPrefix(u8, arg, "-j")) \|text\| {
	416	const n = std.fmt.parseUnsigned(u32, text, 10) catch \|err\|
	417	fatal("unable to parse jobs count '{s}': {t}", .{ text, err });
	418	if (n < 1) fatal("number of jobs must be at least 1", .{});
	419	threaded.setAsyncLimit(.limited(n));
429	420	} else if (mem.eql(u8, arg, "--")) {
430	421	builder.args = argsRest(args, arg_idx);
431	422	break;
...	...	@@ -516,7 +507,6 @@ pub fn main() !void {
516	507	.error_style = error_style,
517	508	.multiline_errors = multiline_errors,
518	509	.summary = summary orelse if (watch or webui_listen != null) .line else .failures,
519		.thread_pool = undefined,
520	510
521	511	.ttyconf = ttyconf,
522	512	};
...	...	@@ -547,16 +537,12 @@ pub fn main() !void {
547	537	break :w try .init();
548	538	};
549	539
550		try run.thread_pool.init(thread_pool_options);
551		defer run.thread_pool.deinit();
552
553	540	const now = Io.Clock.Timestamp.now(io, .awake) catch \|err\| fatal("failed to collect timestamp: {t}", .{err});
554	541
555	542	run.web_server = if (webui_listen) \|listen_address\| ws: {
556	543	if (builtin.single_threaded) unreachable; // `fatal` above
557	544	break :ws .init(.{
558	545	.gpa = gpa,
559		.thread_pool = &run.thread_pool,
560	546	.ttyconf = ttyconf,
561	547	.graph = &graph,
562	548	.all_steps = run.step_stack.keys(),
...	...	@@ -675,7 +661,6 @@ const Run = struct {
675	661	memory_blocked_steps: std.ArrayList(*Step),
676	662	/// Allocated into `gpa`.
677	663	step_stack: std.AutoArrayHashMapUnmanaged(*Step, void),
678		thread_pool: std.Thread.Pool,
679	664	/// Similar to the `tty.Config` returned by `std.debug.lockStderrWriter`,
680	665	/// but also respects the '--color' flag.
681	666	ttyconf: tty.Config,
...	...	@@ -754,14 +739,13 @@ fn runStepNames(
754	739	const gpa = run.gpa;
755	740	const io = b.graph.io;
756	741	const step_stack = &run.step_stack;
757		const thread_pool = &run.thread_pool;
758	742
759	743	{
760	744	const step_prog = parent_prog_node.start("steps", step_stack.count());
761	745	defer step_prog.end();
762	746
763		var wait_group: std.Thread.WaitGroup = .{};
764		defer wait_group.wait();
	747	var group: Io.Group = .init;
	748	defer group.wait(io);
765	749
766	750	// Here we spawn the initial set of tasks with a nice heuristic -
767	751	// dependency order. Each worker when it finishes a step will then
...	...	@@ -771,9 +755,7 @@ fn runStepNames(
771	755	const step = steps_slice[steps_slice.len - i - 1];
772	756	if (step.state == .skipped_oom) continue;
773	757
774		thread_pool.spawnWg(&wait_group, workerMakeOneStep, .{
775		&wait_group, b, step, step_prog, run,
776		});
	758	group.async(io, workerMakeOneStep, .{ &group, b, step, step_prog, run });
777	759	}
778	760	}
779	761
...	...	@@ -855,7 +837,6 @@ fn runStepNames(
855	837	var f = std.Build.Fuzz.init(
856	838	gpa,
857	839	io,
858		thread_pool,
859	840	run.ttyconf,
860	841	step_stack.keys(),
861	842	parent_prog_node,
...	...	@@ -1318,14 +1299,12 @@ fn constructGraphAndCheckForDependencyLoop(
1318	1299	}
1319	1300
1320	1301	fn workerMakeOneStep(
1321		wg: *std.Thread.WaitGroup,
	1302	group: *Io.Group,
1322	1303	b: *std.Build,
1323	1304	s: *Step,
1324	1305	prog_node: std.Progress.Node,
1325	1306	run: *Run,
1326	1307	) void {
1327		const thread_pool = &run.thread_pool;
1328
1329	1308	// First, check the conditions for running this step. If they are not met,
1330	1309	// then we return without doing the step, relying on another worker to
1331	1310	// queue this step up again when dependencies are met.
...	...	@@ -1381,7 +1360,6 @@ fn workerMakeOneStep(
1381	1360
1382	1361	const make_result = s.make(.{
1383	1362	.progress_node = sub_prog_node,
1384		.thread_pool = thread_pool,
1385	1363	.watch = run.watch,
1386	1364	.web_server = if (run.web_server) \|*ws\| ws else null,
1387	1365	.ttyconf = run.ttyconf,
...	...	@@ -1400,6 +1378,8 @@ fn workerMakeOneStep(
1400	1378	printErrorMessages(run.gpa, s, .{}, bw, ttyconf, run.error_style, run.multiline_errors) catch {};
1401	1379	}
1402	1380
	1381	const io = b.graph.io;
	1382
1403	1383	handle_result: {
1404	1384	if (make_result) \|_\| {
1405	1385	@atomicStore(Step.State, &s.state, .success, .seq_cst);
...	...	@@ -1419,9 +1399,7 @@ fn workerMakeOneStep(
1419	1399
1420	1400	// Successful completion of a step, so we queue up its dependants as well.
1421	1401	for (s.dependants.items) \|dep\| {
1422		thread_pool.spawnWg(wg, workerMakeOneStep, .{
1423		wg, b, dep, prog_node, run,
1424		});
	1402	group.async(io, workerMakeOneStep, .{ group, b, dep, prog_node, run });
1425	1403	}
1426	1404	}
1427	1405
...	...	@@ -1444,9 +1422,7 @@ fn workerMakeOneStep(
1444	1422	if (dep.max_rss <= remaining) {
1445	1423	remaining -= dep.max_rss;
1446	1424
1447		thread_pool.spawnWg(wg, workerMakeOneStep, .{
1448		wg, b, dep, prog_node, run,
1449		});
	1425	group.async(io, workerMakeOneStep, .{ group, b, dep, prog_node, run });
1450	1426	} else {
1451	1427	run.memory_blocked_steps.items[i] = dep;
1452	1428	i += 1;

lib/std/Build/Fuzz.zig+16-22

...	...	@@ -22,10 +22,9 @@ mode: Mode,
22	22	/// Allocated into `gpa`.
23	23	run_steps: []const *Step.Run,
24	24
25		wait_group: std.Thread.WaitGroup,
	25	group: Io.Group,
26	26	root_prog_node: std.Progress.Node,
27	27	prog_node: std.Progress.Node,
28		thread_pool: *std.Thread.Pool,
29	28
30	29	/// Protects `coverage_files`.
31	30	coverage_mutex: std.Thread.Mutex,
...	...	@@ -78,7 +77,6 @@ const CoverageMap = struct {
78	77	pub fn init(
79	78	gpa: Allocator,
80	79	io: Io,
81		thread_pool: *std.Thread.Pool,
82	80	ttyconf: tty.Config,
83	81	all_steps: []const *Build.Step,
84	82	root_prog_node: std.Progress.Node,
...	...	@@ -89,20 +87,22 @@ pub fn init(
89	87	defer steps.deinit(gpa);
90	88	const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0);
91	89	defer rebuild_node.end();
92		var rebuild_wg: std.Thread.WaitGroup = .{};
93		defer rebuild_wg.wait();
	90	var rebuild_group: Io.Group = .init;
	91	defer rebuild_group.cancel(io);
94	92
95	93	for (all_steps) \|step\| {
96	94	const run = step.cast(Step.Run) orelse continue;
97	95	if (run.producer == null) continue;
98	96	if (run.fuzz_tests.items.len == 0) continue;
99	97	try steps.append(gpa, run);
100		thread_pool.spawnWg(&rebuild_wg, rebuildTestsWorkerRun, .{ run, gpa, ttyconf, rebuild_node });
	98	rebuild_group.async(io, rebuildTestsWorkerRun, .{ run, gpa, ttyconf, rebuild_node });
101	99	}
102	100
103	101	if (steps.items.len == 0) fatal("no fuzz tests found", .{});
104	102	rebuild_node.setEstimatedTotalItems(steps.items.len);
105		break :steps try gpa.dupe(*Step.Run, steps.items);
	103	const run_steps = try gpa.dupe(*Step.Run, steps.items);
	104	rebuild_group.wait(io);
	105	break :steps run_steps;
106	106	};
107	107	errdefer gpa.free(run_steps);
108	108
...	...	@@ -118,8 +118,7 @@ pub fn init(
118	118	.ttyconf = ttyconf,
119	119	.mode = mode,
120	120	.run_steps = run_steps,
121		.wait_group = .{},
122		.thread_pool = thread_pool,
	121	.group = .init,
123	122	.root_prog_node = root_prog_node,
124	123	.prog_node = .none,
125	124	.coverage_files = .empty,
...	...	@@ -131,29 +130,26 @@ pub fn init(
131	130	}
132	131
133	132	pub fn start(fuzz: *Fuzz) void {
	133	const io = fuzz.io;
134	134	fuzz.prog_node = fuzz.root_prog_node.start("Fuzzing", fuzz.run_steps.len);
135	135
136	136	if (fuzz.mode == .forever) {
137	137	// For polling messages and sending updates to subscribers.
138		fuzz.wait_group.start();
139		_ = std.Thread.spawn(.{}, coverageRun, .{fuzz}) catch \|err\| {
140		fuzz.wait_group.finish();
141		fatal("unable to spawn coverage thread: {s}", .{@errorName(err)});
142		};
	138	fuzz.group.concurrent(io, coverageRun, .{fuzz}) catch \|err\|
	139	fatal("unable to spawn coverage task: {t}", .{err});
143	140	}
144	141
145	142	for (fuzz.run_steps) \|run\| {
146	143	for (run.fuzz_tests.items) \|unit_test_index\| {
147	144	assert(run.rebuilt_executable != null);
148		fuzz.thread_pool.spawnWg(&fuzz.wait_group, fuzzWorkerRun, .{
149		fuzz, run, unit_test_index,
150		});
	145	fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run, unit_test_index });
151	146	}
152	147	}
153	148	}
154	149
155	150	pub fn deinit(fuzz: *Fuzz) void {
156		if (!fuzz.wait_group.isDone()) @panic("TODO: terminate the fuzzer processes");
	151	const io = fuzz.io;
	152	fuzz.group.cancel(io);
157	153	fuzz.prog_node.end();
158	154	fuzz.gpa.free(fuzz.run_steps);
159	155	}
...	...	@@ -335,8 +331,6 @@ pub fn sendUpdate(
335	331	}
336	332
337	333	fn coverageRun(fuzz: *Fuzz) void {
338		defer fuzz.wait_group.finish();
339
340	334	fuzz.queue_mutex.lock();
341	335	defer fuzz.queue_mutex.unlock();
342	336
...	...	@@ -511,8 +505,8 @@ pub fn waitAndPrintReport(fuzz: *Fuzz) void {
511	505	assert(fuzz.mode == .limit);
512	506	const io = fuzz.io;
513	507
514		fuzz.wait_group.wait();
515		fuzz.wait_group.reset();
	508	fuzz.group.wait(io);
	509	fuzz.group = .init;
516	510
517	511	std.debug.print("======= FUZZING REPORT =======\n", .{});
518	512	for (fuzz.msg_queue.items) \|msg\| {

lib/std/Build/Step.zig-1

...	...	@@ -110,7 +110,6 @@ pub const TestResults = struct {
110	110
111	111	pub const MakeOptions = struct {
112	112	progress_node: std.Progress.Node,
113		thread_pool: *std.Thread.Pool,
114	113	watch: bool,
115	114	web_server: switch (builtin.target.cpu.arch) {
116	115	else => ?*Build.WebServer,

lib/std/Build/Step/Run.zig-1

...	...	@@ -1151,7 +1151,6 @@ pub fn rerunInFuzzMode(
1151	1151	const tmp_dir_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int);
1152	1152	try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, .{
1153	1153	.progress_node = prog_node,
1154		.thread_pool = undefined, // not used by `runCommand`
1155	1154	.watch = undefined, // not used by `runCommand`
1156	1155	.web_server = null, // only needed for time reports
1157	1156	.ttyconf = fuzz.ttyconf,

lib/std/Build/WebServer.zig-4

...	...	@@ -1,5 +1,4 @@
1	1	gpa: Allocator,
2		thread_pool: *std.Thread.Pool,
3	2	graph: *const Build.Graph,
4	3	all_steps: []const *Build.Step,
5	4	listen_address: net.IpAddress,
...	...	@@ -53,7 +52,6 @@ pub fn notifyUpdate(ws: *WebServer) void {
53	52
54	53	pub const Options = struct {
55	54	gpa: Allocator,
56		thread_pool: *std.Thread.Pool,
57	55	ttyconf: Io.tty.Config,
58	56	graph: *const std.Build.Graph,
59	57	all_steps: []const *Build.Step,
...	...	@@ -100,7 +98,6 @@ pub fn init(opts: Options) WebServer {
100	98
101	99	return .{
102	100	.gpa = opts.gpa,
103		.thread_pool = opts.thread_pool,
104	101	.ttyconf = opts.ttyconf,
105	102	.graph = opts.graph,
106	103	.all_steps = all_steps,
...	...	@@ -235,7 +232,6 @@ pub fn finishBuild(ws: *WebServer, opts: struct {
235	232	ws.fuzz = Fuzz.init(
236	233	ws.gpa,
237	234	ws.graph.io,
238		ws.thread_pool,
239	235	ws.ttyconf,
240	236	ws.all_steps,
241	237	ws.root_prog_node,

lib/std/Io/Threaded.zig+11-3

...	...	@@ -33,6 +33,9 @@ wait_group: std.Thread.WaitGroup = .{},
33	33	/// immediately.
34	34	///
35	35	/// Defaults to a number equal to logical CPU cores.
	36	///
	37	/// Protected by `mutex` once the I/O instance is already in use. See
	38	/// `setAsyncLimit`.
36	39	async_limit: Io.Limit,
37	40	/// Maximum thread pool size (excluding main thread) for dispatching concurrent
38	41	/// tasks. Until this limit, calls to `Io.concurrent` will increase the thread
...	...	@@ -168,6 +171,12 @@ pub const init_single_threaded: Threaded = .{
168	171	.have_signal_handler = false,
169	172	};
170	173
	174	pub fn setAsyncLimit(t: *Threaded, new_limit: Io.Limit) void {
	175	t.mutex.lock();
	176	defer t.mutex.unlock();
	177	t.async_limit = new_limit;
	178	}
	179
171	180	pub fn deinit(t: *Threaded) void {
172	181	t.join();
173	182	if (is_windows and t.wsa.status == .initialized) {
...	...	@@ -507,7 +516,7 @@ fn async(
507	516	start: const fn (context: const anyopaque, result: *anyopaque) void,
508	517	) ?*Io.AnyFuture {
509	518	const t: *Threaded = @ptrCast(@alignCast(userdata));
510		if (builtin.single_threaded or t.async_limit == .nothing) {
	519	if (builtin.single_threaded) {
511	520	start(context.ptr, result.ptr);
512	521	return null;
513	522	}
...	...	@@ -684,8 +693,7 @@ fn groupAsync(
684	693	start: const fn (Io.Group, context: *const anyopaque) void,
685	694	) void {
686	695	const t: *Threaded = @ptrCast(@alignCast(userdata));
687		if (builtin.single_threaded or t.async_limit == .nothing)
688		return start(group, context.ptr);
	696	if (builtin.single_threaded) return start(group, context.ptr);
689	697
690	698	const gpa = t.allocator;
691	699	const gc = GroupClosure.init(gpa, t, group, context, context_alignment, start) catch

author	Andrew Kelley <andrew@ziglang.org> 2025-11-24 07:37:05-08:00
committer	Andrew Kelley <andrew@ziglang.org> 2025-11-24 14:34:18-08:00
log	a242292644a12b8ca0485759ba45c550265da5bd
tree	e994615836c82f0ff4b7fed8ac0b1c4540ca9149
parent	32dc46aae56623bff9b1fc792d49913f9295be7b

build runner: update from std.Thread.Pool to std.Io