Commit 0156d68f0c9626074ca14d6c2cf677135e47e585 | zig

author	Loris Cro <kappaloris@gmail.com> 2025-11-04 21:11:40+01:00
committer	Loris Cro <kappaloris@gmail.com> 2025-11-07 12:43:25+01:00
log	0156d68f0c9626074ca14d6c2cf677135e47e585
tree	8527f8972db535a6c578bd1cec656c8c1bf3d1a0
parent	19af9fa488ba7ddf2e7fd8f2f06ab594f7e23f91

Io.Threaded PoC reimplementation

This is a reimplementation of Io.Threaded that fixes the issues
highlighted in the recent Zulip discussion. It's poorly tested but it
does successfully run to completion the litmust test example that I
offered in the discussion.

This implementation has the following key design decisions:

- `t.cpu_count` is used as the threadpool size.
- `t.concurrency_limit` is used as the maximum number of
  "burst, one-shot" threads that can be spawned by `io.concurrent` past
  `t.cpu_count`.
- `t.available_thread_count` is the number of threads in the pool that
  is not currently busy with work (the bookkeeping happens in the worker
  function).
- `t.one_shot_thread_count` is the number of active threads that were
  spawned by `io.concurrent` past `t.cpu_count`.

In this implementation:

- `io.async` first tries to decrement `t.available_thread_count`. If
  there are no threads available, it tries to spawn a new one if possible,
  otherwise it runs the task immediately.
- `io.concurrent` first tries to use a thread in the pool same as
  `io.async`, but on failure (no available threads and pool size limit
  reached) it tries to spawn a new one-shot thread. One shot threads
  run a different main function that just executes one task, decrements
  the number of active one shot threads, and then exits.

A relevant future improvement is to have one-shot threads stay on for a
few seconds (and potentially pick up a new task) to amortize spawning
costs.

1 files changed, 114 insertions(+), 79 deletions(-)

lib
- std
  - Io
    Threaded.zig

lib/std/Io/Threaded.zig+114-79

...	...	@@ -24,8 +24,10 @@ run_queue: std.SinglyLinkedList = .{},
24	24	join_requested: bool = false,
25	25	threads: std.ArrayListUnmanaged(std.Thread),
26	26	stack_size: usize,
27		cpu_count: std.Thread.CpuCountError!usize,
28		concurrent_count: usize,
	27	cpu_count: usize, // 0 means no limit
	28	concurrency_limit: usize, // 0 means no limit
	29	available_thread_count: usize = 0,
	30	one_shot_thread_count: usize = 0,
29	31
30	32	wsa: if (is_windows) Wsa else struct {} = .{},
31	33
...	...	@@ -70,8 +72,6 @@ const Closure = struct {
70	72	start: Start,
71	73	node: std.SinglyLinkedList.Node = .{},
72	74	cancel_tid: CancelId,
73		/// Whether this task bumps minimum number of threads in the pool.
74		is_concurrent: bool,
75	75
76	76	const Start = const fn (Closure) void;
77	77
...	...	@@ -103,20 +103,20 @@ pub fn init(
103	103	/// here.
104	104	gpa: Allocator,
105	105	) Threaded {
	106	assert(!builtin.single_threaded); // use 'init_single_threaded' instead
	107
106	108	var t: Threaded = .{
107	109	.allocator = gpa,
108	110	.threads = .empty,
109	111	.stack_size = std.Thread.SpawnConfig.default_stack_size,
110		.cpu_count = std.Thread.getCpuCount(),
111		.concurrent_count = 0,
	112	.cpu_count = std.Thread.getCpuCount() catch 0,
	113	.concurrency_limit = 0,
112	114	.old_sig_io = undefined,
113	115	.old_sig_pipe = undefined,
114	116	.have_signal_handler = false,
115	117	};
116	118
117		if (t.cpu_count) \|n\| {
118		t.threads.ensureTotalCapacityPrecise(gpa, n - 1) catch {};
119		} else \|_\| {}
	119	t.threads.ensureTotalCapacity(gpa, t.cpu_count) catch {};
120	120
121	121	if (posix.Sigaction != void) {
122	122	// This causes sending `posix.SIG.IO` to thread to interrupt blocking
...	...	@@ -145,7 +145,7 @@ pub const init_single_threaded: Threaded = .{
145	145	.threads = .empty,
146	146	.stack_size = std.Thread.SpawnConfig.default_stack_size,
147	147	.cpu_count = 1,
148		.concurrent_count = 0,
	148	.concurrency_limit = 0,
149	149	.old_sig_io = undefined,
150	150	.old_sig_pipe = undefined,
151	151	.have_signal_handler = false,
...	...	@@ -184,18 +184,22 @@ fn worker(t: *Threaded) void {
184	184	while (t.run_queue.popFirst()) \|closure_node\| {
185	185	t.mutex.unlock();
186	186	const closure: *Closure = @fieldParentPtr("node", closure_node);
187		const is_concurrent = closure.is_concurrent;
188	187	closure.start(closure);
189	188	t.mutex.lock();
190		if (is_concurrent) {
191		t.concurrent_count -= 1;
192		}
	189	t.available_thread_count += 1;
193	190	}
194	191	if (t.join_requested) break;
195	192	t.cond.wait(&t.mutex);
196	193	}
197	194	}
198	195
	196	fn oneShotWorker(t: Threaded, closure: Closure) void {
	197	closure.start(closure);
	198	t.mutex.lock();
	199	defer t.mutex.unlock();
	200	t.one_shot_thread_count -= 1;
	201	}
	202
199	203	pub fn io(t: *Threaded) Io {
200	204	return .{
201	205	.userdata = t,
...	...	@@ -448,22 +452,22 @@ fn async(
448	452	context_alignment: std.mem.Alignment,
449	453	start: const fn (context: const anyopaque, result: *anyopaque) void,
450	454	) ?*Io.AnyFuture {
451		if (builtin.single_threaded) {
	455	const t: *Threaded = @ptrCast(@alignCast(userdata));
	456	if (t.cpu_count == 1) {
452	457	start(context.ptr, result.ptr);
453	458	return null;
454	459	}
455		const t: *Threaded = @ptrCast(@alignCast(userdata));
456		const cpu_count = t.cpu_count catch {
457		return concurrent(userdata, result.len, result_alignment, context, context_alignment, start) catch {
458		start(context.ptr, result.ptr);
459		return null;
460		};
461		};
	460
462	461	const gpa = t.allocator;
463	462	const context_offset = context_alignment.forward(@sizeOf(AsyncClosure));
464	463	const result_offset = result_alignment.forward(context_offset + context.len);
465	464	const n = result_offset + result.len;
466		const ac: *AsyncClosure = @ptrCast(@alignCast(gpa.alignedAlloc(u8, .of(AsyncClosure), n) catch {
	465
	466	const ac: *AsyncClosure = @ptrCast(@alignCast(gpa.alignedAlloc(
	467	u8,
	468	.of(AsyncClosure),
	469	n,
	470	) catch {
467	471	start(context.ptr, result.ptr);
468	472	return null;
469	473	}));
...	...	@@ -472,7 +476,6 @@ fn async(
472	476	.closure = .{
473	477	.cancel_tid = .none,
474	478	.start = AsyncClosure.start,
475		.is_concurrent = false,
476	479	},
477	480	.func = start,
478	481	.context_alignment = context_alignment,
...	...	@@ -485,34 +488,38 @@ fn async(
485	488
486	489	t.mutex.lock();
487	490
488		const thread_capacity = cpu_count - 1 + t.concurrent_count;
489
490		t.threads.ensureTotalCapacityPrecise(gpa, thread_capacity) catch {
491		t.mutex.unlock();
492		ac.free(gpa, result.len);
493		start(context.ptr, result.ptr);
494		return null;
495		};
	491	if (t.available_thread_count == 0) {
	492	if (t.cpu_count != 0 and t.threads.items.len >= t.cpu_count) {
	493	t.mutex.unlock();
	494	ac.free(gpa, result.len);
	495	start(context.ptr, result.ptr);
	496	return null;
	497	}
496	498
497		t.run_queue.prepend(&ac.closure.node);
	499	t.threads.ensureUnusedCapacity(gpa, 1) catch {
	500	t.mutex.unlock();
	501	ac.free(gpa, result.len);
	502	start(context.ptr, result.ptr);
	503	return null;
	504	};
498	505
499		if (t.threads.items.len < thread_capacity) {
500		const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch {
501		if (t.threads.items.len == 0) {
502		assert(t.run_queue.popFirst() == &ac.closure.node);
503		t.mutex.unlock();
504		ac.free(gpa, result.len);
505		start(context.ptr, result.ptr);
506		return null;
507		}
508		// Rely on other workers to do it.
	506	const thread = std.Thread.spawn(
	507	.{ .stack_size = t.stack_size },
	508	worker,
	509	.{t},
	510	) catch {
509	511	t.mutex.unlock();
510		t.cond.signal();
511		return @ptrCast(ac);
	512	ac.free(gpa, result.len);
	513	start(context.ptr, result.ptr);
	514	return null;
512	515	};
	516
513	517	t.threads.appendAssumeCapacity(thread);
	518	} else {
	519	t.available_thread_count -= 1;
514	520	}
515	521
	522	t.run_queue.prepend(&ac.closure.node);
516	523	t.mutex.unlock();
517	524	t.cond.signal();
518	525	return @ptrCast(ac);
...	...	@@ -529,20 +536,21 @@ fn concurrent(
529	536	if (builtin.single_threaded) return error.ConcurrencyUnavailable;
530	537
531	538	const t: *Threaded = @ptrCast(@alignCast(userdata));
532		const cpu_count = t.cpu_count catch 1;
533	539	const gpa = t.allocator;
534	540	const context_offset = context_alignment.forward(@sizeOf(AsyncClosure));
535	541	const result_offset = result_alignment.forward(context_offset + context.len);
536	542	const n = result_offset + result_len;
537		const ac_bytes = gpa.alignedAlloc(u8, .of(AsyncClosure), n) catch
	543
	544	const ac_bytes = gpa.alignedAlloc(u8, .of(AsyncClosure), n) catch {
538	545	return error.ConcurrencyUnavailable;
539		const ac: *AsyncClosure = @ptrCast(@alignCast(ac_bytes));
	546	};
	547	errdefer gpa.free(ac_bytes);
540	548
	549	const ac: *AsyncClosure = @ptrCast(@alignCast(ac_bytes));
541	550	ac.* = .{
542	551	.closure = .{
543	552	.cancel_tid = .none,
544	553	.start = AsyncClosure.start,
545		.is_concurrent = true,
546	554	},
547	555	.func = start,
548	556	.context_alignment = context_alignment,
...	...	@@ -550,33 +558,52 @@ fn concurrent(
550	558	.reset_event = .unset,
551	559	.select_condition = null,
552	560	};
	561
553	562	@memcpy(ac.contextPointer()[0..context.len], context);
554	563
555	564	t.mutex.lock();
	565	defer t.mutex.unlock();
556	566
557		t.concurrent_count += 1;
558		const thread_capacity = cpu_count - 1 + t.concurrent_count;
	567	// If there's an avilable thread, use it.
	568	if (t.available_thread_count > 0) {
	569	t.available_thread_count -= 1;
	570	t.run_queue.prepend(&ac.closure.node);
	571	t.cond.signal();
	572	return @ptrCast(ac);
	573	}
559	574
560		t.threads.ensureTotalCapacity(gpa, thread_capacity) catch {
561		t.mutex.unlock();
562		ac.free(gpa, result_len);
563		return error.ConcurrencyUnavailable;
564		};
	575	// If we can spawn a normal worker, spawn it and use it.
	576	if (t.cpu_count == 0 or t.threads.items.len < t.cpu_count) {
	577	t.threads.ensureUnusedCapacity(gpa, 1) catch return error.ConcurrencyUnavailable;
565	578
566		t.run_queue.prepend(&ac.closure.node);
	579	const thread = std.Thread.spawn(
	580	.{ .stack_size = t.stack_size },
	581	worker,
	582	.{t},
	583	) catch return error.ConcurrencyUnavailable;
567	584
568		if (t.threads.items.len < thread_capacity) {
569		const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch {
570		assert(t.run_queue.popFirst() == &ac.closure.node);
571		t.mutex.unlock();
572		ac.free(gpa, result_len);
573		return error.ConcurrencyUnavailable;
574		};
575	585	t.threads.appendAssumeCapacity(thread);
	586	t.run_queue.prepend(&ac.closure.node);
	587	t.cond.signal();
	588	return @ptrCast(ac);
576	589	}
577	590
578		t.mutex.unlock();
579		t.cond.signal();
	591	// If we have a concurrencty limit and we havent' hit it yet,
	592	// spawn a new one-shot thread.
	593	if (t.concurrency_limit != 0 and t.one_shot_thread_count >= t.concurrency_limit) {
	594	return error.ConcurrencyUnavailable;
	595	}
	596
	597	t.one_shot_thread_count += 1;
	598	errdefer t.one_shot_thread_count -= 1;
	599
	600	const thread = std.Thread.spawn(
	601	.{ .stack_size = t.stack_size },
	602	oneShotWorker,
	603	.{ t, &ac.closure },
	604	) catch return error.ConcurrencyUnavailable;
	605	thread.detach();
	606
580	607	return @ptrCast(ac);
581	608	}
582	609
...	...	@@ -647,7 +674,6 @@ fn groupAsync(
647	674	) void {
648	675	if (builtin.single_threaded) return start(group, context.ptr);
649	676	const t: *Threaded = @ptrCast(@alignCast(userdata));
650		const cpu_count = t.cpu_count catch 1;
651	677	const gpa = t.allocator;
652	678	const n = GroupClosure.contextEnd(context_alignment, context.len);
653	679	const gc: *GroupClosure = @ptrCast(@alignCast(gpa.alignedAlloc(u8, .of(GroupClosure), n) catch {
...	...	@@ -657,7 +683,6 @@ fn groupAsync(
657	683	.closure = .{
658	684	.cancel_tid = .none,
659	685	.start = GroupClosure.start,
660		.is_concurrent = false,
661	686	},
662	687	.t = t,
663	688	.group = group,
...	...	@@ -674,26 +699,36 @@ fn groupAsync(
674	699	gc.node = .{ .next = @ptrCast(@alignCast(group.token)) };
675	700	group.token = &gc.node;
676	701
677		const thread_capacity = cpu_count - 1 + t.concurrent_count;
678
679		t.threads.ensureTotalCapacityPrecise(gpa, thread_capacity) catch {
680		t.mutex.unlock();
681		gc.free(gpa);
682		return start(group, context.ptr);
683		};
	702	if (t.available_thread_count == 0) {
	703	if (t.cpu_count != 0 and t.threads.items.len >= t.cpu_count) {
	704	t.mutex.unlock();
	705	gc.free(gpa);
	706	return start(group, context.ptr);
	707	}
684	708
685		t.run_queue.prepend(&gc.closure.node);
	709	t.threads.ensureUnusedCapacity(gpa, 1) catch {
	710	t.mutex.unlock();
	711	gc.free(gpa);
	712	return start(group, context.ptr);
	713	};
686	714
687		if (t.threads.items.len < thread_capacity) {
688		const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch {
689		assert(t.run_queue.popFirst() == &gc.closure.node);
	715	const thread = std.Thread.spawn(
	716	.{ .stack_size = t.stack_size },
	717	worker,
	718	.{t},
	719	) catch {
690	720	t.mutex.unlock();
691	721	gc.free(gpa);
692	722	return start(group, context.ptr);
693	723	};
	724
694	725	t.threads.appendAssumeCapacity(thread);
	726	} else {
	727	t.available_thread_count -= 1;
695	728	}
696	729
	730	t.run_queue.prepend(&gc.closure.node);
	731
697	732	// This needs to be done before unlocking the mutex to avoid a race with
698	733	// the associated task finishing.
699	734	const group_state: *std.atomic.Value(usize) = @ptrCast(&group.state);