authorgravatar for kappaloris@gmail.comLoris Cro <kappaloris@gmail.com> 2025-11-04 21:11:40+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-11-21 19:54:41-08:00
log8eaebf5939491b005e392698ec6e890ebaf0f86b
tree0b4d293734202a30cff6be585149402e4bea1abf
parentd54fbc01234688b37a48b29fee499529a500ccf5

Io.Threaded PoC reimplementation

This is a reimplementation of Io.Threaded that fixes the issues highlighted in the recent Zulip discussion. It's poorly tested but it does successfully run to completion the litmust test example that I offered in the discussion. This implementation has the following key design decisions: - `t.cpu_count` is used as the threadpool size. - `t.concurrency_limit` is used as the maximum number of "burst, one-shot" threads that can be spawned by `io.concurrent` past `t.cpu_count`. - `t.available_thread_count` is the number of threads in the pool that is not currently busy with work (the bookkeeping happens in the worker function). - `t.one_shot_thread_count` is the number of active threads that were spawned by `io.concurrent` past `t.cpu_count`. In this implementation: - `io.async` first tries to decrement `t.available_thread_count`. If there are no threads available, it tries to spawn a new one if possible, otherwise it runs the task immediately. - `io.concurrent` first tries to use a thread in the pool same as `io.async`, but on failure (no available threads and pool size limit reached) it tries to spawn a new one-shot thread. One shot threads run a different main function that just executes one task, decrements the number of active one shot threads, and then exits. A relevant future improvement is to have one-shot threads stay on for a few seconds (and potentially pick up a new task) to amortize spawning costs.

1 files changed, 98 insertions(+), 86 deletions(-)

lib/std/Io/Threaded.zig+98-86
...@@ -24,8 +24,10 @@ run_queue: std.SinglyLinkedList = .{},...@@ -24,8 +24,10 @@ run_queue: std.SinglyLinkedList = .{},
24join_requested: bool = false,24join_requested: bool = false,
25threads: std.ArrayList(std.Thread),25threads: std.ArrayList(std.Thread),
26stack_size: usize,26stack_size: usize,
27cpu_count: std.Thread.CpuCountError!usize,27cpu_count: usize, // 0 means no limit
28concurrent_count: usize,28concurrency_limit: usize, // 0 means no limit
29available_thread_count: usize = 0,
30one_shot_thread_count: usize = 0,
2931
30wsa: if (is_windows) Wsa else struct {} = .{},32wsa: if (is_windows) Wsa else struct {} = .{},
3133
...@@ -70,8 +72,6 @@ const Closure = struct {...@@ -70,8 +72,6 @@ const Closure = struct {
70 start: Start,72 start: Start,
71 node: std.SinglyLinkedList.Node = .{},73 node: std.SinglyLinkedList.Node = .{},
72 cancel_tid: CancelId,74 cancel_tid: CancelId,
73 /// Whether this task bumps minimum number of threads in the pool.
74 is_concurrent: bool,
7575
76 const Start = *const fn (*Closure) void;76 const Start = *const fn (*Closure) void;
7777
...@@ -103,20 +103,20 @@ pub fn init(...@@ -103,20 +103,20 @@ pub fn init(
103 /// here.103 /// here.
104 gpa: Allocator,104 gpa: Allocator,
105) Threaded {105) Threaded {
106 assert(!builtin.single_threaded); // use 'init_single_threaded' instead
107
106 var t: Threaded = .{108 var t: Threaded = .{
107 .allocator = gpa,109 .allocator = gpa,
108 .threads = .empty,110 .threads = .empty,
109 .stack_size = std.Thread.SpawnConfig.default_stack_size,111 .stack_size = std.Thread.SpawnConfig.default_stack_size,
110 .cpu_count = std.Thread.getCpuCount(),112 .cpu_count = std.Thread.getCpuCount() catch 0,
111 .concurrent_count = 0,113 .concurrency_limit = 0,
112 .old_sig_io = undefined,114 .old_sig_io = undefined,
113 .old_sig_pipe = undefined,115 .old_sig_pipe = undefined,
114 .have_signal_handler = false,116 .have_signal_handler = false,
115 };117 };
116118
117 if (t.cpu_count) |n| {119 t.threads.ensureTotalCapacity(gpa, t.cpu_count) catch {};
118 t.threads.ensureTotalCapacityPrecise(gpa, n - 1) catch {};
119 } else |_| {}
120120
121 if (posix.Sigaction != void) {121 if (posix.Sigaction != void) {
122 // This causes sending `posix.SIG.IO` to thread to interrupt blocking122 // This causes sending `posix.SIG.IO` to thread to interrupt blocking
...@@ -145,7 +145,7 @@ pub const init_single_threaded: Threaded = .{...@@ -145,7 +145,7 @@ pub const init_single_threaded: Threaded = .{
145 .threads = .empty,145 .threads = .empty,
146 .stack_size = std.Thread.SpawnConfig.default_stack_size,146 .stack_size = std.Thread.SpawnConfig.default_stack_size,
147 .cpu_count = 1,147 .cpu_count = 1,
148 .concurrent_count = 0,148 .concurrency_limit = 0,
149 .old_sig_io = undefined,149 .old_sig_io = undefined,
150 .old_sig_pipe = undefined,150 .old_sig_pipe = undefined,
151 .have_signal_handler = false,151 .have_signal_handler = false,
...@@ -184,18 +184,22 @@ fn worker(t: *Threaded) void {...@@ -184,18 +184,22 @@ fn worker(t: *Threaded) void {
184 while (t.run_queue.popFirst()) |closure_node| {184 while (t.run_queue.popFirst()) |closure_node| {
185 t.mutex.unlock();185 t.mutex.unlock();
186 const closure: *Closure = @fieldParentPtr("node", closure_node);186 const closure: *Closure = @fieldParentPtr("node", closure_node);
187 const is_concurrent = closure.is_concurrent;
188 closure.start(closure);187 closure.start(closure);
189 t.mutex.lock();188 t.mutex.lock();
190 if (is_concurrent) {189 t.available_thread_count += 1;
191 t.concurrent_count -= 1;
192 }
193 }190 }
194 if (t.join_requested) break;191 if (t.join_requested) break;
195 t.cond.wait(&t.mutex);192 t.cond.wait(&t.mutex);
196 }193 }
197}194}
198195
196fn oneShotWorker(t: *Threaded, closure: *Closure) void {
197 closure.start(closure);
198 t.mutex.lock();
199 defer t.mutex.unlock();
200 t.one_shot_thread_count -= 1;
201}
202
199pub fn io(t: *Threaded) Io {203pub fn io(t: *Threaded) Io {
200 return .{204 return .{
201 .userdata = t,205 .userdata = t,
...@@ -432,7 +436,6 @@ const AsyncClosure = struct {...@@ -432,7 +436,6 @@ const AsyncClosure = struct {
432436
433 fn init(437 fn init(
434 gpa: Allocator,438 gpa: Allocator,
435 mode: enum { async, concurrent },
436 result_len: usize,439 result_len: usize,
437 result_alignment: std.mem.Alignment,440 result_alignment: std.mem.Alignment,
438 context: []const u8,441 context: []const u8,
...@@ -454,10 +457,6 @@ const AsyncClosure = struct {...@@ -454,10 +457,6 @@ const AsyncClosure = struct {
454 .closure = .{457 .closure = .{
455 .cancel_tid = .none,458 .cancel_tid = .none,
456 .start = start,459 .start = start,
457 .is_concurrent = switch (mode) {
458 .async => false,
459 .concurrent => true,
460 },
461 },460 },
462 .func = func,461 .func = func,
463 .context_alignment = context_alignment,462 .context_alignment = context_alignment,
...@@ -490,55 +489,51 @@ fn async(...@@ -490,55 +489,51 @@ fn async(
490 context_alignment: std.mem.Alignment,489 context_alignment: std.mem.Alignment,
491 start: *const fn (context: *const anyopaque, result: *anyopaque) void,490 start: *const fn (context: *const anyopaque, result: *anyopaque) void,
492) ?*Io.AnyFuture {491) ?*Io.AnyFuture {
493 if (builtin.single_threaded) {492 const t: *Threaded = @ptrCast(@alignCast(userdata));
493 if (t.cpu_count == 1) {
494 start(context.ptr, result.ptr);494 start(context.ptr, result.ptr);
495 return null;495 return null;
496 }496 }
497
498 const t: *Threaded = @ptrCast(@alignCast(userdata));
499 const cpu_count = t.cpu_count catch {
500 return concurrent(userdata, result.len, result_alignment, context, context_alignment, start) catch {
501 start(context.ptr, result.ptr);
502 return null;
503 };
504 };
505
506 const gpa = t.allocator;497 const gpa = t.allocator;
507 const ac = AsyncClosure.init(gpa, .async, result.len, result_alignment, context, context_alignment, start) catch {498 const ac = AsyncClosure.init(gpa, result.len, result_alignment, context, context_alignment, start) catch {
508 start(context.ptr, result.ptr);499 start(context.ptr, result.ptr);
509 return null;500 return null;
510 };501 };
511502
512 t.mutex.lock();503 t.mutex.lock();
513504
514 const thread_capacity = cpu_count - 1 + t.concurrent_count;505 if (t.available_thread_count == 0) {
515506 if (t.cpu_count != 0 and t.threads.items.len >= t.cpu_count) {
516 t.threads.ensureTotalCapacityPrecise(gpa, thread_capacity) catch {507 t.mutex.unlock();
517 t.mutex.unlock();508 ac.deinit(gpa);
518 ac.deinit(gpa);509 start(context.ptr, result.ptr);
519 start(context.ptr, result.ptr);510 return null;
520 return null;511 }
521 };
522512
523 t.run_queue.prepend(&ac.closure.node);513 t.threads.ensureUnusedCapacity(gpa, 1) catch {
514 t.mutex.unlock();
515 ac.deinit(gpa);
516 start(context.ptr, result.ptr);
517 return null;
518 };
524519
525 if (t.threads.items.len < thread_capacity) {520 const thread = std.Thread.spawn(
526 const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch {521 .{ .stack_size = t.stack_size },
527 if (t.threads.items.len == 0) {522 worker,
528 assert(t.run_queue.popFirst() == &ac.closure.node);523 .{t},
529 t.mutex.unlock();524 ) catch {
530 ac.deinit(gpa);
531 start(context.ptr, result.ptr);
532 return null;
533 }
534 // Rely on other workers to do it.
535 t.mutex.unlock();525 t.mutex.unlock();
536 t.cond.signal();526 ac.deinit(gpa);
537 return @ptrCast(ac);527 start(context.ptr, result.ptr);
528 return null;
538 };529 };
530
539 t.threads.appendAssumeCapacity(thread);531 t.threads.appendAssumeCapacity(thread);
532 } else {
533 t.available_thread_count -= 1;
540 }534 }
541535
536 t.run_queue.prepend(&ac.closure.node);
542 t.mutex.unlock();537 t.mutex.unlock();
543 t.cond.signal();538 t.cond.signal();
544 return @ptrCast(ac);539 return @ptrCast(ac);
...@@ -555,38 +550,49 @@ fn concurrent(...@@ -555,38 +550,49 @@ fn concurrent(
555 if (builtin.single_threaded) return error.ConcurrencyUnavailable;550 if (builtin.single_threaded) return error.ConcurrencyUnavailable;
556551
557 const t: *Threaded = @ptrCast(@alignCast(userdata));552 const t: *Threaded = @ptrCast(@alignCast(userdata));
558 const cpu_count = t.cpu_count catch 1;
559553
560 const gpa = t.allocator;554 const gpa = t.allocator;
561 const ac = AsyncClosure.init(gpa, .concurrent, result_len, result_alignment, context, context_alignment, start) catch {555 const ac = AsyncClosure.init(gpa, result_len, result_alignment, context, context_alignment, start) catch {
562 return error.ConcurrencyUnavailable;556 return error.ConcurrencyUnavailable;
563 };557 };
558 errdefer ac.deinit(gpa);
564559
565 t.mutex.lock();560 t.mutex.lock();
561 defer t.mutex.unlock();
566562
567 t.concurrent_count += 1;563 // If there's an avilable thread, use it.
568 const thread_capacity = cpu_count - 1 + t.concurrent_count;564 if (t.available_thread_count > 0) {
569565 t.available_thread_count -= 1;
570 t.threads.ensureTotalCapacity(gpa, thread_capacity) catch {566 t.run_queue.prepend(&ac.closure.node);
571 t.mutex.unlock();567 t.cond.signal();
572 ac.deinit(gpa);568 return @ptrCast(ac);
573 return error.ConcurrencyUnavailable;569 }
574 };
575570
576 t.run_queue.prepend(&ac.closure.node);571 // If we can spawn a normal worker, spawn it and use it.
572 if (t.cpu_count == 0 or t.threads.items.len < t.cpu_count) {
573 t.threads.ensureUnusedCapacity(gpa, 1) catch return error.ConcurrencyUnavailable;
577574
578 if (t.threads.items.len < thread_capacity) {575 const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch
579 const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch {
580 assert(t.run_queue.popFirst() == &ac.closure.node);
581 t.mutex.unlock();
582 ac.deinit(gpa);
583 return error.ConcurrencyUnavailable;576 return error.ConcurrencyUnavailable;
584 };577
585 t.threads.appendAssumeCapacity(thread);578 t.threads.appendAssumeCapacity(thread);
579 t.run_queue.prepend(&ac.closure.node);
580 t.cond.signal();
581 return @ptrCast(ac);
586 }582 }
587583
588 t.mutex.unlock();584 // If we have a concurrencty limit and we havent' hit it yet,
589 t.cond.signal();585 // spawn a new one-shot thread.
586 if (t.concurrency_limit != 0 and t.one_shot_thread_count >= t.concurrency_limit)
587 return error.ConcurrencyUnavailable;
588
589 t.one_shot_thread_count += 1;
590 errdefer t.one_shot_thread_count -= 1;
591
592 const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, oneShotWorker, .{ t, &ac.closure }) catch
593 return error.ConcurrencyUnavailable;
594 thread.detach();
595
590 return @ptrCast(ac);596 return @ptrCast(ac);
591}597}
592598
...@@ -652,7 +658,6 @@ const GroupClosure = struct {...@@ -652,7 +658,6 @@ const GroupClosure = struct {
652 .closure = .{658 .closure = .{
653 .cancel_tid = .none,659 .cancel_tid = .none,
654 .start = start,660 .start = start,
655 .is_concurrent = false,
656 },661 },
657 .t = t,662 .t = t,
658 .group = group,663 .group = group,
...@@ -684,12 +689,9 @@ fn groupAsync(...@@ -684,12 +689,9 @@ fn groupAsync(
684 if (builtin.single_threaded) return start(group, context.ptr);689 if (builtin.single_threaded) return start(group, context.ptr);
685690
686 const t: *Threaded = @ptrCast(@alignCast(userdata));691 const t: *Threaded = @ptrCast(@alignCast(userdata));
687 const cpu_count = t.cpu_count catch 1;
688
689 const gpa = t.allocator;692 const gpa = t.allocator;
690 const gc = GroupClosure.init(gpa, t, group, context, context_alignment, start) catch {693 const gc = GroupClosure.init(gpa, t, group, context, context_alignment, start) catch
691 return start(group, context.ptr);694 return start(group, context.ptr);
692 };
693695
694 t.mutex.lock();696 t.mutex.lock();
695697
...@@ -697,26 +699,36 @@ fn groupAsync(...@@ -697,26 +699,36 @@ fn groupAsync(
697 gc.node = .{ .next = @ptrCast(@alignCast(group.token)) };699 gc.node = .{ .next = @ptrCast(@alignCast(group.token)) };
698 group.token = &gc.node;700 group.token = &gc.node;
699701
700 const thread_capacity = cpu_count - 1 + t.concurrent_count;702 if (t.available_thread_count == 0) {
701703 if (t.cpu_count != 0 and t.threads.items.len >= t.cpu_count) {
702 t.threads.ensureTotalCapacityPrecise(gpa, thread_capacity) catch {704 t.mutex.unlock();
703 t.mutex.unlock();705 gc.deinit(gpa);
704 gc.deinit(gpa);706 return start(group, context.ptr);
705 return start(group, context.ptr);707 }
706 };
707708
708 t.run_queue.prepend(&gc.closure.node);709 t.threads.ensureUnusedCapacity(gpa, 1) catch {
710 t.mutex.unlock();
711 gc.deinit(gpa);
712 return start(group, context.ptr);
713 };
709714
710 if (t.threads.items.len < thread_capacity) {715 const thread = std.Thread.spawn(
711 const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch {716 .{ .stack_size = t.stack_size },
712 assert(t.run_queue.popFirst() == &gc.closure.node);717 worker,
718 .{t},
719 ) catch {
713 t.mutex.unlock();720 t.mutex.unlock();
714 gc.deinit(gpa);721 gc.deinit(gpa);
715 return start(group, context.ptr);722 return start(group, context.ptr);
716 };723 };
724
717 t.threads.appendAssumeCapacity(thread);725 t.threads.appendAssumeCapacity(thread);
726 } else {
727 t.available_thread_count -= 1;
718 }728 }
719729
730 t.run_queue.prepend(&gc.closure.node);
731
720 // This needs to be done before unlocking the mutex to avoid a race with732 // This needs to be done before unlocking the mutex to avoid a race with
721 // the associated task finishing.733 // the associated task finishing.
722 const group_state: *std.atomic.Value(usize) = @ptrCast(&group.state);734 const group_state: *std.atomic.Value(usize) = @ptrCast(&group.state);