authorgravatar for kappaloris@gmail.comLoris Cro <kappaloris@gmail.com> 2025-11-04 21:11:40+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-11-21 19:54:41-08:00
log8eaebf5939491b005e392698ec6e890ebaf0f86b
tree0b4d293734202a30cff6be585149402e4bea1abf
parentd54fbc01234688b37a48b29fee499529a500ccf5

Io.Threaded PoC reimplementation

This is a reimplementation of Io.Threaded that fixes the issues highlighted in the recent Zulip discussion. It's poorly tested but it does successfully run to completion the litmust test example that I offered in the discussion. This implementation has the following key design decisions: - `t.cpu_count` is used as the threadpool size. - `t.concurrency_limit` is used as the maximum number of "burst, one-shot" threads that can be spawned by `io.concurrent` past `t.cpu_count`. - `t.available_thread_count` is the number of threads in the pool that is not currently busy with work (the bookkeeping happens in the worker function). - `t.one_shot_thread_count` is the number of active threads that were spawned by `io.concurrent` past `t.cpu_count`. In this implementation: - `io.async` first tries to decrement `t.available_thread_count`. If there are no threads available, it tries to spawn a new one if possible, otherwise it runs the task immediately. - `io.concurrent` first tries to use a thread in the pool same as `io.async`, but on failure (no available threads and pool size limit reached) it tries to spawn a new one-shot thread. One shot threads run a different main function that just executes one task, decrements the number of active one shot threads, and then exits. A relevant future improvement is to have one-shot threads stay on for a few seconds (and potentially pick up a new task) to amortize spawning costs.

1 files changed, 98 insertions(+), 86 deletions(-)

lib/std/Io/Threaded.zig+98-86
......@@ -24,8 +24,10 @@ run_queue: std.SinglyLinkedList = .{},
2424join_requested: bool = false,
2525threads: std.ArrayList(std.Thread),
2626stack_size: usize,
27cpu_count: std.Thread.CpuCountError!usize,
28concurrent_count: usize,
27cpu_count: usize, // 0 means no limit
28concurrency_limit: usize, // 0 means no limit
29available_thread_count: usize = 0,
30one_shot_thread_count: usize = 0,
2931
3032wsa: if (is_windows) Wsa else struct {} = .{},
3133
......@@ -70,8 +72,6 @@ const Closure = struct {
7072 start: Start,
7173 node: std.SinglyLinkedList.Node = .{},
7274 cancel_tid: CancelId,
73 /// Whether this task bumps minimum number of threads in the pool.
74 is_concurrent: bool,
7575
7676 const Start = *const fn (*Closure) void;
7777
......@@ -103,20 +103,20 @@ pub fn init(
103103 /// here.
104104 gpa: Allocator,
105105) Threaded {
106 assert(!builtin.single_threaded); // use 'init_single_threaded' instead
107
106108 var t: Threaded = .{
107109 .allocator = gpa,
108110 .threads = .empty,
109111 .stack_size = std.Thread.SpawnConfig.default_stack_size,
110 .cpu_count = std.Thread.getCpuCount(),
111 .concurrent_count = 0,
112 .cpu_count = std.Thread.getCpuCount() catch 0,
113 .concurrency_limit = 0,
112114 .old_sig_io = undefined,
113115 .old_sig_pipe = undefined,
114116 .have_signal_handler = false,
115117 };
116118
117 if (t.cpu_count) |n| {
118 t.threads.ensureTotalCapacityPrecise(gpa, n - 1) catch {};
119 } else |_| {}
119 t.threads.ensureTotalCapacity(gpa, t.cpu_count) catch {};
120120
121121 if (posix.Sigaction != void) {
122122 // This causes sending `posix.SIG.IO` to thread to interrupt blocking
......@@ -145,7 +145,7 @@ pub const init_single_threaded: Threaded = .{
145145 .threads = .empty,
146146 .stack_size = std.Thread.SpawnConfig.default_stack_size,
147147 .cpu_count = 1,
148 .concurrent_count = 0,
148 .concurrency_limit = 0,
149149 .old_sig_io = undefined,
150150 .old_sig_pipe = undefined,
151151 .have_signal_handler = false,
......@@ -184,18 +184,22 @@ fn worker(t: *Threaded) void {
184184 while (t.run_queue.popFirst()) |closure_node| {
185185 t.mutex.unlock();
186186 const closure: *Closure = @fieldParentPtr("node", closure_node);
187 const is_concurrent = closure.is_concurrent;
188187 closure.start(closure);
189188 t.mutex.lock();
190 if (is_concurrent) {
191 t.concurrent_count -= 1;
192 }
189 t.available_thread_count += 1;
193190 }
194191 if (t.join_requested) break;
195192 t.cond.wait(&t.mutex);
196193 }
197194}
198195
196fn oneShotWorker(t: *Threaded, closure: *Closure) void {
197 closure.start(closure);
198 t.mutex.lock();
199 defer t.mutex.unlock();
200 t.one_shot_thread_count -= 1;
201}
202
199203pub fn io(t: *Threaded) Io {
200204 return .{
201205 .userdata = t,
......@@ -432,7 +436,6 @@ const AsyncClosure = struct {
432436
433437 fn init(
434438 gpa: Allocator,
435 mode: enum { async, concurrent },
436439 result_len: usize,
437440 result_alignment: std.mem.Alignment,
438441 context: []const u8,
......@@ -454,10 +457,6 @@ const AsyncClosure = struct {
454457 .closure = .{
455458 .cancel_tid = .none,
456459 .start = start,
457 .is_concurrent = switch (mode) {
458 .async => false,
459 .concurrent => true,
460 },
461460 },
462461 .func = func,
463462 .context_alignment = context_alignment,
......@@ -490,55 +489,51 @@ fn async(
490489 context_alignment: std.mem.Alignment,
491490 start: *const fn (context: *const anyopaque, result: *anyopaque) void,
492491) ?*Io.AnyFuture {
493 if (builtin.single_threaded) {
492 const t: *Threaded = @ptrCast(@alignCast(userdata));
493 if (t.cpu_count == 1) {
494494 start(context.ptr, result.ptr);
495495 return null;
496496 }
497
498 const t: *Threaded = @ptrCast(@alignCast(userdata));
499 const cpu_count = t.cpu_count catch {
500 return concurrent(userdata, result.len, result_alignment, context, context_alignment, start) catch {
501 start(context.ptr, result.ptr);
502 return null;
503 };
504 };
505
506497 const gpa = t.allocator;
507 const ac = AsyncClosure.init(gpa, .async, result.len, result_alignment, context, context_alignment, start) catch {
498 const ac = AsyncClosure.init(gpa, result.len, result_alignment, context, context_alignment, start) catch {
508499 start(context.ptr, result.ptr);
509500 return null;
510501 };
511502
512503 t.mutex.lock();
513504
514 const thread_capacity = cpu_count - 1 + t.concurrent_count;
515
516 t.threads.ensureTotalCapacityPrecise(gpa, thread_capacity) catch {
517 t.mutex.unlock();
518 ac.deinit(gpa);
519 start(context.ptr, result.ptr);
520 return null;
521 };
505 if (t.available_thread_count == 0) {
506 if (t.cpu_count != 0 and t.threads.items.len >= t.cpu_count) {
507 t.mutex.unlock();
508 ac.deinit(gpa);
509 start(context.ptr, result.ptr);
510 return null;
511 }
522512
523 t.run_queue.prepend(&ac.closure.node);
513 t.threads.ensureUnusedCapacity(gpa, 1) catch {
514 t.mutex.unlock();
515 ac.deinit(gpa);
516 start(context.ptr, result.ptr);
517 return null;
518 };
524519
525 if (t.threads.items.len < thread_capacity) {
526 const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch {
527 if (t.threads.items.len == 0) {
528 assert(t.run_queue.popFirst() == &ac.closure.node);
529 t.mutex.unlock();
530 ac.deinit(gpa);
531 start(context.ptr, result.ptr);
532 return null;
533 }
534 // Rely on other workers to do it.
520 const thread = std.Thread.spawn(
521 .{ .stack_size = t.stack_size },
522 worker,
523 .{t},
524 ) catch {
535525 t.mutex.unlock();
536 t.cond.signal();
537 return @ptrCast(ac);
526 ac.deinit(gpa);
527 start(context.ptr, result.ptr);
528 return null;
538529 };
530
539531 t.threads.appendAssumeCapacity(thread);
532 } else {
533 t.available_thread_count -= 1;
540534 }
541535
536 t.run_queue.prepend(&ac.closure.node);
542537 t.mutex.unlock();
543538 t.cond.signal();
544539 return @ptrCast(ac);
......@@ -555,38 +550,49 @@ fn concurrent(
555550 if (builtin.single_threaded) return error.ConcurrencyUnavailable;
556551
557552 const t: *Threaded = @ptrCast(@alignCast(userdata));
558 const cpu_count = t.cpu_count catch 1;
559553
560554 const gpa = t.allocator;
561 const ac = AsyncClosure.init(gpa, .concurrent, result_len, result_alignment, context, context_alignment, start) catch {
555 const ac = AsyncClosure.init(gpa, result_len, result_alignment, context, context_alignment, start) catch {
562556 return error.ConcurrencyUnavailable;
563557 };
558 errdefer ac.deinit(gpa);
564559
565560 t.mutex.lock();
561 defer t.mutex.unlock();
566562
567 t.concurrent_count += 1;
568 const thread_capacity = cpu_count - 1 + t.concurrent_count;
569
570 t.threads.ensureTotalCapacity(gpa, thread_capacity) catch {
571 t.mutex.unlock();
572 ac.deinit(gpa);
573 return error.ConcurrencyUnavailable;
574 };
563 // If there's an avilable thread, use it.
564 if (t.available_thread_count > 0) {
565 t.available_thread_count -= 1;
566 t.run_queue.prepend(&ac.closure.node);
567 t.cond.signal();
568 return @ptrCast(ac);
569 }
575570
576 t.run_queue.prepend(&ac.closure.node);
571 // If we can spawn a normal worker, spawn it and use it.
572 if (t.cpu_count == 0 or t.threads.items.len < t.cpu_count) {
573 t.threads.ensureUnusedCapacity(gpa, 1) catch return error.ConcurrencyUnavailable;
577574
578 if (t.threads.items.len < thread_capacity) {
579 const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch {
580 assert(t.run_queue.popFirst() == &ac.closure.node);
581 t.mutex.unlock();
582 ac.deinit(gpa);
575 const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch
583576 return error.ConcurrencyUnavailable;
584 };
577
585578 t.threads.appendAssumeCapacity(thread);
579 t.run_queue.prepend(&ac.closure.node);
580 t.cond.signal();
581 return @ptrCast(ac);
586582 }
587583
588 t.mutex.unlock();
589 t.cond.signal();
584 // If we have a concurrencty limit and we havent' hit it yet,
585 // spawn a new one-shot thread.
586 if (t.concurrency_limit != 0 and t.one_shot_thread_count >= t.concurrency_limit)
587 return error.ConcurrencyUnavailable;
588
589 t.one_shot_thread_count += 1;
590 errdefer t.one_shot_thread_count -= 1;
591
592 const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, oneShotWorker, .{ t, &ac.closure }) catch
593 return error.ConcurrencyUnavailable;
594 thread.detach();
595
590596 return @ptrCast(ac);
591597}
592598
......@@ -652,7 +658,6 @@ const GroupClosure = struct {
652658 .closure = .{
653659 .cancel_tid = .none,
654660 .start = start,
655 .is_concurrent = false,
656661 },
657662 .t = t,
658663 .group = group,
......@@ -684,12 +689,9 @@ fn groupAsync(
684689 if (builtin.single_threaded) return start(group, context.ptr);
685690
686691 const t: *Threaded = @ptrCast(@alignCast(userdata));
687 const cpu_count = t.cpu_count catch 1;
688
689692 const gpa = t.allocator;
690 const gc = GroupClosure.init(gpa, t, group, context, context_alignment, start) catch {
693 const gc = GroupClosure.init(gpa, t, group, context, context_alignment, start) catch
691694 return start(group, context.ptr);
692 };
693695
694696 t.mutex.lock();
695697
......@@ -697,26 +699,36 @@ fn groupAsync(
697699 gc.node = .{ .next = @ptrCast(@alignCast(group.token)) };
698700 group.token = &gc.node;
699701
700 const thread_capacity = cpu_count - 1 + t.concurrent_count;
701
702 t.threads.ensureTotalCapacityPrecise(gpa, thread_capacity) catch {
703 t.mutex.unlock();
704 gc.deinit(gpa);
705 return start(group, context.ptr);
706 };
702 if (t.available_thread_count == 0) {
703 if (t.cpu_count != 0 and t.threads.items.len >= t.cpu_count) {
704 t.mutex.unlock();
705 gc.deinit(gpa);
706 return start(group, context.ptr);
707 }
707708
708 t.run_queue.prepend(&gc.closure.node);
709 t.threads.ensureUnusedCapacity(gpa, 1) catch {
710 t.mutex.unlock();
711 gc.deinit(gpa);
712 return start(group, context.ptr);
713 };
709714
710 if (t.threads.items.len < thread_capacity) {
711 const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch {
712 assert(t.run_queue.popFirst() == &gc.closure.node);
715 const thread = std.Thread.spawn(
716 .{ .stack_size = t.stack_size },
717 worker,
718 .{t},
719 ) catch {
713720 t.mutex.unlock();
714721 gc.deinit(gpa);
715722 return start(group, context.ptr);
716723 };
724
717725 t.threads.appendAssumeCapacity(thread);
726 } else {
727 t.available_thread_count -= 1;
718728 }
719729
730 t.run_queue.prepend(&gc.closure.node);
731
720732 // This needs to be done before unlocking the mutex to avoid a race with
721733 // the associated task finishing.
722734 const group_state: *std.atomic.Value(usize) = @ptrCast(&group.state);