| ... | ... | @@ -24,8 +24,10 @@ run_queue: std.SinglyLinkedList = .{}, |
| 24 | 24 | join_requested: bool = false, |
| 25 | 25 | threads: std.ArrayList(std.Thread), |
| 26 | 26 | stack_size: usize, |
| 27 | | cpu_count: std.Thread.CpuCountError!usize, |
| 28 | | concurrent_count: usize, |
| 27 | cpu_count: usize, // 0 means no limit |
| 28 | concurrency_limit: usize, // 0 means no limit |
| 29 | available_thread_count: usize = 0, |
| 30 | one_shot_thread_count: usize = 0, |
| 29 | 31 | |
| 30 | 32 | wsa: if (is_windows) Wsa else struct {} = .{}, |
| 31 | 33 | |
| ... | ... | @@ -70,8 +72,6 @@ const Closure = struct { |
| 70 | 72 | start: Start, |
| 71 | 73 | node: std.SinglyLinkedList.Node = .{}, |
| 72 | 74 | cancel_tid: CancelId, |
| 73 | | /// Whether this task bumps minimum number of threads in the pool. |
| 74 | | is_concurrent: bool, |
| 75 | 75 | |
| 76 | 76 | const Start = *const fn (*Closure) void; |
| 77 | 77 | |
| ... | ... | @@ -103,20 +103,20 @@ pub fn init( |
| 103 | 103 | /// here. |
| 104 | 104 | gpa: Allocator, |
| 105 | 105 | ) Threaded { |
| 106 | assert(!builtin.single_threaded); // use 'init_single_threaded' instead |
| 107 | |
| 106 | 108 | var t: Threaded = .{ |
| 107 | 109 | .allocator = gpa, |
| 108 | 110 | .threads = .empty, |
| 109 | 111 | .stack_size = std.Thread.SpawnConfig.default_stack_size, |
| 110 | | .cpu_count = std.Thread.getCpuCount(), |
| 111 | | .concurrent_count = 0, |
| 112 | .cpu_count = std.Thread.getCpuCount() catch 0, |
| 113 | .concurrency_limit = 0, |
| 112 | 114 | .old_sig_io = undefined, |
| 113 | 115 | .old_sig_pipe = undefined, |
| 114 | 116 | .have_signal_handler = false, |
| 115 | 117 | }; |
| 116 | 118 | |
| 117 | | if (t.cpu_count) |n| { |
| 118 | | t.threads.ensureTotalCapacityPrecise(gpa, n - 1) catch {}; |
| 119 | | } else |_| {} |
| 119 | t.threads.ensureTotalCapacity(gpa, t.cpu_count) catch {}; |
| 120 | 120 | |
| 121 | 121 | if (posix.Sigaction != void) { |
| 122 | 122 | // This causes sending `posix.SIG.IO` to thread to interrupt blocking |
| ... | ... | @@ -145,7 +145,7 @@ pub const init_single_threaded: Threaded = .{ |
| 145 | 145 | .threads = .empty, |
| 146 | 146 | .stack_size = std.Thread.SpawnConfig.default_stack_size, |
| 147 | 147 | .cpu_count = 1, |
| 148 | | .concurrent_count = 0, |
| 148 | .concurrency_limit = 0, |
| 149 | 149 | .old_sig_io = undefined, |
| 150 | 150 | .old_sig_pipe = undefined, |
| 151 | 151 | .have_signal_handler = false, |
| ... | ... | @@ -184,18 +184,22 @@ fn worker(t: *Threaded) void { |
| 184 | 184 | while (t.run_queue.popFirst()) |closure_node| { |
| 185 | 185 | t.mutex.unlock(); |
| 186 | 186 | const closure: *Closure = @fieldParentPtr("node", closure_node); |
| 187 | | const is_concurrent = closure.is_concurrent; |
| 188 | 187 | closure.start(closure); |
| 189 | 188 | t.mutex.lock(); |
| 190 | | if (is_concurrent) { |
| 191 | | t.concurrent_count -= 1; |
| 192 | | } |
| 189 | t.available_thread_count += 1; |
| 193 | 190 | } |
| 194 | 191 | if (t.join_requested) break; |
| 195 | 192 | t.cond.wait(&t.mutex); |
| 196 | 193 | } |
| 197 | 194 | } |
| 198 | 195 | |
| 196 | fn oneShotWorker(t: *Threaded, closure: *Closure) void { |
| 197 | closure.start(closure); |
| 198 | t.mutex.lock(); |
| 199 | defer t.mutex.unlock(); |
| 200 | t.one_shot_thread_count -= 1; |
| 201 | } |
| 202 | |
| 199 | 203 | pub fn io(t: *Threaded) Io { |
| 200 | 204 | return .{ |
| 201 | 205 | .userdata = t, |
| ... | ... | @@ -432,7 +436,6 @@ const AsyncClosure = struct { |
| 432 | 436 | |
| 433 | 437 | fn init( |
| 434 | 438 | gpa: Allocator, |
| 435 | | mode: enum { async, concurrent }, |
| 436 | 439 | result_len: usize, |
| 437 | 440 | result_alignment: std.mem.Alignment, |
| 438 | 441 | context: []const u8, |
| ... | ... | @@ -454,10 +457,6 @@ const AsyncClosure = struct { |
| 454 | 457 | .closure = .{ |
| 455 | 458 | .cancel_tid = .none, |
| 456 | 459 | .start = start, |
| 457 | | .is_concurrent = switch (mode) { |
| 458 | | .async => false, |
| 459 | | .concurrent => true, |
| 460 | | }, |
| 461 | 460 | }, |
| 462 | 461 | .func = func, |
| 463 | 462 | .context_alignment = context_alignment, |
| ... | ... | @@ -490,55 +489,51 @@ fn async( |
| 490 | 489 | context_alignment: std.mem.Alignment, |
| 491 | 490 | start: *const fn (context: *const anyopaque, result: *anyopaque) void, |
| 492 | 491 | ) ?*Io.AnyFuture { |
| 493 | | if (builtin.single_threaded) { |
| 492 | const t: *Threaded = @ptrCast(@alignCast(userdata)); |
| 493 | if (t.cpu_count == 1) { |
| 494 | 494 | start(context.ptr, result.ptr); |
| 495 | 495 | return null; |
| 496 | 496 | } |
| 497 | | |
| 498 | | const t: *Threaded = @ptrCast(@alignCast(userdata)); |
| 499 | | const cpu_count = t.cpu_count catch { |
| 500 | | return concurrent(userdata, result.len, result_alignment, context, context_alignment, start) catch { |
| 501 | | start(context.ptr, result.ptr); |
| 502 | | return null; |
| 503 | | }; |
| 504 | | }; |
| 505 | | |
| 506 | 497 | const gpa = t.allocator; |
| 507 | | const ac = AsyncClosure.init(gpa, .async, result.len, result_alignment, context, context_alignment, start) catch { |
| 498 | const ac = AsyncClosure.init(gpa, result.len, result_alignment, context, context_alignment, start) catch { |
| 508 | 499 | start(context.ptr, result.ptr); |
| 509 | 500 | return null; |
| 510 | 501 | }; |
| 511 | 502 | |
| 512 | 503 | t.mutex.lock(); |
| 513 | 504 | |
| 514 | | const thread_capacity = cpu_count - 1 + t.concurrent_count; |
| 515 | | |
| 516 | | t.threads.ensureTotalCapacityPrecise(gpa, thread_capacity) catch { |
| 517 | | t.mutex.unlock(); |
| 518 | | ac.deinit(gpa); |
| 519 | | start(context.ptr, result.ptr); |
| 520 | | return null; |
| 521 | | }; |
| 505 | if (t.available_thread_count == 0) { |
| 506 | if (t.cpu_count != 0 and t.threads.items.len >= t.cpu_count) { |
| 507 | t.mutex.unlock(); |
| 508 | ac.deinit(gpa); |
| 509 | start(context.ptr, result.ptr); |
| 510 | return null; |
| 511 | } |
| 522 | 512 | |
| 523 | | t.run_queue.prepend(&ac.closure.node); |
| 513 | t.threads.ensureUnusedCapacity(gpa, 1) catch { |
| 514 | t.mutex.unlock(); |
| 515 | ac.deinit(gpa); |
| 516 | start(context.ptr, result.ptr); |
| 517 | return null; |
| 518 | }; |
| 524 | 519 | |
| 525 | | if (t.threads.items.len < thread_capacity) { |
| 526 | | const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch { |
| 527 | | if (t.threads.items.len == 0) { |
| 528 | | assert(t.run_queue.popFirst() == &ac.closure.node); |
| 529 | | t.mutex.unlock(); |
| 530 | | ac.deinit(gpa); |
| 531 | | start(context.ptr, result.ptr); |
| 532 | | return null; |
| 533 | | } |
| 534 | | // Rely on other workers to do it. |
| 520 | const thread = std.Thread.spawn( |
| 521 | .{ .stack_size = t.stack_size }, |
| 522 | worker, |
| 523 | .{t}, |
| 524 | ) catch { |
| 535 | 525 | t.mutex.unlock(); |
| 536 | | t.cond.signal(); |
| 537 | | return @ptrCast(ac); |
| 526 | ac.deinit(gpa); |
| 527 | start(context.ptr, result.ptr); |
| 528 | return null; |
| 538 | 529 | }; |
| 530 | |
| 539 | 531 | t.threads.appendAssumeCapacity(thread); |
| 532 | } else { |
| 533 | t.available_thread_count -= 1; |
| 540 | 534 | } |
| 541 | 535 | |
| 536 | t.run_queue.prepend(&ac.closure.node); |
| 542 | 537 | t.mutex.unlock(); |
| 543 | 538 | t.cond.signal(); |
| 544 | 539 | return @ptrCast(ac); |
| ... | ... | @@ -555,38 +550,49 @@ fn concurrent( |
| 555 | 550 | if (builtin.single_threaded) return error.ConcurrencyUnavailable; |
| 556 | 551 | |
| 557 | 552 | const t: *Threaded = @ptrCast(@alignCast(userdata)); |
| 558 | | const cpu_count = t.cpu_count catch 1; |
| 559 | 553 | |
| 560 | 554 | const gpa = t.allocator; |
| 561 | | const ac = AsyncClosure.init(gpa, .concurrent, result_len, result_alignment, context, context_alignment, start) catch { |
| 555 | const ac = AsyncClosure.init(gpa, result_len, result_alignment, context, context_alignment, start) catch { |
| 562 | 556 | return error.ConcurrencyUnavailable; |
| 563 | 557 | }; |
| 558 | errdefer ac.deinit(gpa); |
| 564 | 559 | |
| 565 | 560 | t.mutex.lock(); |
| 561 | defer t.mutex.unlock(); |
| 566 | 562 | |
| 567 | | t.concurrent_count += 1; |
| 568 | | const thread_capacity = cpu_count - 1 + t.concurrent_count; |
| 569 | | |
| 570 | | t.threads.ensureTotalCapacity(gpa, thread_capacity) catch { |
| 571 | | t.mutex.unlock(); |
| 572 | | ac.deinit(gpa); |
| 573 | | return error.ConcurrencyUnavailable; |
| 574 | | }; |
| 563 | // If there's an avilable thread, use it. |
| 564 | if (t.available_thread_count > 0) { |
| 565 | t.available_thread_count -= 1; |
| 566 | t.run_queue.prepend(&ac.closure.node); |
| 567 | t.cond.signal(); |
| 568 | return @ptrCast(ac); |
| 569 | } |
| 575 | 570 | |
| 576 | | t.run_queue.prepend(&ac.closure.node); |
| 571 | // If we can spawn a normal worker, spawn it and use it. |
| 572 | if (t.cpu_count == 0 or t.threads.items.len < t.cpu_count) { |
| 573 | t.threads.ensureUnusedCapacity(gpa, 1) catch return error.ConcurrencyUnavailable; |
| 577 | 574 | |
| 578 | | if (t.threads.items.len < thread_capacity) { |
| 579 | | const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch { |
| 580 | | assert(t.run_queue.popFirst() == &ac.closure.node); |
| 581 | | t.mutex.unlock(); |
| 582 | | ac.deinit(gpa); |
| 575 | const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch |
| 583 | 576 | return error.ConcurrencyUnavailable; |
| 584 | | }; |
| 577 | |
| 585 | 578 | t.threads.appendAssumeCapacity(thread); |
| 579 | t.run_queue.prepend(&ac.closure.node); |
| 580 | t.cond.signal(); |
| 581 | return @ptrCast(ac); |
| 586 | 582 | } |
| 587 | 583 | |
| 588 | | t.mutex.unlock(); |
| 589 | | t.cond.signal(); |
| 584 | // If we have a concurrencty limit and we havent' hit it yet, |
| 585 | // spawn a new one-shot thread. |
| 586 | if (t.concurrency_limit != 0 and t.one_shot_thread_count >= t.concurrency_limit) |
| 587 | return error.ConcurrencyUnavailable; |
| 588 | |
| 589 | t.one_shot_thread_count += 1; |
| 590 | errdefer t.one_shot_thread_count -= 1; |
| 591 | |
| 592 | const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, oneShotWorker, .{ t, &ac.closure }) catch |
| 593 | return error.ConcurrencyUnavailable; |
| 594 | thread.detach(); |
| 595 | |
| 590 | 596 | return @ptrCast(ac); |
| 591 | 597 | } |
| 592 | 598 | |
| ... | ... | @@ -652,7 +658,6 @@ const GroupClosure = struct { |
| 652 | 658 | .closure = .{ |
| 653 | 659 | .cancel_tid = .none, |
| 654 | 660 | .start = start, |
| 655 | | .is_concurrent = false, |
| 656 | 661 | }, |
| 657 | 662 | .t = t, |
| 658 | 663 | .group = group, |
| ... | ... | @@ -684,12 +689,9 @@ fn groupAsync( |
| 684 | 689 | if (builtin.single_threaded) return start(group, context.ptr); |
| 685 | 690 | |
| 686 | 691 | const t: *Threaded = @ptrCast(@alignCast(userdata)); |
| 687 | | const cpu_count = t.cpu_count catch 1; |
| 688 | | |
| 689 | 692 | const gpa = t.allocator; |
| 690 | | const gc = GroupClosure.init(gpa, t, group, context, context_alignment, start) catch { |
| 693 | const gc = GroupClosure.init(gpa, t, group, context, context_alignment, start) catch |
| 691 | 694 | return start(group, context.ptr); |
| 692 | | }; |
| 693 | 695 | |
| 694 | 696 | t.mutex.lock(); |
| 695 | 697 | |
| ... | ... | @@ -697,26 +699,36 @@ fn groupAsync( |
| 697 | 699 | gc.node = .{ .next = @ptrCast(@alignCast(group.token)) }; |
| 698 | 700 | group.token = &gc.node; |
| 699 | 701 | |
| 700 | | const thread_capacity = cpu_count - 1 + t.concurrent_count; |
| 701 | | |
| 702 | | t.threads.ensureTotalCapacityPrecise(gpa, thread_capacity) catch { |
| 703 | | t.mutex.unlock(); |
| 704 | | gc.deinit(gpa); |
| 705 | | return start(group, context.ptr); |
| 706 | | }; |
| 702 | if (t.available_thread_count == 0) { |
| 703 | if (t.cpu_count != 0 and t.threads.items.len >= t.cpu_count) { |
| 704 | t.mutex.unlock(); |
| 705 | gc.deinit(gpa); |
| 706 | return start(group, context.ptr); |
| 707 | } |
| 707 | 708 | |
| 708 | | t.run_queue.prepend(&gc.closure.node); |
| 709 | t.threads.ensureUnusedCapacity(gpa, 1) catch { |
| 710 | t.mutex.unlock(); |
| 711 | gc.deinit(gpa); |
| 712 | return start(group, context.ptr); |
| 713 | }; |
| 709 | 714 | |
| 710 | | if (t.threads.items.len < thread_capacity) { |
| 711 | | const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch { |
| 712 | | assert(t.run_queue.popFirst() == &gc.closure.node); |
| 715 | const thread = std.Thread.spawn( |
| 716 | .{ .stack_size = t.stack_size }, |
| 717 | worker, |
| 718 | .{t}, |
| 719 | ) catch { |
| 713 | 720 | t.mutex.unlock(); |
| 714 | 721 | gc.deinit(gpa); |
| 715 | 722 | return start(group, context.ptr); |
| 716 | 723 | }; |
| 724 | |
| 717 | 725 | t.threads.appendAssumeCapacity(thread); |
| 726 | } else { |
| 727 | t.available_thread_count -= 1; |
| 718 | 728 | } |
| 719 | 729 | |
| 730 | t.run_queue.prepend(&gc.closure.node); |
| 731 | |
| 720 | 732 | // This needs to be done before unlocking the mutex to avoid a race with |
| 721 | 733 | // the associated task finishing. |
| 722 | 734 | const group_state: *std.atomic.Value(usize) = @ptrCast(&group.state); |