| ... | @@ -380,8 +380,7 @@ fn schedule(el: *EventLoop, thread: *Thread, ready_queue: Fiber.Queue) void { | ... | @@ -380,8 +380,7 @@ fn schedule(el: *EventLoop, thread: *Thread, ready_queue: Fiber.Queue) void { |
| 380 | | 380 | |
| 381 | fn mainIdle(el: *EventLoop, message: *const SwitchMessage) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Context)))) noreturn { | 381 | fn mainIdle(el: *EventLoop, message: *const SwitchMessage) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Context)))) noreturn { |
| 382 | message.handle(el); | 382 | message.handle(el); |
| 383 | const thread: *Thread = &el.threads.allocated[0]; | 383 | el.idle(&el.threads.allocated[0]); |
| 384 | el.idle(thread); | | |
| 385 | el.yield(@ptrCast(&el.main_fiber_buffer), .nothing); | 384 | el.yield(@ptrCast(&el.main_fiber_buffer), .nothing); |
| 386 | unreachable; // switched to dead fiber | 385 | unreachable; // switched to dead fiber |
| 387 | } | 386 | } |
| ... | @@ -480,10 +479,14 @@ const SwitchMessage = struct { | ... | @@ -480,10 +479,14 @@ const SwitchMessage = struct { |
| 480 | reschedule, | 479 | reschedule, |
| 481 | recycle: *Fiber, | 480 | recycle: *Fiber, |
| 482 | register_awaiter: *?*Fiber, | 481 | register_awaiter: *?*Fiber, |
| 483 | lock_mutex: struct { | 482 | mutex_lock: struct { |
| 484 | prev_state: Io.Mutex.State, | 483 | prev_state: Io.Mutex.State, |
| 485 | mutex: *Io.Mutex, | 484 | mutex: *Io.Mutex, |
| 486 | }, | 485 | }, |
| | 486 | condition_wait: struct { |
| | 487 | cond: *Io.Condition, |
| | 488 | mutex: *Io.Mutex, |
| | 489 | }, |
| 487 | exit, | 490 | exit, |
| 488 | }; | 491 | }; |
| 489 | | 492 | |
| ... | @@ -492,7 +495,7 @@ const SwitchMessage = struct { | ... | @@ -492,7 +495,7 @@ const SwitchMessage = struct { |
| 492 | thread.current_context = message.contexts.ready; | 495 | thread.current_context = message.contexts.ready; |
| 493 | switch (message.pending_task) { | 496 | switch (message.pending_task) { |
| 494 | .nothing => {}, | 497 | .nothing => {}, |
| 495 | .reschedule => { | 498 | .reschedule => if (message.contexts.prev != &thread.idle_context) { |
| 496 | const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev)); | 499 | const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev)); |
| 497 | assert(prev_fiber.queue_next == null); | 500 | assert(prev_fiber.queue_next == null); |
| 498 | el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber }); | 501 | el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber }); |
| ... | @@ -511,16 +514,16 @@ const SwitchMessage = struct { | ... | @@ -511,16 +514,16 @@ const SwitchMessage = struct { |
| 511 | .acq_rel, | 514 | .acq_rel, |
| 512 | ) == Fiber.finished) el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber }); | 515 | ) == Fiber.finished) el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber }); |
| 513 | }, | 516 | }, |
| 514 | .lock_mutex => |lock_mutex| { | 517 | .mutex_lock => |mutex_lock| { |
| 515 | const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev)); | 518 | const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev)); |
| 516 | assert(prev_fiber.queue_next == null); | 519 | assert(prev_fiber.queue_next == null); |
| 517 | var prev_state = lock_mutex.prev_state; | 520 | var prev_state = mutex_lock.prev_state; |
| 518 | while (switch (prev_state) { | 521 | while (switch (prev_state) { |
| 519 | else => next_state: { | 522 | else => next_state: { |
| 520 | prev_fiber.queue_next = @ptrFromInt(@intFromEnum(prev_state)); | 523 | prev_fiber.queue_next = @ptrFromInt(@intFromEnum(prev_state)); |
| 521 | break :next_state @cmpxchgWeak( | 524 | break :next_state @cmpxchgWeak( |
| 522 | Io.Mutex.State, | 525 | Io.Mutex.State, |
| 523 | &lock_mutex.mutex.state, | 526 | &mutex_lock.mutex.state, |
| 524 | prev_state, | 527 | prev_state, |
| 525 | @enumFromInt(@intFromPtr(prev_fiber)), | 528 | @enumFromInt(@intFromPtr(prev_fiber)), |
| 526 | .release, | 529 | .release, |
| ... | @@ -529,7 +532,7 @@ const SwitchMessage = struct { | ... | @@ -529,7 +532,7 @@ const SwitchMessage = struct { |
| 529 | }, | 532 | }, |
| 530 | .unlocked => @cmpxchgWeak( | 533 | .unlocked => @cmpxchgWeak( |
| 531 | Io.Mutex.State, | 534 | Io.Mutex.State, |
| 532 | &lock_mutex.mutex.state, | 535 | &mutex_lock.mutex.state, |
| 533 | .unlocked, | 536 | .unlocked, |
| 534 | .locked_once, | 537 | .locked_once, |
| 535 | .acquire, | 538 | .acquire, |
| ... | @@ -541,6 +544,13 @@ const SwitchMessage = struct { | ... | @@ -541,6 +544,13 @@ const SwitchMessage = struct { |
| 541 | }, | 544 | }, |
| 542 | }) |next_state| prev_state = next_state; | 545 | }) |next_state| prev_state = next_state; |
| 543 | }, | 546 | }, |
| | 547 | .condition_wait => |condition_wait| { |
| | 548 | const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev)); |
| | 549 | assert(prev_fiber.queue_next == null); |
| | 550 | const cond_state: *?*Fiber = @ptrCast(&condition_wait.cond.state); |
| | 551 | assert(@atomicRmw(?*Fiber, cond_state, .Xchg, prev_fiber, .release) == null); // More than one wait on same Condition is illegal. |
| | 552 | condition_wait.mutex.unlock(el.io()); |
| | 553 | }, |
| 544 | .exit => for (el.threads.allocated[0..@atomicLoad(u32, &el.threads.active, .acquire)]) |*each_thread| { | 554 | .exit => for (el.threads.allocated[0..@atomicLoad(u32, &el.threads.active, .acquire)]) |*each_thread| { |
| 545 | getSqe(&thread.io_uring).* = .{ | 555 | getSqe(&thread.io_uring).* = .{ |
| 546 | .opcode = .MSG_RING, | 556 | .opcode = .MSG_RING, |
| ... | @@ -1242,7 +1252,7 @@ fn sleep(userdata: ?*anyopaque, clockid: std.posix.clockid_t, deadline: Io.Deadl | ... | @@ -1242,7 +1252,7 @@ fn sleep(userdata: ?*anyopaque, clockid: std.posix.clockid_t, deadline: Io.Deadl |
| 1242 | | 1252 | |
| 1243 | fn mutexLock(userdata: ?*anyopaque, prev_state: Io.Mutex.State, mutex: *Io.Mutex) error{Canceled}!void { | 1253 | fn mutexLock(userdata: ?*anyopaque, prev_state: Io.Mutex.State, mutex: *Io.Mutex) error{Canceled}!void { |
| 1244 | const el: *EventLoop = @alignCast(@ptrCast(userdata)); | 1254 | const el: *EventLoop = @alignCast(@ptrCast(userdata)); |
| 1245 | el.yield(null, .{ .lock_mutex = .{ | 1255 | el.yield(null, .{ .mutex_lock = .{ |
| 1246 | .prev_state = prev_state, | 1256 | .prev_state = prev_state, |
| 1247 | .mutex = mutex, | 1257 | .mutex = mutex, |
| 1248 | } }); | 1258 | } }); |
| ... | @@ -1271,13 +1281,10 @@ fn mutexUnlock(userdata: ?*anyopaque, prev_state: Io.Mutex.State, mutex: *Io.Mut | ... | @@ -1271,13 +1281,10 @@ fn mutexUnlock(userdata: ?*anyopaque, prev_state: Io.Mutex.State, mutex: *Io.Mut |
| 1271 | | 1281 | |
| 1272 | fn conditionWait(userdata: ?*anyopaque, cond: *Io.Condition, mutex: *Io.Mutex) Io.Cancelable!void { | 1282 | fn conditionWait(userdata: ?*anyopaque, cond: *Io.Condition, mutex: *Io.Mutex) Io.Cancelable!void { |
| 1273 | const el: *EventLoop = @alignCast(@ptrCast(userdata)); | 1283 | const el: *EventLoop = @alignCast(@ptrCast(userdata)); |
| 1274 | const cond_state: *?*Fiber = @ptrCast(&cond.state); | 1284 | el.yield(null, .{ .condition_wait = .{ |
| 1275 | const thread: *Thread = .current(); | 1285 | .cond = cond, |
| 1276 | const fiber = thread.currentFiber(); | 1286 | .mutex = mutex, |
| 1277 | const prev = @atomicRmw(?*Fiber, cond_state, .Xchg, fiber, .acquire); | 1287 | } }); |
| 1278 | assert(prev == null); // More than one wait on same Condition is illegal. | | |
| 1279 | mutex.unlock(el.io()); | | |
| 1280 | el.yield(null, .nothing); | | |
| 1281 | try mutex.lock(el.io()); | 1288 | try mutex.lock(el.io()); |
| 1282 | } | 1289 | } |
| 1283 | | 1290 | |