authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-03-27 12:03:35-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-10-02 16:30:59-07:00
log19e7613a2d9c487c7f6cb9bfe0815c7d33ac68df
tree5edfba9745289d40ec4bf0bff41f290b815ca0ef
parenta9723598d7f5b21144c7ad21c450573524a63736

EventLoop: rewrite context switching

1 files changed, 72 insertions(+), 73 deletions(-)

lib/std/Io/EventLoop.zig+72-73
......@@ -1,4 +1,5 @@
11const std = @import("../std.zig");
2const builtin = @import("builtin");
23const assert = std.debug.assert;
34const Allocator = std.mem.Allocator;
45const Io = std.Io;
......@@ -16,7 +17,7 @@ const max_result_len = 64;
1617const min_stack_size = 4 * 1024 * 1024;
1718
1819const Fiber = struct {
19 regs: Regs,
20 context: Context,
2021 awaiter: ?*Fiber,
2122 queue_node: std.DoublyLinkedList(void).Node,
2223
......@@ -66,34 +67,28 @@ fn allocateFiber(el: *EventLoop, result_len: usize) error{OutOfMemory}!*Fiber {
6667}
6768
6869fn yield(el: *EventLoop, optional_fiber: ?*Fiber, register_awaiter: ?*?*Fiber) void {
70 const ready_fiber: *Fiber = optional_fiber orelse if (ready_node: {
71 el.mutex.lock();
72 defer el.mutex.unlock();
73 break :ready_node el.queue.pop();
74 }) |ready_node|
75 @fieldParentPtr("queue_node", ready_node)
76 else if (register_awaiter) |_| // time to switch to an idle fiber?
77 @panic("no other fiber to switch to in order to be able to register this fiber as an awaiter")
78 else // nothing to do
79 return;
6980 const message: SwitchMessage = .{
70 .ready_fiber = optional_fiber orelse if (ready_node: {
71 el.mutex.lock();
72 defer el.mutex.unlock();
73 break :ready_node el.queue.pop();
74 }) |ready_node|
75 @fieldParentPtr("queue_node", ready_node)
76 else if (register_awaiter) |_|
77 @panic("no other fiber to switch to in order to be able to register this fiber as an awaiter") // time to switch to an idle fiber?
78 else
79 return, // nothing to do
81 .prev_context = &current_fiber.context,
82 .ready_context = &ready_fiber.context,
8083 .register_awaiter = register_awaiter,
8184 };
82 std.log.debug("switching from {*} to {*}", .{ current_fiber, message.ready_fiber });
83 SwitchMessage.handle(@ptrFromInt(contextSwitch(&current_fiber.regs, &message.ready_fiber.regs, @intFromPtr(&message))), el);
85 std.log.debug("switching from {*} to {*}", .{
86 @as(*Fiber, @fieldParentPtr("context", message.prev_context)),
87 @as(*Fiber, @fieldParentPtr("context", message.ready_context)),
88 });
89 contextSwitch(&message).handle(el);
8490}
8591
86const SwitchMessage = struct {
87 ready_fiber: *Fiber,
88 register_awaiter: ?*?*Fiber,
89
90 fn handle(message: *const SwitchMessage, el: *EventLoop) void {
91 const prev_fiber = current_fiber;
92 current_fiber = message.ready_fiber;
93 if (message.register_awaiter) |awaiter| if (@atomicRmw(?*Fiber, awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished) el.schedule(prev_fiber);
94 }
95};
96
9792fn schedule(el: *EventLoop, fiber: *Fiber) void {
9893 el.mutex.lock();
9994 defer el.mutex.unlock();
......@@ -109,47 +104,62 @@ fn recycle(el: *EventLoop, fiber: *Fiber) void {
109104 el.free.append(&fiber.queue_node);
110105}
111106
112const Regs = extern struct {
107const SwitchMessage = extern struct {
108 prev_context: *Context,
109 ready_context: *Context,
110 register_awaiter: ?*?*Fiber,
111
112 fn handle(message: *const SwitchMessage, el: *EventLoop) void {
113 const prev_fiber: *Fiber = @fieldParentPtr("context", message.prev_context);
114 current_fiber = @fieldParentPtr("context", message.ready_context);
115 if (message.register_awaiter) |awaiter| if (@atomicRmw(?*Fiber, awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished) el.schedule(prev_fiber);
116 }
117};
118
119const Context = extern struct {
113120 rsp: usize,
114 r15: usize,
115 r14: usize,
116 r13: usize,
117 r12: usize,
118 rbx: usize,
119121 rbp: usize,
122 rip: usize,
120123};
121124
122const contextSwitch: *const fn (old: *Regs, new: *Regs, message: usize) callconv(.c) usize = @ptrCast(&contextSwitch_naked);
123
124noinline fn contextSwitch_naked() callconv(.naked) void {
125 asm volatile (
126 \\movq %%rsp, 0x00(%%rdi)
127 \\movq %%r15, 0x08(%%rdi)
128 \\movq %%r14, 0x10(%%rdi)
129 \\movq %%r13, 0x18(%%rdi)
130 \\movq %%r12, 0x20(%%rdi)
131 \\movq %%rbx, 0x28(%%rdi)
132 \\movq %%rbp, 0x30(%%rdi)
133 \\
134 \\movq 0x00(%%rsi), %%rsp
135 \\movq 0x08(%%rsi), %%r15
136 \\movq 0x10(%%rsi), %%r14
137 \\movq 0x18(%%rsi), %%r13
138 \\movq 0x20(%%rsi), %%r12
139 \\movq 0x28(%%rsi), %%rbx
140 \\movq 0x30(%%rsi), %%rbp
141 \\
142 \\movq %%rdx, %%rax
143 \\ret
144 );
125inline fn contextSwitch(message: *const SwitchMessage) *const SwitchMessage {
126 return switch (builtin.cpu.arch) {
127 .x86_64 => asm volatile (
128 \\ movq 0(%%rsi), %%rax
129 \\ movq 8(%%rsi), %%rcx
130 \\ leaq 0f(%%rip), %%rdx
131 \\ movq %%rsp, 0(%%rax)
132 \\ movq %%rbp, 8(%%rax)
133 \\ movq %%rdx, 16(%%rax)
134 \\ movq 0(%%rcx), %%rsp
135 \\ movq 8(%%rcx), %%rbp
136 \\ jmpq *16(%%rcx)
137 \\0:
138 : [received_message] "={rsi}" (-> *const SwitchMessage),
139 : [message_to_send] "{rsi}" (message),
140 : "rax", "rcx", "rdx", "rbx", "rdi", //
141 "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", //
142 "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7", //
143 "zmm0", "zmm1", "zmm2", "zmm3", "zmm4", "zmm5", "zmm6", "zmm7", //
144 "zmm8", "zmm9", "zmm10", "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", //
145 "zmm16", "zmm17", "zmm18", "zmm19", "zmm20", "zmm21", "zmm22", "zmm23", //
146 "zmm24", "zmm25", "zmm26", "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", //
147 "fpsr", "fpcr", "mxcsr", "rflags", "dirflag", "memory"
148 ),
149 else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)),
150 };
145151}
146152
147fn popRet() callconv(.naked) void {
148 asm volatile (
149 \\pop %%rdi
150 \\movq %%rax, %%rsi
151 \\ret
152 );
153fn fiberEntry() callconv(.naked) void {
154 switch (builtin.cpu.arch) {
155 .x86_64 => asm volatile (
156 \\ leaq 8(%%rsp), %%rdi
157 \\ jmp %[AsyncClosure_call:P]
158 :
159 : [AsyncClosure_call] "X" (&AsyncClosure.call),
160 ),
161 else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)),
162 }
153163}
154164
155165pub fn @"async"(
......@@ -179,21 +189,10 @@ pub fn @"async"(
179189 .start = start,
180190 };
181191 const stack_end: [*]align(16) usize = @alignCast(@ptrCast(closure));
182 const stack_top = (stack_end - 4)[0..4];
183 stack_top.* = .{
184 @intFromPtr(&popRet),
185 @intFromPtr(closure),
186 @intFromPtr(&AsyncClosure.call),
187 0,
188 };
189 fiber.regs = .{
190 .rsp = @intFromPtr(stack_top),
191 .r15 = 0,
192 .r14 = 0,
193 .r13 = 0,
194 .r12 = 0,
195 .rbx = 0,
192 fiber.context = .{
193 .rsp = @intFromPtr(stack_end - 1),
196194 .rbp = 0,
195 .rip = @intFromPtr(&fiberEntry),
197196 };
198197
199198 event_loop.schedule(fiber);