authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-03-28 10:05:52-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-10-02 16:30:59-07:00
log2f2019645c218f7d2b1f7da2d8168859110ce1cd
tree2c4ee1875f717e1755af04f9883b3fcdc8492a1c
parentdfbf68e5fa711655462622bd4b75a8f01d622032

EventLoop: move context after the async closure

This avoids needing to store more sizes and alignments. Only the result alignment needs to be stored, because `Fiber` is at a fixed zero offset.

1 files changed, 86 insertions(+), 92 deletions(-)

lib/std/Io/EventLoop.zig+86-92
......@@ -4,28 +4,23 @@ const assert = std.debug.assert;
44const Allocator = std.mem.Allocator;
55const Io = std.Io;
66const EventLoop = @This();
7const Alignment = std.mem.Alignment;
78
89gpa: Allocator,
910mutex: std.Thread.Mutex,
1011cond: std.Thread.Condition,
1112queue: std.DoublyLinkedList(void),
1213free: std.DoublyLinkedList(void),
13main_fiber_buffer: [@sizeOf(Fiber) + max_result_len]u8 align(@alignOf(Fiber)),
14main_context: Context,
1415exit_awaiter: ?*Fiber,
1516idle_count: usize,
1617threads: std.ArrayListUnmanaged(Thread),
1718
1819threadlocal var current_idle_context: *Context = undefined;
19threadlocal var current_fiber_context: *Context = undefined;
20threadlocal var current_context: *Context = undefined;
2021
21/// Also used for context.
22const max_result_len = 64;
23/// Also used for context.
24const max_result_align: std.mem.Alignment = .@"16";
25
26const min_stack_size = 4 * 1024 * 1024;
22/// Empirically saw 10KB being used by the self-hosted backend for logging.
2723const idle_stack_size = 32 * 1024;
28const stack_align = 16;
2924
3025const Thread = struct {
3126 thread: std.Thread,
......@@ -33,45 +28,53 @@ const Thread = struct {
3328};
3429
3530const Fiber = struct {
36 _: void align(max_result_align.toByteUnits()) = {},
37
3831 context: Context,
3932 awaiter: ?*Fiber,
4033 queue_node: std.DoublyLinkedList(void).Node,
34 result_align: Alignment,
4135
4236 const finished: ?*Fiber = @ptrFromInt(std.mem.alignBackward(usize, std.math.maxInt(usize), @alignOf(Fiber)));
4337
44 fn allocatedSlice(f: *Fiber) []align(@alignOf(Fiber)) u8 {
45 const base: [*]align(@alignOf(Fiber)) u8 = @ptrCast(f);
46 return base[0..std.mem.alignForward(
38 const max_result_align: Alignment = .@"16";
39 const max_result_size = max_result_align.forward(64);
40 /// This includes any stack realignments that need to happen, and also the
41 /// initial frame return address slot and argument frame, depending on target.
42 const min_stack_size = 4 * 1024 * 1024;
43 const max_context_align: Alignment = .@"16";
44 const max_context_size = max_context_align.forward(1024);
45 const allocation_size = std.mem.alignForward(
46 usize,
47 std.mem.alignForward(
4748 usize,
48 resultOffset() + max_result_len + min_stack_size,
49 std.heap.page_size_max,
50 )];
51 }
52
53 fn argsOffset() usize {
54 return max_result_align.forward(@sizeOf(Fiber));
55 }
56
57 fn resultOffset() usize {
58 return max_result_align.forward(argsOffset() + max_result_len);
59 }
60
61 fn argsSlice(f: *Fiber) []u8 {
62 const base: [*]align(@alignOf(Fiber)) u8 = @ptrCast(f);
63 return base[argsOffset()..][0..max_result_len];
49 max_result_align.forward(@sizeOf(Fiber)) + max_result_size + min_stack_size,
50 @max(@alignOf(AsyncClosure), max_context_align.toByteUnits()),
51 ) + @sizeOf(AsyncClosure) + max_context_size,
52 std.heap.page_size_max,
53 );
54
55 fn allocate(el: *EventLoop) error{OutOfMemory}!*Fiber {
56 return if (free_node: {
57 el.mutex.lock();
58 defer el.mutex.unlock();
59 break :free_node el.free.pop();
60 }) |free_node|
61 @alignCast(@fieldParentPtr("queue_node", free_node))
62 else
63 @ptrCast(try el.gpa.alignedAlloc(u8, @alignOf(Fiber), allocation_size));
6464 }
6565
66 fn resultSlice(f: *Fiber) []u8 {
67 const base: [*]align(@alignOf(Fiber)) u8 = @ptrCast(f);
68 return base[resultOffset()..][0..max_result_len];
66 fn allocatedSlice(f: *Fiber) []align(@alignOf(Fiber)) u8 {
67 return @as([*]align(@alignOf(Fiber)) u8, @ptrCast(f))[0..allocation_size];
6968 }
7069
71 fn stackEndPointer(f: *Fiber) [*]u8 {
70 fn allocatedEnd(f: *Fiber) [*]u8 {
7271 const allocated_slice = f.allocatedSlice();
7372 return allocated_slice[allocated_slice.len..].ptr;
7473 }
74
75 fn resultPointer(f: *Fiber) [*]u8 {
76 return @ptrFromInt(f.result_align.forward(@intFromPtr(f) + @sizeOf(Fiber)));
77 }
7578};
7679
7780pub fn io(el: *EventLoop) Io {
......@@ -88,7 +91,7 @@ pub fn init(el: *EventLoop, gpa: Allocator) error{OutOfMemory}!void {
8891 const threads_bytes = ((std.Thread.getCpuCount() catch 1) -| 1) * @sizeOf(Thread);
8992 const idle_context_offset = std.mem.alignForward(usize, threads_bytes, @alignOf(Context));
9093 const idle_stack_end_offset = std.mem.alignForward(usize, idle_context_offset + idle_stack_size, std.heap.page_size_max);
91 const allocated_slice = try gpa.alignedAlloc(u8, @max(@alignOf(Thread), @alignOf(Context), stack_align), idle_stack_end_offset);
94 const allocated_slice = try gpa.alignedAlloc(u8, @max(@alignOf(Thread), @alignOf(Context)), idle_stack_end_offset);
9295 errdefer gpa.free(allocated_slice);
9396 el.* = .{
9497 .gpa = gpa,
......@@ -96,13 +99,13 @@ pub fn init(el: *EventLoop, gpa: Allocator) error{OutOfMemory}!void {
9699 .cond = .{},
97100 .queue = .{},
98101 .free = .{},
99 .main_fiber_buffer = undefined,
102 .main_context = undefined,
100103 .exit_awaiter = null,
101104 .idle_count = 0,
102105 .threads = .initBuffer(@ptrCast(allocated_slice[0..threads_bytes])),
103106 };
104107 const main_idle_context: *Context = @alignCast(std.mem.bytesAsValue(Context, allocated_slice[idle_context_offset..][0..@sizeOf(Context)]));
105 const idle_stack_end: [*]align(stack_align) usize = @alignCast(@ptrCast(allocated_slice[idle_stack_end_offset..].ptr));
108 const idle_stack_end: [*]align(@max(@alignOf(Thread), @alignOf(Context))) usize = @alignCast(@ptrCast(allocated_slice[idle_stack_end_offset..].ptr));
106109 (idle_stack_end - 1)[0..1].* = .{@intFromPtr(el)};
107110 main_idle_context.* = .{
108111 .rsp = @intFromPtr(idle_stack_end - 1),
......@@ -111,9 +114,8 @@ pub fn init(el: *EventLoop, gpa: Allocator) error{OutOfMemory}!void {
111114 };
112115 std.log.debug("created main idle {*}", .{main_idle_context});
113116 current_idle_context = main_idle_context;
114 const current_fiber: *Fiber = @ptrCast(&el.main_fiber_buffer);
115 std.log.debug("created main fiber {*}", .{current_fiber});
116 current_fiber_context = &current_fiber.context;
117 std.log.debug("created main {*}", .{&el.main_context});
118 current_context = &el.main_context;
117119}
118120
119121pub fn deinit(el: *EventLoop) void {
......@@ -125,27 +127,11 @@ pub fn deinit(el: *EventLoop) void {
125127 }
126128 const idle_context_offset = std.mem.alignForward(usize, el.threads.items.len * @sizeOf(Thread), @alignOf(Context));
127129 const idle_stack_end = std.mem.alignForward(usize, idle_context_offset + idle_stack_size, std.heap.page_size_max);
128 const allocated_ptr: [*]align(@max(@alignOf(Thread), @alignOf(Context), stack_align)) u8 = @alignCast(@ptrCast(el.threads.items.ptr));
130 const allocated_ptr: [*]align(@max(@alignOf(Thread), @alignOf(Context))) u8 = @alignCast(@ptrCast(el.threads.items.ptr));
129131 for (el.threads.items) |*thread| thread.thread.join();
130132 el.gpa.free(allocated_ptr[0..idle_stack_end]);
131133}
132134
133fn allocateFiber(el: *EventLoop) error{OutOfMemory}!*Fiber {
134 const free_node = free_node: {
135 el.mutex.lock();
136 defer el.mutex.unlock();
137 break :free_node el.free.pop();
138 } orelse {
139 const n = std.mem.alignForward(
140 usize,
141 Fiber.resultOffset() + max_result_len + min_stack_size,
142 std.heap.page_size_max,
143 );
144 return @alignCast(@ptrCast(try el.gpa.alignedAlloc(u8, @alignOf(Fiber), n)));
145 };
146 return @alignCast(@fieldParentPtr("queue_node", free_node));
147}
148
149135fn yield(el: *EventLoop, optional_fiber: ?*Fiber, register_awaiter: ?*?*Fiber) void {
150136 const ready_context: *Context = ready_context: {
151137 const ready_fiber: *Fiber = optional_fiber orelse if (ready_node: {
......@@ -159,7 +145,7 @@ fn yield(el: *EventLoop, optional_fiber: ?*Fiber, register_awaiter: ?*?*Fiber) v
159145 break :ready_context &ready_fiber.context;
160146 };
161147 const message: SwitchMessage = .{
162 .prev_context = current_fiber_context,
148 .prev_context = current_context,
163149 .ready_context = ready_context,
164150 .register_awaiter = register_awaiter,
165151 };
......@@ -189,14 +175,13 @@ fn schedule(el: *EventLoop, fiber: *Fiber) void {
189175
190176fn recycle(el: *EventLoop, fiber: *Fiber) void {
191177 std.log.debug("recyling {*}", .{fiber});
192 fiber.awaiter = undefined;
193 @memset(fiber.resultSlice(), undefined);
178 @memset(fiber.allocatedSlice(), undefined);
194179 el.mutex.lock();
195180 defer el.mutex.unlock();
196181 el.free.append(&fiber.queue_node);
197182}
198183
199fn mainIdle(el: *EventLoop, message: *const SwitchMessage) callconv(.c) noreturn {
184fn mainIdle(el: *EventLoop, message: *const SwitchMessage) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Context)))) noreturn {
200185 message.handle(el);
201186 el.yield(el.idle(), null);
202187 unreachable; // switched to dead fiber
......@@ -205,7 +190,7 @@ fn mainIdle(el: *EventLoop, message: *const SwitchMessage) callconv(.c) noreturn
205190fn threadEntry(el: *EventLoop, thread: *Thread) void {
206191 std.log.debug("created thread idle {*}", .{&thread.idle_context});
207192 current_idle_context = &thread.idle_context;
208 current_fiber_context = &thread.idle_context;
193 current_context = &thread.idle_context;
209194 _ = el.idle();
210195}
211196
......@@ -230,7 +215,7 @@ const SwitchMessage = extern struct {
230215 register_awaiter: ?*?*Fiber,
231216
232217 fn handle(message: *const SwitchMessage, el: *EventLoop) void {
233 current_fiber_context = message.ready_context;
218 current_context = message.ready_context;
234219 if (message.register_awaiter) |awaiter| {
235220 const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.prev_context));
236221 if (@atomicRmw(?*Fiber, awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished) el.schedule(prev_fiber);
......@@ -238,10 +223,13 @@ const SwitchMessage = extern struct {
238223 }
239224};
240225
241const Context = extern struct {
242 rsp: usize,
243 rbp: usize,
244 rip: usize,
226const Context = switch (builtin.cpu.arch) {
227 .x86_64 => extern struct {
228 rsp: u64,
229 rbp: u64,
230 rip: u64,
231 },
232 else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)),
245233};
246234
247235inline fn contextSwitch(message: *const SwitchMessage) *const SwitchMessage {
......@@ -299,42 +287,45 @@ fn fiberEntry() callconv(.naked) void {
299287pub fn @"async"(
300288 userdata: ?*anyopaque,
301289 result: []u8,
302 result_alignment: std.mem.Alignment,
290 result_alignment: Alignment,
303291 context: []const u8,
304 context_alignment: std.mem.Alignment,
292 context_alignment: Alignment,
305293 start: *const fn (context: *const anyopaque, result: *anyopaque) void,
306294) ?*std.Io.AnyFuture {
307 assert(result_alignment.compare(.lte, max_result_align)); // TODO
308 assert(context_alignment.compare(.lte, max_result_align)); // TODO
309 assert(result.len <= max_result_len); // TODO
310 assert(context.len <= max_result_len); // TODO
295 assert(result_alignment.compare(.lte, Fiber.max_result_align)); // TODO
296 assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
297 assert(result.len <= Fiber.max_result_size); // TODO
298 assert(context.len <= Fiber.max_context_size); // TODO
311299
312300 const event_loop: *EventLoop = @alignCast(@ptrCast(userdata));
313 const fiber = event_loop.allocateFiber() catch {
301 const fiber = Fiber.allocate(event_loop) catch {
314302 start(context.ptr, result.ptr);
315303 return null;
316304 };
317 fiber.awaiter = null;
318 fiber.queue_node = .{ .data = {} };
319 @memcpy(fiber.argsSlice()[0..context.len], context);
320305 std.log.debug("allocated {*}", .{fiber});
321306
322 const closure: *AsyncClosure = @ptrFromInt(std.mem.alignBackward(
323 usize,
324 @intFromPtr(fiber.stackEndPointer() - @sizeOf(AsyncClosure)),
325 @max(@alignOf(AsyncClosure), stack_align),
326 ));
307 const closure: *AsyncClosure = @ptrFromInt(Fiber.max_context_align.max(.of(AsyncClosure)).backward(
308 @intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
309 ) - @sizeOf(AsyncClosure));
310 fiber.* = .{
311 .context = switch (builtin.cpu.arch) {
312 .x86_64 => .{
313 .rsp = @intFromPtr(closure) - @sizeOf(usize),
314 .rbp = 0,
315 .rip = @intFromPtr(&fiberEntry),
316 },
317 else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)),
318 },
319 .awaiter = null,
320 .queue_node = undefined,
321 .result_align = result_alignment,
322 };
327323 closure.* = .{
328324 .event_loop = event_loop,
329325 .fiber = fiber,
330326 .start = start,
331327 };
332 const stack_end: [*]align(stack_align) usize = @alignCast(@ptrCast(closure));
333 fiber.context = .{
334 .rsp = @intFromPtr(stack_end - 1),
335 .rbp = 0,
336 .rip = @intFromPtr(&fiberEntry),
337 };
328 @memcpy(closure.contextPointer(), context);
338329
339330 event_loop.schedule(fiber);
340331 return @ptrCast(fiber);
......@@ -345,10 +336,14 @@ const AsyncClosure = struct {
345336 fiber: *Fiber,
346337 start: *const fn (context: *const anyopaque, result: *anyopaque) void,
347338
348 fn call(closure: *AsyncClosure, message: *const SwitchMessage) callconv(.c) noreturn {
339 fn contextPointer(closure: *AsyncClosure) [*]align(Fiber.max_context_align.toByteUnits()) u8 {
340 return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(AsyncClosure));
341 }
342
343 fn call(closure: *AsyncClosure, message: *const SwitchMessage) callconv(.withStackAlign(.c, @alignOf(AsyncClosure))) noreturn {
349344 message.handle(closure.event_loop);
350345 std.log.debug("{*} performing async", .{closure.fiber});
351 closure.start(closure.fiber.argsSlice().ptr, closure.fiber.resultSlice().ptr);
346 closure.start(closure.contextPointer(), closure.fiber.resultPointer());
352347 const awaiter = @atomicRmw(?*Fiber, &closure.fiber.awaiter, .Xchg, Fiber.finished, .acq_rel);
353348 closure.event_loop.yield(awaiter, null);
354349 unreachable; // switched to dead fiber
......@@ -358,8 +353,7 @@ const AsyncClosure = struct {
358353pub fn @"await"(userdata: ?*anyopaque, any_future: *std.Io.AnyFuture, result: []u8) void {
359354 const event_loop: *EventLoop = @alignCast(@ptrCast(userdata));
360355 const future_fiber: *Fiber = @alignCast(@ptrCast(any_future));
361 const result_src = future_fiber.resultSlice()[0..result.len];
362356 if (@atomicLoad(?*Fiber, &future_fiber.awaiter, .acquire) != Fiber.finished) event_loop.yield(null, &future_fiber.awaiter);
363 @memcpy(result, result_src);
357 @memcpy(result, future_fiber.resultPointer());
364358 event_loop.recycle(future_fiber);
365359}