| author |  Andrew Kelley <andrew@ziglang.org> 2025-09-21 17:26:00-07:00 |
| committer | Andrew Kelley <andrew@ziglang.org> 2025-10-02 16:30:59-07:00 |
| log | 0f2427c5d2e16626c6c88621e8aa3aa1b4aa8785 |
| tree | 9f5c876b04e539411411522bf3f89d85dcb9d232 |
| parent | 43eea7beecaeb29d048cf971d6d08d0297109ce0 |
std.Io: rename ThreadPool to Threaded
3 files changed, 1263 insertions(+), 1263 deletions(-)
lib
lib/std/Io.zig+3-3
| ... | @@ -558,7 +558,7 @@ test { | ... | @@ -558,7 +558,7 @@ test { |
| 558 | const Io = @This(); | 558 | const Io = @This(); |
| 559 | 559 | ||
| 560 | pub const EventLoop = @import("Io/EventLoop.zig"); | 560 | pub const EventLoop = @import("Io/EventLoop.zig"); |
| 561 | pub const ThreadPool = @import("Io/ThreadPool.zig"); | 561 | pub const Threaded = @import("Io/Threaded.zig"); |
| 562 | pub const net = @import("Io/net.zig"); | 562 | pub const net = @import("Io/net.zig"); |
| 563 | 563 | ||
| 564 | userdata: ?*anyopaque, | 564 | userdata: ?*anyopaque, |
| ... | @@ -668,8 +668,8 @@ pub const VTable = struct { | ... | @@ -668,8 +668,8 @@ pub const VTable = struct { |
| 668 | netRead: *const fn (?*anyopaque, src: net.Stream, data: [][]u8) net.Stream.Reader.Error!usize, | 668 | netRead: *const fn (?*anyopaque, src: net.Stream, data: [][]u8) net.Stream.Reader.Error!usize, |
| 669 | netWrite: *const fn (?*anyopaque, dest: net.Stream, header: []const u8, data: []const []const u8, splat: usize) net.Stream.Writer.Error!usize, | 669 | netWrite: *const fn (?*anyopaque, dest: net.Stream, header: []const u8, data: []const []const u8, splat: usize) net.Stream.Writer.Error!usize, |
| 670 | netClose: *const fn (?*anyopaque, stream: net.Stream) void, | 670 | netClose: *const fn (?*anyopaque, stream: net.Stream) void, |
| 671 | /// Equivalent to libc "if_nametoindex". | 671 | netInterfaceNameResolve: *const fn (?*anyopaque, net.Interface.Name) net.Interface.Name.ResolveError!net.Interface, |
| 672 | netInterfaceIndex: *const fn (?*anyopaque, name: []const u8) net.InterfaceIndexError!u32, | 672 | netInterfaceName: *const fn (?*anyopaque, net.Interface) net.Interface.NameError!net.Interface.Name, |
| 673 | }; | 673 | }; |
| 674 | 674 | ||
| 675 | pub const Cancelable = error{ | 675 | pub const Cancelable = error{ |
lib/std/Io/ThreadPool.zig deleted-1260
| ... | @@ -1,1260 +0,0 @@ | ||
| 1 | const Pool = @This(); | ||
| 2 | |||
| 3 | const builtin = @import("builtin"); | ||
| 4 | const native_os = builtin.os.tag; | ||
| 5 | const is_windows = native_os == .windows; | ||
| 6 | const windows = std.os.windows; | ||
| 7 | |||
| 8 | const std = @import("../std.zig"); | ||
| 9 | const Allocator = std.mem.Allocator; | ||
| 10 | const assert = std.debug.assert; | ||
| 11 | const WaitGroup = std.Thread.WaitGroup; | ||
| 12 | const posix = std.posix; | ||
| 13 | const Io = std.Io; | ||
| 14 | |||
| 15 | /// Thread-safe. | ||
| 16 | allocator: Allocator, | ||
| 17 | mutex: std.Thread.Mutex = .{}, | ||
| 18 | cond: std.Thread.Condition = .{}, | ||
| 19 | run_queue: std.SinglyLinkedList = .{}, | ||
| 20 | join_requested: bool = false, | ||
| 21 | threads: std.ArrayListUnmanaged(std.Thread), | ||
| 22 | stack_size: usize, | ||
| 23 | cpu_count: std.Thread.CpuCountError!usize, | ||
| 24 | parallel_count: usize, | ||
| 25 | |||
| 26 | threadlocal var current_closure: ?*AsyncClosure = null; | ||
| 27 | |||
| 28 | const max_iovecs_len = 8; | ||
| 29 | const splat_buffer_size = 64; | ||
| 30 | |||
| 31 | comptime { | ||
| 32 | assert(max_iovecs_len <= posix.IOV_MAX); | ||
| 33 | } | ||
| 34 | |||
| 35 | pub const Runnable = struct { | ||
| 36 | start: Start, | ||
| 37 | node: std.SinglyLinkedList.Node = .{}, | ||
| 38 | is_parallel: bool, | ||
| 39 | |||
| 40 | pub const Start = *const fn (*Runnable) void; | ||
| 41 | }; | ||
| 42 | |||
| 43 | pub const InitError = std.Thread.CpuCountError || Allocator.Error; | ||
| 44 | |||
| 45 | pub fn init(gpa: Allocator) Pool { | ||
| 46 | var pool: Pool = .{ | ||
| 47 | .allocator = gpa, | ||
| 48 | .threads = .empty, | ||
| 49 | .stack_size = std.Thread.SpawnConfig.default_stack_size, | ||
| 50 | .cpu_count = std.Thread.getCpuCount(), | ||
| 51 | .parallel_count = 0, | ||
| 52 | }; | ||
| 53 | if (pool.cpu_count) |n| { | ||
| 54 | pool.threads.ensureTotalCapacityPrecise(gpa, n - 1) catch {}; | ||
| 55 | } else |_| {} | ||
| 56 | return pool; | ||
| 57 | } | ||
| 58 | |||
| 59 | pub fn deinit(pool: *Pool) void { | ||
| 60 | const gpa = pool.allocator; | ||
| 61 | pool.join(); | ||
| 62 | pool.threads.deinit(gpa); | ||
| 63 | pool.* = undefined; | ||
| 64 | } | ||
| 65 | |||
| 66 | fn join(pool: *Pool) void { | ||
| 67 | if (builtin.single_threaded) return; | ||
| 68 | { | ||
| 69 | pool.mutex.lock(); | ||
| 70 | defer pool.mutex.unlock(); | ||
| 71 | pool.join_requested = true; | ||
| 72 | } | ||
| 73 | pool.cond.broadcast(); | ||
| 74 | for (pool.threads.items) |thread| thread.join(); | ||
| 75 | } | ||
| 76 | |||
| 77 | fn worker(pool: *Pool) void { | ||
| 78 | pool.mutex.lock(); | ||
| 79 | defer pool.mutex.unlock(); | ||
| 80 | |||
| 81 | while (true) { | ||
| 82 | while (pool.run_queue.popFirst()) |run_node| { | ||
| 83 | pool.mutex.unlock(); | ||
| 84 | const runnable: *Runnable = @fieldParentPtr("node", run_node); | ||
| 85 | runnable.start(runnable); | ||
| 86 | pool.mutex.lock(); | ||
| 87 | if (runnable.is_parallel) { | ||
| 88 | // TODO also pop thread and join sometimes | ||
| 89 | pool.parallel_count -= 1; | ||
| 90 | } | ||
| 91 | } | ||
| 92 | if (pool.join_requested) break; | ||
| 93 | pool.cond.wait(&pool.mutex); | ||
| 94 | } | ||
| 95 | } | ||
| 96 | |||
| 97 | pub fn io(pool: *Pool) Io { | ||
| 98 | return .{ | ||
| 99 | .userdata = pool, | ||
| 100 | .vtable = &.{ | ||
| 101 | .async = async, | ||
| 102 | .asyncConcurrent = asyncConcurrent, | ||
| 103 | .await = await, | ||
| 104 | .asyncDetached = asyncDetached, | ||
| 105 | .cancel = cancel, | ||
| 106 | .cancelRequested = cancelRequested, | ||
| 107 | .select = select, | ||
| 108 | |||
| 109 | .mutexLock = mutexLock, | ||
| 110 | .mutexUnlock = mutexUnlock, | ||
| 111 | |||
| 112 | .conditionWait = conditionWait, | ||
| 113 | .conditionWake = conditionWake, | ||
| 114 | |||
| 115 | .createFile = createFile, | ||
| 116 | .fileOpen = fileOpen, | ||
| 117 | .fileClose = fileClose, | ||
| 118 | .pwrite = pwrite, | ||
| 119 | .fileReadStreaming = fileReadStreaming, | ||
| 120 | .fileReadPositional = fileReadPositional, | ||
| 121 | .fileSeekBy = fileSeekBy, | ||
| 122 | .fileSeekTo = fileSeekTo, | ||
| 123 | |||
| 124 | .now = now, | ||
| 125 | .sleep = sleep, | ||
| 126 | |||
| 127 | .listen = listen, | ||
| 128 | .accept = accept, | ||
| 129 | .netRead = switch (builtin.os.tag) { | ||
| 130 | .windows => @panic("TODO"), | ||
| 131 | else => netReadPosix, | ||
| 132 | }, | ||
| 133 | .netWrite = switch (builtin.os.tag) { | ||
| 134 | .windows => @panic("TODO"), | ||
| 135 | else => netWritePosix, | ||
| 136 | }, | ||
| 137 | .netClose = netClose, | ||
| 138 | .netInterfaceIndex = netInterfaceIndex, | ||
| 139 | }, | ||
| 140 | }; | ||
| 141 | } | ||
| 142 | |||
| 143 | const AsyncClosure = struct { | ||
| 144 | func: *const fn (context: *anyopaque, result: *anyopaque) void, | ||
| 145 | runnable: Runnable, | ||
| 146 | reset_event: std.Thread.ResetEvent, | ||
| 147 | select_condition: ?*std.Thread.ResetEvent, | ||
| 148 | cancel_tid: std.Thread.Id, | ||
| 149 | context_offset: usize, | ||
| 150 | result_offset: usize, | ||
| 151 | |||
| 152 | const done_reset_event: *std.Thread.ResetEvent = @ptrFromInt(@alignOf(std.Thread.ResetEvent)); | ||
| 153 | |||
| 154 | const canceling_tid: std.Thread.Id = switch (@typeInfo(std.Thread.Id)) { | ||
| 155 | .int => |int_info| switch (int_info.signedness) { | ||
| 156 | .signed => -1, | ||
| 157 | .unsigned => std.math.maxInt(std.Thread.Id), | ||
| 158 | }, | ||
| 159 | .pointer => @ptrFromInt(std.math.maxInt(usize)), | ||
| 160 | else => @compileError("unsupported std.Thread.Id: " ++ @typeName(std.Thread.Id)), | ||
| 161 | }; | ||
| 162 | |||
| 163 | fn start(runnable: *Runnable) void { | ||
| 164 | const closure: *AsyncClosure = @alignCast(@fieldParentPtr("runnable", runnable)); | ||
| 165 | const tid = std.Thread.getCurrentId(); | ||
| 166 | if (@cmpxchgStrong( | ||
| 167 | std.Thread.Id, | ||
| 168 | &closure.cancel_tid, | ||
| 169 | 0, | ||
| 170 | tid, | ||
| 171 | .acq_rel, | ||
| 172 | .acquire, | ||
| 173 | )) |cancel_tid| { | ||
| 174 | assert(cancel_tid == canceling_tid); | ||
| 175 | closure.reset_event.set(); | ||
| 176 | return; | ||
| 177 | } | ||
| 178 | current_closure = closure; | ||
| 179 | closure.func(closure.contextPointer(), closure.resultPointer()); | ||
| 180 | current_closure = null; | ||
| 181 | if (@cmpxchgStrong( | ||
| 182 | std.Thread.Id, | ||
| 183 | &closure.cancel_tid, | ||
| 184 | tid, | ||
| 185 | 0, | ||
| 186 | .acq_rel, | ||
| 187 | .acquire, | ||
| 188 | )) |cancel_tid| assert(cancel_tid == canceling_tid); | ||
| 189 | |||
| 190 | if (@atomicRmw( | ||
| 191 | ?*std.Thread.ResetEvent, | ||
| 192 | &closure.select_condition, | ||
| 193 | .Xchg, | ||
| 194 | done_reset_event, | ||
| 195 | .release, | ||
| 196 | )) |select_reset| { | ||
| 197 | assert(select_reset != done_reset_event); | ||
| 198 | select_reset.set(); | ||
| 199 | } | ||
| 200 | closure.reset_event.set(); | ||
| 201 | } | ||
| 202 | |||
| 203 | fn contextOffset(context_alignment: std.mem.Alignment) usize { | ||
| 204 | return context_alignment.forward(@sizeOf(AsyncClosure)); | ||
| 205 | } | ||
| 206 | |||
| 207 | fn resultOffset( | ||
| 208 | context_alignment: std.mem.Alignment, | ||
| 209 | context_len: usize, | ||
| 210 | result_alignment: std.mem.Alignment, | ||
| 211 | ) usize { | ||
| 212 | return result_alignment.forward(contextOffset(context_alignment) + context_len); | ||
| 213 | } | ||
| 214 | |||
| 215 | fn resultPointer(closure: *AsyncClosure) [*]u8 { | ||
| 216 | const base: [*]u8 = @ptrCast(closure); | ||
| 217 | return base + closure.result_offset; | ||
| 218 | } | ||
| 219 | |||
| 220 | fn contextPointer(closure: *AsyncClosure) [*]u8 { | ||
| 221 | const base: [*]u8 = @ptrCast(closure); | ||
| 222 | return base + closure.context_offset; | ||
| 223 | } | ||
| 224 | |||
| 225 | fn waitAndFree(closure: *AsyncClosure, gpa: Allocator, result: []u8) void { | ||
| 226 | closure.reset_event.wait(); | ||
| 227 | @memcpy(result, closure.resultPointer()[0..result.len]); | ||
| 228 | free(closure, gpa, result.len); | ||
| 229 | } | ||
| 230 | |||
| 231 | fn free(closure: *AsyncClosure, gpa: Allocator, result_len: usize) void { | ||
| 232 | const base: [*]align(@alignOf(AsyncClosure)) u8 = @ptrCast(closure); | ||
| 233 | gpa.free(base[0 .. closure.result_offset + result_len]); | ||
| 234 | } | ||
| 235 | }; | ||
| 236 | |||
| 237 | fn async( | ||
| 238 | userdata: ?*anyopaque, | ||
| 239 | result: []u8, | ||
| 240 | result_alignment: std.mem.Alignment, | ||
| 241 | context: []const u8, | ||
| 242 | context_alignment: std.mem.Alignment, | ||
| 243 | start: *const fn (context: *const anyopaque, result: *anyopaque) void, | ||
| 244 | ) ?*Io.AnyFuture { | ||
| 245 | if (builtin.single_threaded) { | ||
| 246 | start(context.ptr, result.ptr); | ||
| 247 | return null; | ||
| 248 | } | ||
| 249 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 250 | const cpu_count = pool.cpu_count catch { | ||
| 251 | return asyncConcurrent(userdata, result.len, result_alignment, context, context_alignment, start) catch { | ||
| 252 | start(context.ptr, result.ptr); | ||
| 253 | return null; | ||
| 254 | }; | ||
| 255 | }; | ||
| 256 | const gpa = pool.allocator; | ||
| 257 | const context_offset = context_alignment.forward(@sizeOf(AsyncClosure)); | ||
| 258 | const result_offset = result_alignment.forward(context_offset + context.len); | ||
| 259 | const n = result_offset + result.len; | ||
| 260 | const closure: *AsyncClosure = @ptrCast(@alignCast(gpa.alignedAlloc(u8, .of(AsyncClosure), n) catch { | ||
| 261 | start(context.ptr, result.ptr); | ||
| 262 | return null; | ||
| 263 | })); | ||
| 264 | |||
| 265 | closure.* = .{ | ||
| 266 | .func = start, | ||
| 267 | .context_offset = context_offset, | ||
| 268 | .result_offset = result_offset, | ||
| 269 | .reset_event = .{}, | ||
| 270 | .cancel_tid = 0, | ||
| 271 | .select_condition = null, | ||
| 272 | .runnable = .{ | ||
| 273 | .start = AsyncClosure.start, | ||
| 274 | .is_parallel = false, | ||
| 275 | }, | ||
| 276 | }; | ||
| 277 | |||
| 278 | @memcpy(closure.contextPointer()[0..context.len], context); | ||
| 279 | |||
| 280 | pool.mutex.lock(); | ||
| 281 | |||
| 282 | const thread_capacity = cpu_count - 1 + pool.parallel_count; | ||
| 283 | |||
| 284 | pool.threads.ensureTotalCapacityPrecise(gpa, thread_capacity) catch { | ||
| 285 | pool.mutex.unlock(); | ||
| 286 | closure.free(gpa, result.len); | ||
| 287 | start(context.ptr, result.ptr); | ||
| 288 | return null; | ||
| 289 | }; | ||
| 290 | |||
| 291 | pool.run_queue.prepend(&closure.runnable.node); | ||
| 292 | |||
| 293 | if (pool.threads.items.len < thread_capacity) { | ||
| 294 | const thread = std.Thread.spawn(.{ .stack_size = pool.stack_size }, worker, .{pool}) catch { | ||
| 295 | if (pool.threads.items.len == 0) { | ||
| 296 | assert(pool.run_queue.popFirst() == &closure.runnable.node); | ||
| 297 | pool.mutex.unlock(); | ||
| 298 | closure.free(gpa, result.len); | ||
| 299 | start(context.ptr, result.ptr); | ||
| 300 | return null; | ||
| 301 | } | ||
| 302 | // Rely on other workers to do it. | ||
| 303 | pool.mutex.unlock(); | ||
| 304 | pool.cond.signal(); | ||
| 305 | return @ptrCast(closure); | ||
| 306 | }; | ||
| 307 | pool.threads.appendAssumeCapacity(thread); | ||
| 308 | } | ||
| 309 | |||
| 310 | pool.mutex.unlock(); | ||
| 311 | pool.cond.signal(); | ||
| 312 | return @ptrCast(closure); | ||
| 313 | } | ||
| 314 | |||
| 315 | fn asyncConcurrent( | ||
| 316 | userdata: ?*anyopaque, | ||
| 317 | result_len: usize, | ||
| 318 | result_alignment: std.mem.Alignment, | ||
| 319 | context: []const u8, | ||
| 320 | context_alignment: std.mem.Alignment, | ||
| 321 | start: *const fn (context: *const anyopaque, result: *anyopaque) void, | ||
| 322 | ) error{OutOfMemory}!*Io.AnyFuture { | ||
| 323 | if (builtin.single_threaded) unreachable; | ||
| 324 | |||
| 325 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 326 | const cpu_count = pool.cpu_count catch 1; | ||
| 327 | const gpa = pool.allocator; | ||
| 328 | const context_offset = context_alignment.forward(@sizeOf(AsyncClosure)); | ||
| 329 | const result_offset = result_alignment.forward(context_offset + context.len); | ||
| 330 | const n = result_offset + result_len; | ||
| 331 | const closure: *AsyncClosure = @ptrCast(@alignCast(try gpa.alignedAlloc(u8, .of(AsyncClosure), n))); | ||
| 332 | |||
| 333 | closure.* = .{ | ||
| 334 | .func = start, | ||
| 335 | .context_offset = context_offset, | ||
| 336 | .result_offset = result_offset, | ||
| 337 | .reset_event = .{}, | ||
| 338 | .cancel_tid = 0, | ||
| 339 | .select_condition = null, | ||
| 340 | .runnable = .{ | ||
| 341 | .start = AsyncClosure.start, | ||
| 342 | .is_parallel = true, | ||
| 343 | }, | ||
| 344 | }; | ||
| 345 | @memcpy(closure.contextPointer()[0..context.len], context); | ||
| 346 | |||
| 347 | pool.mutex.lock(); | ||
| 348 | |||
| 349 | pool.parallel_count += 1; | ||
| 350 | const thread_capacity = cpu_count - 1 + pool.parallel_count; | ||
| 351 | |||
| 352 | pool.threads.ensureTotalCapacity(gpa, thread_capacity) catch { | ||
| 353 | pool.mutex.unlock(); | ||
| 354 | closure.free(gpa, result_len); | ||
| 355 | return error.OutOfMemory; | ||
| 356 | }; | ||
| 357 | |||
| 358 | pool.run_queue.prepend(&closure.runnable.node); | ||
| 359 | |||
| 360 | if (pool.threads.items.len < thread_capacity) { | ||
| 361 | const thread = std.Thread.spawn(.{ .stack_size = pool.stack_size }, worker, .{pool}) catch { | ||
| 362 | assert(pool.run_queue.popFirst() == &closure.runnable.node); | ||
| 363 | pool.mutex.unlock(); | ||
| 364 | closure.free(gpa, result_len); | ||
| 365 | return error.OutOfMemory; | ||
| 366 | }; | ||
| 367 | pool.threads.appendAssumeCapacity(thread); | ||
| 368 | } | ||
| 369 | |||
| 370 | pool.mutex.unlock(); | ||
| 371 | pool.cond.signal(); | ||
| 372 | return @ptrCast(closure); | ||
| 373 | } | ||
| 374 | |||
| 375 | const DetachedClosure = struct { | ||
| 376 | pool: *Pool, | ||
| 377 | func: *const fn (context: *anyopaque) void, | ||
| 378 | runnable: Runnable, | ||
| 379 | context_alignment: std.mem.Alignment, | ||
| 380 | context_len: usize, | ||
| 381 | |||
| 382 | fn start(runnable: *Runnable) void { | ||
| 383 | const closure: *DetachedClosure = @alignCast(@fieldParentPtr("runnable", runnable)); | ||
| 384 | closure.func(closure.contextPointer()); | ||
| 385 | const gpa = closure.pool.allocator; | ||
| 386 | free(closure, gpa); | ||
| 387 | } | ||
| 388 | |||
| 389 | fn free(closure: *DetachedClosure, gpa: Allocator) void { | ||
| 390 | const base: [*]align(@alignOf(DetachedClosure)) u8 = @ptrCast(closure); | ||
| 391 | gpa.free(base[0..contextEnd(closure.context_alignment, closure.context_len)]); | ||
| 392 | } | ||
| 393 | |||
| 394 | fn contextOffset(context_alignment: std.mem.Alignment) usize { | ||
| 395 | return context_alignment.forward(@sizeOf(DetachedClosure)); | ||
| 396 | } | ||
| 397 | |||
| 398 | fn contextEnd(context_alignment: std.mem.Alignment, context_len: usize) usize { | ||
| 399 | return contextOffset(context_alignment) + context_len; | ||
| 400 | } | ||
| 401 | |||
| 402 | fn contextPointer(closure: *DetachedClosure) [*]u8 { | ||
| 403 | const base: [*]u8 = @ptrCast(closure); | ||
| 404 | return base + contextOffset(closure.context_alignment); | ||
| 405 | } | ||
| 406 | }; | ||
| 407 | |||
| 408 | fn asyncDetached( | ||
| 409 | userdata: ?*anyopaque, | ||
| 410 | context: []const u8, | ||
| 411 | context_alignment: std.mem.Alignment, | ||
| 412 | start: *const fn (context: *const anyopaque) void, | ||
| 413 | ) void { | ||
| 414 | if (builtin.single_threaded) return start(context.ptr); | ||
| 415 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 416 | const cpu_count = pool.cpu_count catch 1; | ||
| 417 | const gpa = pool.allocator; | ||
| 418 | const n = DetachedClosure.contextEnd(context_alignment, context.len); | ||
| 419 | const closure: *DetachedClosure = @ptrCast(@alignCast(gpa.alignedAlloc(u8, .of(DetachedClosure), n) catch { | ||
| 420 | return start(context.ptr); | ||
| 421 | })); | ||
| 422 | closure.* = .{ | ||
| 423 | .pool = pool, | ||
| 424 | .func = start, | ||
| 425 | .context_alignment = context_alignment, | ||
| 426 | .context_len = context.len, | ||
| 427 | .runnable = .{ | ||
| 428 | .start = DetachedClosure.start, | ||
| 429 | .is_parallel = false, | ||
| 430 | }, | ||
| 431 | }; | ||
| 432 | @memcpy(closure.contextPointer()[0..context.len], context); | ||
| 433 | |||
| 434 | pool.mutex.lock(); | ||
| 435 | |||
| 436 | const thread_capacity = cpu_count - 1 + pool.parallel_count; | ||
| 437 | |||
| 438 | pool.threads.ensureTotalCapacityPrecise(gpa, thread_capacity) catch { | ||
| 439 | pool.mutex.unlock(); | ||
| 440 | closure.free(gpa); | ||
| 441 | return start(context.ptr); | ||
| 442 | }; | ||
| 443 | |||
| 444 | pool.run_queue.prepend(&closure.runnable.node); | ||
| 445 | |||
| 446 | if (pool.threads.items.len < thread_capacity) { | ||
| 447 | const thread = std.Thread.spawn(.{ .stack_size = pool.stack_size }, worker, .{pool}) catch { | ||
| 448 | assert(pool.run_queue.popFirst() == &closure.runnable.node); | ||
| 449 | pool.mutex.unlock(); | ||
| 450 | closure.free(gpa); | ||
| 451 | return start(context.ptr); | ||
| 452 | }; | ||
| 453 | pool.threads.appendAssumeCapacity(thread); | ||
| 454 | } | ||
| 455 | |||
| 456 | pool.mutex.unlock(); | ||
| 457 | pool.cond.signal(); | ||
| 458 | } | ||
| 459 | |||
| 460 | fn await( | ||
| 461 | userdata: ?*anyopaque, | ||
| 462 | any_future: *std.Io.AnyFuture, | ||
| 463 | result: []u8, | ||
| 464 | result_alignment: std.mem.Alignment, | ||
| 465 | ) void { | ||
| 466 | _ = result_alignment; | ||
| 467 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 468 | const closure: *AsyncClosure = @ptrCast(@alignCast(any_future)); | ||
| 469 | closure.waitAndFree(pool.allocator, result); | ||
| 470 | } | ||
| 471 | |||
| 472 | fn cancel( | ||
| 473 | userdata: ?*anyopaque, | ||
| 474 | any_future: *Io.AnyFuture, | ||
| 475 | result: []u8, | ||
| 476 | result_alignment: std.mem.Alignment, | ||
| 477 | ) void { | ||
| 478 | _ = result_alignment; | ||
| 479 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 480 | const closure: *AsyncClosure = @ptrCast(@alignCast(any_future)); | ||
| 481 | switch (@atomicRmw( | ||
| 482 | std.Thread.Id, | ||
| 483 | &closure.cancel_tid, | ||
| 484 | .Xchg, | ||
| 485 | AsyncClosure.canceling_tid, | ||
| 486 | .acq_rel, | ||
| 487 | )) { | ||
| 488 | 0, AsyncClosure.canceling_tid => {}, | ||
| 489 | else => |cancel_tid| switch (builtin.os.tag) { | ||
| 490 | .linux => _ = std.os.linux.tgkill( | ||
| 491 | std.os.linux.getpid(), | ||
| 492 | @bitCast(cancel_tid), | ||
| 493 | posix.SIG.IO, | ||
| 494 | ), | ||
| 495 | else => {}, | ||
| 496 | }, | ||
| 497 | } | ||
| 498 | closure.waitAndFree(pool.allocator, result); | ||
| 499 | } | ||
| 500 | |||
| 501 | fn cancelRequested(userdata: ?*anyopaque) bool { | ||
| 502 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 503 | _ = pool; | ||
| 504 | const closure = current_closure orelse return false; | ||
| 505 | return @atomicLoad(std.Thread.Id, &closure.cancel_tid, .acquire) == AsyncClosure.canceling_tid; | ||
| 506 | } | ||
| 507 | |||
| 508 | fn checkCancel(pool: *Pool) error{Canceled}!void { | ||
| 509 | if (cancelRequested(pool)) return error.Canceled; | ||
| 510 | } | ||
| 511 | |||
| 512 | fn mutexLock(userdata: ?*anyopaque, prev_state: Io.Mutex.State, mutex: *Io.Mutex) error{Canceled}!void { | ||
| 513 | _ = userdata; | ||
| 514 | if (prev_state == .contended) { | ||
| 515 | std.Thread.Futex.wait(@ptrCast(&mutex.state), @intFromEnum(Io.Mutex.State.contended)); | ||
| 516 | } | ||
| 517 | while (@atomicRmw( | ||
| 518 | Io.Mutex.State, | ||
| 519 | &mutex.state, | ||
| 520 | .Xchg, | ||
| 521 | .contended, | ||
| 522 | .acquire, | ||
| 523 | ) != .unlocked) { | ||
| 524 | std.Thread.Futex.wait(@ptrCast(&mutex.state), @intFromEnum(Io.Mutex.State.contended)); | ||
| 525 | } | ||
| 526 | } | ||
| 527 | fn mutexUnlock(userdata: ?*anyopaque, prev_state: Io.Mutex.State, mutex: *Io.Mutex) void { | ||
| 528 | _ = userdata; | ||
| 529 | _ = prev_state; | ||
| 530 | if (@atomicRmw(Io.Mutex.State, &mutex.state, .Xchg, .unlocked, .release) == .contended) { | ||
| 531 | std.Thread.Futex.wake(@ptrCast(&mutex.state), 1); | ||
| 532 | } | ||
| 533 | } | ||
| 534 | |||
| 535 | fn conditionWait(userdata: ?*anyopaque, cond: *Io.Condition, mutex: *Io.Mutex) Io.Cancelable!void { | ||
| 536 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 537 | comptime assert(@TypeOf(cond.state) == u64); | ||
| 538 | const ints: *[2]std.atomic.Value(u32) = @ptrCast(&cond.state); | ||
| 539 | const cond_state = &ints[0]; | ||
| 540 | const cond_epoch = &ints[1]; | ||
| 541 | const one_waiter = 1; | ||
| 542 | const waiter_mask = 0xffff; | ||
| 543 | const one_signal = 1 << 16; | ||
| 544 | const signal_mask = 0xffff << 16; | ||
| 545 | // Observe the epoch, then check the state again to see if we should wake up. | ||
| 546 | // The epoch must be observed before we check the state or we could potentially miss a wake() and deadlock: | ||
| 547 | // | ||
| 548 | // - T1: s = LOAD(&state) | ||
| 549 | // - T2: UPDATE(&s, signal) | ||
| 550 | // - T2: UPDATE(&epoch, 1) + FUTEX_WAKE(&epoch) | ||
| 551 | // - T1: e = LOAD(&epoch) (was reordered after the state load) | ||
| 552 | // - T1: s & signals == 0 -> FUTEX_WAIT(&epoch, e) (missed the state update + the epoch change) | ||
| 553 | // | ||
| 554 | // Acquire barrier to ensure the epoch load happens before the state load. | ||
| 555 | var epoch = cond_epoch.load(.acquire); | ||
| 556 | var state = cond_state.fetchAdd(one_waiter, .monotonic); | ||
| 557 | assert(state & waiter_mask != waiter_mask); | ||
| 558 | state += one_waiter; | ||
| 559 | |||
| 560 | mutex.unlock(pool.io()); | ||
| 561 | defer mutex.lock(pool.io()) catch @panic("TODO"); | ||
| 562 | |||
| 563 | var futex_deadline = std.Thread.Futex.Deadline.init(null); | ||
| 564 | |||
| 565 | while (true) { | ||
| 566 | futex_deadline.wait(cond_epoch, epoch) catch |err| switch (err) { | ||
| 567 | error.Timeout => unreachable, | ||
| 568 | }; | ||
| 569 | |||
| 570 | epoch = cond_epoch.load(.acquire); | ||
| 571 | state = cond_state.load(.monotonic); | ||
| 572 | |||
| 573 | // Try to wake up by consuming a signal and decremented the waiter we added previously. | ||
| 574 | // Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return. | ||
| 575 | while (state & signal_mask != 0) { | ||
| 576 | const new_state = state - one_waiter - one_signal; | ||
| 577 | state = cond_state.cmpxchgWeak(state, new_state, .acquire, .monotonic) orelse return; | ||
| 578 | } | ||
| 579 | } | ||
| 580 | } | ||
| 581 | |||
| 582 | fn conditionWake(userdata: ?*anyopaque, cond: *Io.Condition, wake: Io.Condition.Wake) void { | ||
| 583 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 584 | _ = pool; | ||
| 585 | comptime assert(@TypeOf(cond.state) == u64); | ||
| 586 | const ints: *[2]std.atomic.Value(u32) = @ptrCast(&cond.state); | ||
| 587 | const cond_state = &ints[0]; | ||
| 588 | const cond_epoch = &ints[1]; | ||
| 589 | const one_waiter = 1; | ||
| 590 | const waiter_mask = 0xffff; | ||
| 591 | const one_signal = 1 << 16; | ||
| 592 | const signal_mask = 0xffff << 16; | ||
| 593 | var state = cond_state.load(.monotonic); | ||
| 594 | while (true) { | ||
| 595 | const waiters = (state & waiter_mask) / one_waiter; | ||
| 596 | const signals = (state & signal_mask) / one_signal; | ||
| 597 | |||
| 598 | // Reserves which waiters to wake up by incrementing the signals count. | ||
| 599 | // Therefore, the signals count is always less than or equal to the waiters count. | ||
| 600 | // We don't need to Futex.wake if there's nothing to wake up or if other wake() threads have reserved to wake up the current waiters. | ||
| 601 | const wakeable = waiters - signals; | ||
| 602 | if (wakeable == 0) { | ||
| 603 | return; | ||
| 604 | } | ||
| 605 | |||
| 606 | const to_wake = switch (wake) { | ||
| 607 | .one => 1, | ||
| 608 | .all => wakeable, | ||
| 609 | }; | ||
| 610 | |||
| 611 | // Reserve the amount of waiters to wake by incrementing the signals count. | ||
| 612 | // Release barrier ensures code before the wake() happens before the signal it posted and consumed by the wait() threads. | ||
| 613 | const new_state = state + (one_signal * to_wake); | ||
| 614 | state = cond_state.cmpxchgWeak(state, new_state, .release, .monotonic) orelse { | ||
| 615 | // Wake up the waiting threads we reserved above by changing the epoch value. | ||
| 616 | // NOTE: a waiting thread could miss a wake up if *exactly* ((1<<32)-1) wake()s happen between it observing the epoch and sleeping on it. | ||
| 617 | // This is very unlikely due to how many precise amount of Futex.wake() calls that would be between the waiting thread's potential preemption. | ||
| 618 | // | ||
| 619 | // Release barrier ensures the signal being added to the state happens before the epoch is changed. | ||
| 620 | // If not, the waiting thread could potentially deadlock from missing both the state and epoch change: | ||
| 621 | // | ||
| 622 | // - T2: UPDATE(&epoch, 1) (reordered before the state change) | ||
| 623 | // - T1: e = LOAD(&epoch) | ||
| 624 | // - T1: s = LOAD(&state) | ||
| 625 | // - T2: UPDATE(&state, signal) + FUTEX_WAKE(&epoch) | ||
| 626 | // - T1: s & signals == 0 -> FUTEX_WAIT(&epoch, e) (missed both epoch change and state change) | ||
| 627 | _ = cond_epoch.fetchAdd(1, .release); | ||
| 628 | std.Thread.Futex.wake(cond_epoch, to_wake); | ||
| 629 | return; | ||
| 630 | }; | ||
| 631 | } | ||
| 632 | } | ||
| 633 | |||
| 634 | fn createFile( | ||
| 635 | userdata: ?*anyopaque, | ||
| 636 | dir: Io.Dir, | ||
| 637 | sub_path: []const u8, | ||
| 638 | flags: Io.File.CreateFlags, | ||
| 639 | ) Io.File.OpenError!Io.File { | ||
| 640 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 641 | try pool.checkCancel(); | ||
| 642 | const fs_dir: std.fs.Dir = .{ .fd = dir.handle }; | ||
| 643 | const fs_file = try fs_dir.createFile(sub_path, flags); | ||
| 644 | return .{ .handle = fs_file.handle }; | ||
| 645 | } | ||
| 646 | |||
| 647 | fn fileOpen( | ||
| 648 | userdata: ?*anyopaque, | ||
| 649 | dir: Io.Dir, | ||
| 650 | sub_path: []const u8, | ||
| 651 | flags: Io.File.OpenFlags, | ||
| 652 | ) Io.File.OpenError!Io.File { | ||
| 653 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 654 | try pool.checkCancel(); | ||
| 655 | const fs_dir: std.fs.Dir = .{ .fd = dir.handle }; | ||
| 656 | const fs_file = try fs_dir.openFile(sub_path, flags); | ||
| 657 | return .{ .handle = fs_file.handle }; | ||
| 658 | } | ||
| 659 | |||
| 660 | fn fileClose(userdata: ?*anyopaque, file: Io.File) void { | ||
| 661 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 662 | _ = pool; | ||
| 663 | const fs_file: std.fs.File = .{ .handle = file.handle }; | ||
| 664 | return fs_file.close(); | ||
| 665 | } | ||
| 666 | |||
| 667 | fn fileReadStreaming(userdata: ?*anyopaque, file: Io.File, data: [][]u8) Io.File.ReadStreamingError!usize { | ||
| 668 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 669 | |||
| 670 | if (is_windows) { | ||
| 671 | const DWORD = windows.DWORD; | ||
| 672 | var index: usize = 0; | ||
| 673 | var truncate: usize = 0; | ||
| 674 | var total: usize = 0; | ||
| 675 | while (index < data.len) { | ||
| 676 | try pool.checkCancel(); | ||
| 677 | { | ||
| 678 | const untruncated = data[index]; | ||
| 679 | data[index] = untruncated[truncate..]; | ||
| 680 | defer data[index] = untruncated; | ||
| 681 | const buffer = data[index..]; | ||
| 682 | const want_read_count: DWORD = @min(std.math.maxInt(DWORD), buffer.len); | ||
| 683 | var n: DWORD = undefined; | ||
| 684 | if (windows.kernel32.ReadFile(file.handle, buffer.ptr, want_read_count, &n, null) == 0) { | ||
| 685 | switch (windows.GetLastError()) { | ||
| 686 | .IO_PENDING => unreachable, | ||
| 687 | .OPERATION_ABORTED => continue, | ||
| 688 | .BROKEN_PIPE => return 0, | ||
| 689 | .HANDLE_EOF => return 0, | ||
| 690 | .NETNAME_DELETED => return error.ConnectionResetByPeer, | ||
| 691 | .LOCK_VIOLATION => return error.LockViolation, | ||
| 692 | .ACCESS_DENIED => return error.AccessDenied, | ||
| 693 | .INVALID_HANDLE => return error.NotOpenForReading, | ||
| 694 | else => |err| return windows.unexpectedError(err), | ||
| 695 | } | ||
| 696 | } | ||
| 697 | total += n; | ||
| 698 | truncate += n; | ||
| 699 | } | ||
| 700 | while (index < data.len and truncate >= data[index].len) { | ||
| 701 | truncate -= data[index].len; | ||
| 702 | index += 1; | ||
| 703 | } | ||
| 704 | } | ||
| 705 | return total; | ||
| 706 | } | ||
| 707 | |||
| 708 | var iovecs_buffer: [max_iovecs_len]posix.iovec = undefined; | ||
| 709 | var i: usize = 0; | ||
| 710 | for (data) |buf| { | ||
| 711 | if (iovecs_buffer.len - i == 0) break; | ||
| 712 | if (buf.len != 0) { | ||
| 713 | iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len }; | ||
| 714 | i += 1; | ||
| 715 | } | ||
| 716 | } | ||
| 717 | const dest = iovecs_buffer[0..i]; | ||
| 718 | assert(dest[0].len > 0); | ||
| 719 | |||
| 720 | if (native_os == .wasi and !builtin.link_libc) { | ||
| 721 | try pool.checkCancel(); | ||
| 722 | var nread: usize = undefined; | ||
| 723 | switch (std.os.wasi.fd_read(file.handle, dest.ptr, dest.len, &nread)) { | ||
| 724 | .SUCCESS => return nread, | ||
| 725 | .INTR => unreachable, | ||
| 726 | .INVAL => unreachable, | ||
| 727 | .FAULT => unreachable, | ||
| 728 | .AGAIN => unreachable, // currently not support in WASI | ||
| 729 | .BADF => return error.NotOpenForReading, // can be a race condition | ||
| 730 | .IO => return error.InputOutput, | ||
| 731 | .ISDIR => return error.IsDir, | ||
| 732 | .NOBUFS => return error.SystemResources, | ||
| 733 | .NOMEM => return error.SystemResources, | ||
| 734 | .NOTCONN => return error.SocketNotConnected, | ||
| 735 | .CONNRESET => return error.ConnectionResetByPeer, | ||
| 736 | .TIMEDOUT => return error.ConnectionTimedOut, | ||
| 737 | .NOTCAPABLE => return error.AccessDenied, | ||
| 738 | else => |err| return posix.unexpectedErrno(err), | ||
| 739 | } | ||
| 740 | } | ||
| 741 | |||
| 742 | while (true) { | ||
| 743 | try pool.checkCancel(); | ||
| 744 | const rc = posix.system.readv(file.handle, dest.ptr, dest.len); | ||
| 745 | switch (posix.errno(rc)) { | ||
| 746 | .SUCCESS => return @intCast(rc), | ||
| 747 | .INTR => continue, | ||
| 748 | .INVAL => unreachable, | ||
| 749 | .FAULT => unreachable, | ||
| 750 | .SRCH => return error.ProcessNotFound, | ||
| 751 | .AGAIN => return error.WouldBlock, | ||
| 752 | .BADF => return error.NotOpenForReading, // can be a race condition | ||
| 753 | .IO => return error.InputOutput, | ||
| 754 | .ISDIR => return error.IsDir, | ||
| 755 | .NOBUFS => return error.SystemResources, | ||
| 756 | .NOMEM => return error.SystemResources, | ||
| 757 | .NOTCONN => return error.SocketNotConnected, | ||
| 758 | .CONNRESET => return error.ConnectionResetByPeer, | ||
| 759 | .TIMEDOUT => return error.ConnectionTimedOut, | ||
| 760 | else => |err| return posix.unexpectedErrno(err), | ||
| 761 | } | ||
| 762 | } | ||
| 763 | } | ||
| 764 | |||
| 765 | fn fileReadPositional(userdata: ?*anyopaque, file: Io.File, data: [][]u8, offset: u64) Io.File.ReadPositionalError!usize { | ||
| 766 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 767 | |||
| 768 | const have_pread_but_not_preadv = switch (native_os) { | ||
| 769 | .windows, .macos, .ios, .watchos, .tvos, .visionos, .haiku, .serenity => true, | ||
| 770 | else => false, | ||
| 771 | }; | ||
| 772 | if (have_pread_but_not_preadv) { | ||
| 773 | @compileError("TODO"); | ||
| 774 | } | ||
| 775 | |||
| 776 | if (is_windows) { | ||
| 777 | const DWORD = windows.DWORD; | ||
| 778 | const OVERLAPPED = windows.OVERLAPPED; | ||
| 779 | var index: usize = 0; | ||
| 780 | var truncate: usize = 0; | ||
| 781 | var total: usize = 0; | ||
| 782 | while (true) { | ||
| 783 | try pool.checkCancel(); | ||
| 784 | { | ||
| 785 | const untruncated = data[index]; | ||
| 786 | data[index] = untruncated[truncate..]; | ||
| 787 | defer data[index] = untruncated; | ||
| 788 | const buffer = data[index..]; | ||
| 789 | const want_read_count: DWORD = @min(std.math.maxInt(DWORD), buffer.len); | ||
| 790 | var n: DWORD = undefined; | ||
| 791 | var overlapped_data: OVERLAPPED = undefined; | ||
| 792 | const overlapped: ?*OVERLAPPED = if (offset) |off| blk: { | ||
| 793 | overlapped_data = .{ | ||
| 794 | .Internal = 0, | ||
| 795 | .InternalHigh = 0, | ||
| 796 | .DUMMYUNIONNAME = .{ | ||
| 797 | .DUMMYSTRUCTNAME = .{ | ||
| 798 | .Offset = @as(u32, @truncate(off)), | ||
| 799 | .OffsetHigh = @as(u32, @truncate(off >> 32)), | ||
| 800 | }, | ||
| 801 | }, | ||
| 802 | .hEvent = null, | ||
| 803 | }; | ||
| 804 | break :blk &overlapped_data; | ||
| 805 | } else null; | ||
| 806 | if (windows.kernel32.ReadFile(file.handle, buffer.ptr, want_read_count, &n, overlapped) == 0) { | ||
| 807 | switch (windows.GetLastError()) { | ||
| 808 | .IO_PENDING => unreachable, | ||
| 809 | .OPERATION_ABORTED => continue, | ||
| 810 | .BROKEN_PIPE => return 0, | ||
| 811 | .HANDLE_EOF => return 0, | ||
| 812 | .NETNAME_DELETED => return error.ConnectionResetByPeer, | ||
| 813 | .LOCK_VIOLATION => return error.LockViolation, | ||
| 814 | .ACCESS_DENIED => return error.AccessDenied, | ||
| 815 | .INVALID_HANDLE => return error.NotOpenForReading, | ||
| 816 | else => |err| return windows.unexpectedError(err), | ||
| 817 | } | ||
| 818 | } | ||
| 819 | total += n; | ||
| 820 | truncate += n; | ||
| 821 | } | ||
| 822 | while (index < data.len and truncate >= data[index].len) { | ||
| 823 | truncate -= data[index].len; | ||
| 824 | index += 1; | ||
| 825 | } | ||
| 826 | } | ||
| 827 | return total; | ||
| 828 | } | ||
| 829 | |||
| 830 | var iovecs_buffer: [max_iovecs_len]posix.iovec = undefined; | ||
| 831 | var i: usize = 0; | ||
| 832 | for (data) |buf| { | ||
| 833 | if (iovecs_buffer.len - i == 0) break; | ||
| 834 | if (buf.len != 0) { | ||
| 835 | iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len }; | ||
| 836 | i += 1; | ||
| 837 | } | ||
| 838 | } | ||
| 839 | const dest = iovecs_buffer[0..i]; | ||
| 840 | assert(dest[0].len > 0); | ||
| 841 | |||
| 842 | if (native_os == .wasi and !builtin.link_libc) { | ||
| 843 | try pool.checkCancel(); | ||
| 844 | var nread: usize = undefined; | ||
| 845 | switch (std.os.wasi.fd_pread(file.handle, dest.ptr, dest.len, offset, &nread)) { | ||
| 846 | .SUCCESS => return nread, | ||
| 847 | .INTR => unreachable, | ||
| 848 | .INVAL => unreachable, | ||
| 849 | .FAULT => unreachable, | ||
| 850 | .AGAIN => unreachable, | ||
| 851 | .BADF => return error.NotOpenForReading, // can be a race condition | ||
| 852 | .IO => return error.InputOutput, | ||
| 853 | .ISDIR => return error.IsDir, | ||
| 854 | .NOBUFS => return error.SystemResources, | ||
| 855 | .NOMEM => return error.SystemResources, | ||
| 856 | .NOTCONN => return error.SocketNotConnected, | ||
| 857 | .CONNRESET => return error.ConnectionResetByPeer, | ||
| 858 | .TIMEDOUT => return error.ConnectionTimedOut, | ||
| 859 | .NXIO => return error.Unseekable, | ||
| 860 | .SPIPE => return error.Unseekable, | ||
| 861 | .OVERFLOW => return error.Unseekable, | ||
| 862 | .NOTCAPABLE => return error.AccessDenied, | ||
| 863 | else => |err| return posix.unexpectedErrno(err), | ||
| 864 | } | ||
| 865 | } | ||
| 866 | |||
| 867 | const preadv_sym = if (posix.lfs64_abi) posix.system.preadv64 else posix.system.preadv; | ||
| 868 | while (true) { | ||
| 869 | try pool.checkCancel(); | ||
| 870 | const rc = preadv_sym(file.handle, dest.ptr, dest.len, @bitCast(offset)); | ||
| 871 | switch (posix.errno(rc)) { | ||
| 872 | .SUCCESS => return @bitCast(rc), | ||
| 873 | .INTR => continue, | ||
| 874 | .INVAL => unreachable, | ||
| 875 | .FAULT => unreachable, | ||
| 876 | .SRCH => return error.ProcessNotFound, | ||
| 877 | .AGAIN => return error.WouldBlock, | ||
| 878 | .BADF => return error.NotOpenForReading, // can be a race condition | ||
| 879 | .IO => return error.InputOutput, | ||
| 880 | .ISDIR => return error.IsDir, | ||
| 881 | .NOBUFS => return error.SystemResources, | ||
| 882 | .NOMEM => return error.SystemResources, | ||
| 883 | .NOTCONN => return error.SocketNotConnected, | ||
| 884 | .CONNRESET => return error.ConnectionResetByPeer, | ||
| 885 | .TIMEDOUT => return error.ConnectionTimedOut, | ||
| 886 | .NXIO => return error.Unseekable, | ||
| 887 | .SPIPE => return error.Unseekable, | ||
| 888 | .OVERFLOW => return error.Unseekable, | ||
| 889 | else => |err| return posix.unexpectedErrno(err), | ||
| 890 | } | ||
| 891 | } | ||
| 892 | } | ||
| 893 | |||
| 894 | fn fileSeekBy(userdata: ?*anyopaque, file: Io.File, offset: i64) Io.File.SeekError!void { | ||
| 895 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 896 | try pool.checkCancel(); | ||
| 897 | |||
| 898 | _ = file; | ||
| 899 | _ = offset; | ||
| 900 | @panic("TODO"); | ||
| 901 | } | ||
| 902 | |||
| 903 | fn fileSeekTo(userdata: ?*anyopaque, file: Io.File, offset: u64) Io.File.SeekError!void { | ||
| 904 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 905 | try pool.checkCancel(); | ||
| 906 | |||
| 907 | _ = file; | ||
| 908 | _ = offset; | ||
| 909 | @panic("TODO"); | ||
| 910 | } | ||
| 911 | |||
| 912 | fn pwrite(userdata: ?*anyopaque, file: Io.File, buffer: []const u8, offset: posix.off_t) Io.File.PWriteError!usize { | ||
| 913 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 914 | try pool.checkCancel(); | ||
| 915 | const fs_file: std.fs.File = .{ .handle = file.handle }; | ||
| 916 | return switch (offset) { | ||
| 917 | -1 => fs_file.write(buffer), | ||
| 918 | else => fs_file.pwrite(buffer, @bitCast(offset)), | ||
| 919 | }; | ||
| 920 | } | ||
| 921 | |||
| 922 | fn now(userdata: ?*anyopaque, clockid: posix.clockid_t) Io.ClockGetTimeError!Io.Timestamp { | ||
| 923 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 924 | try pool.checkCancel(); | ||
| 925 | const timespec = try posix.clock_gettime(clockid); | ||
| 926 | return @enumFromInt(@as(i128, timespec.sec) * std.time.ns_per_s + timespec.nsec); | ||
| 927 | } | ||
| 928 | |||
| 929 | fn sleep(userdata: ?*anyopaque, clockid: posix.clockid_t, deadline: Io.Deadline) Io.SleepError!void { | ||
| 930 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 931 | const deadline_nanoseconds: i96 = switch (deadline) { | ||
| 932 | .duration => |duration| duration.nanoseconds, | ||
| 933 | .timestamp => |timestamp| @intFromEnum(timestamp), | ||
| 934 | }; | ||
| 935 | var timespec: posix.timespec = .{ | ||
| 936 | .sec = @intCast(@divFloor(deadline_nanoseconds, std.time.ns_per_s)), | ||
| 937 | .nsec = @intCast(@mod(deadline_nanoseconds, std.time.ns_per_s)), | ||
| 938 | }; | ||
| 939 | while (true) { | ||
| 940 | try pool.checkCancel(); | ||
| 941 | switch (std.os.linux.E.init(std.os.linux.clock_nanosleep(clockid, .{ .ABSTIME = switch (deadline) { | ||
| 942 | .duration => false, | ||
| 943 | .timestamp => true, | ||
| 944 | } }, ×pec, ×pec))) { | ||
| 945 | .SUCCESS => return, | ||
| 946 | .FAULT => unreachable, | ||
| 947 | .INTR => {}, | ||
| 948 | .INVAL => return error.UnsupportedClock, | ||
| 949 | else => |err| return posix.unexpectedErrno(err), | ||
| 950 | } | ||
| 951 | } | ||
| 952 | } | ||
| 953 | |||
| 954 | fn select(userdata: ?*anyopaque, futures: []const *Io.AnyFuture) usize { | ||
| 955 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 956 | _ = pool; | ||
| 957 | |||
| 958 | var reset_event: std.Thread.ResetEvent = .{}; | ||
| 959 | |||
| 960 | for (futures, 0..) |future, i| { | ||
| 961 | const closure: *AsyncClosure = @ptrCast(@alignCast(future)); | ||
| 962 | if (@atomicRmw(?*std.Thread.ResetEvent, &closure.select_condition, .Xchg, &reset_event, .seq_cst) == AsyncClosure.done_reset_event) { | ||
| 963 | for (futures[0..i]) |cleanup_future| { | ||
| 964 | const cleanup_closure: *AsyncClosure = @ptrCast(@alignCast(cleanup_future)); | ||
| 965 | if (@atomicRmw(?*std.Thread.ResetEvent, &cleanup_closure.select_condition, .Xchg, null, .seq_cst) == AsyncClosure.done_reset_event) { | ||
| 966 | cleanup_closure.reset_event.wait(); // Ensure no reference to our stack-allocated reset_event. | ||
| 967 | } | ||
| 968 | } | ||
| 969 | return i; | ||
| 970 | } | ||
| 971 | } | ||
| 972 | |||
| 973 | reset_event.wait(); | ||
| 974 | |||
| 975 | var result: ?usize = null; | ||
| 976 | for (futures, 0..) |future, i| { | ||
| 977 | const closure: *AsyncClosure = @ptrCast(@alignCast(future)); | ||
| 978 | if (@atomicRmw(?*std.Thread.ResetEvent, &closure.select_condition, .Xchg, null, .seq_cst) == AsyncClosure.done_reset_event) { | ||
| 979 | closure.reset_event.wait(); // Ensure no reference to our stack-allocated reset_event. | ||
| 980 | if (result == null) result = i; // In case multiple are ready, return first. | ||
| 981 | } | ||
| 982 | } | ||
| 983 | return result.?; | ||
| 984 | } | ||
| 985 | |||
| 986 | fn listen(userdata: ?*anyopaque, address: Io.net.IpAddress, options: Io.net.ListenOptions) Io.net.ListenError!Io.net.Server { | ||
| 987 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 988 | try pool.checkCancel(); | ||
| 989 | |||
| 990 | const nonblock: u32 = if (options.force_nonblocking) posix.SOCK.NONBLOCK else 0; | ||
| 991 | const sock_flags = posix.SOCK.STREAM | posix.SOCK.CLOEXEC | nonblock; | ||
| 992 | const proto: u32 = posix.IPPROTO.TCP; | ||
| 993 | const family = posixAddressFamily(address); | ||
| 994 | const sockfd = try posix.socket(family, sock_flags, proto); | ||
| 995 | const stream: std.net.Stream = .{ .handle = sockfd }; | ||
| 996 | errdefer stream.close(); | ||
| 997 | |||
| 998 | if (options.reuse_address) { | ||
| 999 | try posix.setsockopt( | ||
| 1000 | sockfd, | ||
| 1001 | posix.SOL.SOCKET, | ||
| 1002 | posix.SO.REUSEADDR, | ||
| 1003 | &std.mem.toBytes(@as(c_int, 1)), | ||
| 1004 | ); | ||
| 1005 | if (@hasDecl(posix.SO, "REUSEPORT") and family != posix.AF.UNIX) { | ||
| 1006 | try posix.setsockopt( | ||
| 1007 | sockfd, | ||
| 1008 | posix.SOL.SOCKET, | ||
| 1009 | posix.SO.REUSEPORT, | ||
| 1010 | &std.mem.toBytes(@as(c_int, 1)), | ||
| 1011 | ); | ||
| 1012 | } | ||
| 1013 | } | ||
| 1014 | |||
| 1015 | var storage: PosixAddress = undefined; | ||
| 1016 | var socklen = addressToPosix(address, &storage); | ||
| 1017 | try posix.bind(sockfd, &storage.any, socklen); | ||
| 1018 | try posix.listen(sockfd, options.kernel_backlog); | ||
| 1019 | try posix.getsockname(sockfd, &storage.any, &socklen); | ||
| 1020 | return .{ | ||
| 1021 | .listen_address = addressFromPosix(&storage), | ||
| 1022 | .stream = .{ .handle = stream.handle }, | ||
| 1023 | }; | ||
| 1024 | } | ||
| 1025 | |||
| 1026 | fn accept(userdata: ?*anyopaque, server: *Io.net.Server) Io.net.Server.AcceptError!Io.net.Server.Connection { | ||
| 1027 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 1028 | try pool.checkCancel(); | ||
| 1029 | |||
| 1030 | var storage: PosixAddress = undefined; | ||
| 1031 | var addr_len: posix.socklen_t = @sizeOf(PosixAddress); | ||
| 1032 | const fd = try posix.accept(server.stream.handle, &storage.any, &addr_len, posix.SOCK.CLOEXEC); | ||
| 1033 | return .{ | ||
| 1034 | .stream = .{ .handle = fd }, | ||
| 1035 | .address = addressFromPosix(&storage), | ||
| 1036 | }; | ||
| 1037 | } | ||
| 1038 | |||
| 1039 | fn netReadPosix(userdata: ?*anyopaque, stream: Io.net.Stream, data: [][]u8) Io.net.Stream.Reader.Error!usize { | ||
| 1040 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 1041 | try pool.checkCancel(); | ||
| 1042 | |||
| 1043 | var iovecs_buffer: [max_iovecs_len]posix.iovec = undefined; | ||
| 1044 | var i: usize = 0; | ||
| 1045 | for (data) |buf| { | ||
| 1046 | if (iovecs_buffer.len - i == 0) break; | ||
| 1047 | if (buf.len != 0) { | ||
| 1048 | iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len }; | ||
| 1049 | i += 1; | ||
| 1050 | } | ||
| 1051 | } | ||
| 1052 | const dest = iovecs_buffer[0..i]; | ||
| 1053 | assert(dest[0].len > 0); | ||
| 1054 | const n = try posix.readv(stream.handle, dest); | ||
| 1055 | if (n == 0) return error.EndOfStream; | ||
| 1056 | return n; | ||
| 1057 | } | ||
| 1058 | |||
| 1059 | fn netWritePosix( | ||
| 1060 | userdata: ?*anyopaque, | ||
| 1061 | stream: Io.net.Stream, | ||
| 1062 | header: []const u8, | ||
| 1063 | data: []const []const u8, | ||
| 1064 | splat: usize, | ||
| 1065 | ) Io.net.Stream.Writer.Error!usize { | ||
| 1066 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 1067 | try pool.checkCancel(); | ||
| 1068 | |||
| 1069 | var iovecs: [max_iovecs_len]posix.iovec_const = undefined; | ||
| 1070 | var msg: posix.msghdr_const = .{ | ||
| 1071 | .name = null, | ||
| 1072 | .namelen = 0, | ||
| 1073 | .iov = &iovecs, | ||
| 1074 | .iovlen = 0, | ||
| 1075 | .control = null, | ||
| 1076 | .controllen = 0, | ||
| 1077 | .flags = 0, | ||
| 1078 | }; | ||
| 1079 | addBuf(&iovecs, &msg.iovlen, header); | ||
| 1080 | for (data[0 .. data.len - 1]) |bytes| addBuf(&iovecs, &msg.iovlen, bytes); | ||
| 1081 | const pattern = data[data.len - 1]; | ||
| 1082 | if (iovecs.len - msg.iovlen != 0) switch (splat) { | ||
| 1083 | 0 => {}, | ||
| 1084 | 1 => addBuf(&iovecs, &msg.iovlen, pattern), | ||
| 1085 | else => switch (pattern.len) { | ||
| 1086 | 0 => {}, | ||
| 1087 | 1 => { | ||
| 1088 | var backup_buffer: [splat_buffer_size]u8 = undefined; | ||
| 1089 | const splat_buffer = &backup_buffer; | ||
| 1090 | const memset_len = @min(splat_buffer.len, splat); | ||
| 1091 | const buf = splat_buffer[0..memset_len]; | ||
| 1092 | @memset(buf, pattern[0]); | ||
| 1093 | addBuf(&iovecs, &msg.iovlen, buf); | ||
| 1094 | var remaining_splat = splat - buf.len; | ||
| 1095 | while (remaining_splat > splat_buffer.len and iovecs.len - msg.iovlen != 0) { | ||
| 1096 | assert(buf.len == splat_buffer.len); | ||
| 1097 | addBuf(&iovecs, &msg.iovlen, splat_buffer); | ||
| 1098 | remaining_splat -= splat_buffer.len; | ||
| 1099 | } | ||
| 1100 | addBuf(&iovecs, &msg.iovlen, splat_buffer[0..remaining_splat]); | ||
| 1101 | }, | ||
| 1102 | else => for (0..@min(splat, iovecs.len - msg.iovlen)) |_| { | ||
| 1103 | addBuf(&iovecs, &msg.iovlen, pattern); | ||
| 1104 | }, | ||
| 1105 | }, | ||
| 1106 | }; | ||
| 1107 | const flags = posix.MSG.NOSIGNAL; | ||
| 1108 | return posix.sendmsg(stream.handle, &msg, flags); | ||
| 1109 | } | ||
| 1110 | |||
| 1111 | fn addBuf(v: []posix.iovec_const, i: *@FieldType(posix.msghdr_const, "iovlen"), bytes: []const u8) void { | ||
| 1112 | // OS checks ptr addr before length so zero length vectors must be omitted. | ||
| 1113 | if (bytes.len == 0) return; | ||
| 1114 | if (v.len - i.* == 0) return; | ||
| 1115 | v[i.*] = .{ .base = bytes.ptr, .len = bytes.len }; | ||
| 1116 | i.* += 1; | ||
| 1117 | } | ||
| 1118 | |||
| 1119 | fn netClose(userdata: ?*anyopaque, stream: Io.net.Stream) void { | ||
| 1120 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 1121 | _ = pool; | ||
| 1122 | const net_stream: std.net.Stream = .{ .handle = stream.handle }; | ||
| 1123 | return net_stream.close(); | ||
| 1124 | } | ||
| 1125 | |||
| 1126 | fn netInterfaceIndex(userdata: ?*anyopaque, name: []const u8) Io.net.InterfaceIndexError!u32 { | ||
| 1127 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 1128 | try pool.checkCancel(); | ||
| 1129 | |||
| 1130 | if (native_os == .linux) { | ||
| 1131 | if (name.len >= posix.IFNAMESIZE) return error.InterfaceNotFound; | ||
| 1132 | var ifr: posix.ifreq = undefined; | ||
| 1133 | @memcpy(ifr.ifrn.name[0..name.len], name); | ||
| 1134 | ifr.ifrn.name[name.len] = 0; | ||
| 1135 | |||
| 1136 | const rc = posix.system.socket(posix.AF.UNIX, posix.SOCK.DGRAM | posix.SOCK.CLOEXEC, 0); | ||
| 1137 | const sock_fd: posix.fd_t = switch (posix.errno(rc)) { | ||
| 1138 | .SUCCESS => @intCast(rc), | ||
| 1139 | .ACCES => return error.AccessDenied, | ||
| 1140 | .MFILE => return error.SystemResources, | ||
| 1141 | .NFILE => return error.SystemResources, | ||
| 1142 | .NOBUFS => return error.SystemResources, | ||
| 1143 | .NOMEM => return error.SystemResources, | ||
| 1144 | else => |err| return posix.unexpectedErrno(err), | ||
| 1145 | }; | ||
| 1146 | defer posix.close(sock_fd); | ||
| 1147 | |||
| 1148 | while (true) { | ||
| 1149 | try pool.checkCancel(); | ||
| 1150 | switch (posix.errno(posix.system.ioctl(sock_fd, posix.SIOCGIFINDEX, @intFromPtr(&ifr)))) { | ||
| 1151 | .SUCCESS => return @bitCast(ifr.ifru.ivalue), | ||
| 1152 | .INVAL => |err| return badErrno(err), // Bad parameters. | ||
| 1153 | .NOTTY => |err| return badErrno(err), | ||
| 1154 | .NXIO => |err| return badErrno(err), | ||
| 1155 | .BADF => |err| return badErrno(err), // Always a race condition. | ||
| 1156 | .FAULT => |err| return badErrno(err), // Bad pointer parameter. | ||
| 1157 | .INTR => continue, | ||
| 1158 | .IO => |err| return badErrno(err), // sock_fd is not a file descriptor | ||
| 1159 | .NODEV => return error.InterfaceNotFound, | ||
| 1160 | else => |err| return posix.unexpectedErrno(err), | ||
| 1161 | } | ||
| 1162 | } | ||
| 1163 | } | ||
| 1164 | |||
| 1165 | if (native_os.isDarwin()) { | ||
| 1166 | if (name.len >= posix.IFNAMESIZE) return error.InterfaceNotFound; | ||
| 1167 | var if_name: [posix.IFNAMESIZE:0]u8 = undefined; | ||
| 1168 | @memcpy(if_name[0..name.len], name); | ||
| 1169 | if_name[name.len] = 0; | ||
| 1170 | const if_slice = if_name[0..name.len :0]; | ||
| 1171 | const index = std.c.if_nametoindex(if_slice); | ||
| 1172 | if (index == 0) return error.InterfaceNotFound; | ||
| 1173 | return @bitCast(index); | ||
| 1174 | } | ||
| 1175 | |||
| 1176 | if (native_os == .windows) { | ||
| 1177 | if (name.len >= posix.IFNAMESIZE) return error.InterfaceNotFound; | ||
| 1178 | var interface_name: [posix.IFNAMESIZE:0]u8 = undefined; | ||
| 1179 | @memcpy(interface_name[0..name.len], name); | ||
| 1180 | interface_name[name.len] = 0; | ||
| 1181 | const index = std.os.windows.ws2_32.if_nametoindex(@as([*:0]const u8, &interface_name)); | ||
| 1182 | if (index == 0) return error.InterfaceNotFound; | ||
| 1183 | return index; | ||
| 1184 | } | ||
| 1185 | |||
| 1186 | @compileError("std.net.if_nametoindex unimplemented for this OS"); | ||
| 1187 | } | ||
| 1188 | |||
| 1189 | const PosixAddress = extern union { | ||
| 1190 | any: posix.sockaddr, | ||
| 1191 | in: posix.sockaddr.in, | ||
| 1192 | in6: posix.sockaddr.in6, | ||
| 1193 | }; | ||
| 1194 | |||
| 1195 | fn posixAddressFamily(a: Io.net.IpAddress) posix.sa_family_t { | ||
| 1196 | return switch (a) { | ||
| 1197 | .ip4 => posix.AF.INET, | ||
| 1198 | .ip6 => posix.AF.INET6, | ||
| 1199 | }; | ||
| 1200 | } | ||
| 1201 | |||
| 1202 | fn addressFromPosix(posix_address: *PosixAddress) Io.net.IpAddress { | ||
| 1203 | return switch (posix_address.any.family) { | ||
| 1204 | posix.AF.INET => .{ .ip4 = address4FromPosix(&posix_address.in) }, | ||
| 1205 | posix.AF.INET6 => .{ .ip6 = address6FromPosix(&posix_address.in6) }, | ||
| 1206 | else => unreachable, | ||
| 1207 | }; | ||
| 1208 | } | ||
| 1209 | |||
| 1210 | fn addressToPosix(a: Io.net.IpAddress, storage: *PosixAddress) posix.socklen_t { | ||
| 1211 | return switch (a) { | ||
| 1212 | .ip4 => |ip4| { | ||
| 1213 | storage.in = address4ToPosix(ip4); | ||
| 1214 | return @sizeOf(posix.sockaddr.in); | ||
| 1215 | }, | ||
| 1216 | .ip6 => |ip6| { | ||
| 1217 | storage.in6 = address6ToPosix(ip6); | ||
| 1218 | return @sizeOf(posix.sockaddr.in6); | ||
| 1219 | }, | ||
| 1220 | }; | ||
| 1221 | } | ||
| 1222 | |||
| 1223 | fn address4FromPosix(in: *posix.sockaddr.in) Io.net.Ip4Address { | ||
| 1224 | return .{ | ||
| 1225 | .port = std.mem.bigToNative(u16, in.port), | ||
| 1226 | .bytes = @bitCast(in.addr), | ||
| 1227 | }; | ||
| 1228 | } | ||
| 1229 | |||
| 1230 | fn address6FromPosix(in6: *posix.sockaddr.in6) Io.net.Ip6Address { | ||
| 1231 | return .{ | ||
| 1232 | .port = std.mem.bigToNative(u16, in6.port), | ||
| 1233 | .bytes = in6.addr, | ||
| 1234 | .flowinfo = in6.flowinfo, | ||
| 1235 | .scope_id = in6.scope_id, | ||
| 1236 | }; | ||
| 1237 | } | ||
| 1238 | |||
| 1239 | fn address4ToPosix(a: Io.net.Ip4Address) posix.sockaddr.in { | ||
| 1240 | return .{ | ||
| 1241 | .port = std.mem.nativeToBig(u16, a.port), | ||
| 1242 | .addr = @bitCast(a.bytes), | ||
| 1243 | }; | ||
| 1244 | } | ||
| 1245 | |||
| 1246 | fn address6ToPosix(a: Io.net.Ip6Address) posix.sockaddr.in6 { | ||
| 1247 | return .{ | ||
| 1248 | .port = std.mem.nativeToBig(u16, a.port), | ||
| 1249 | .flowinfo = a.flowinfo, | ||
| 1250 | .addr = a.bytes, | ||
| 1251 | .scope_id = a.scope_id, | ||
| 1252 | }; | ||
| 1253 | } | ||
| 1254 | |||
| 1255 | fn badErrno(err: posix.E) Io.UnexpectedError { | ||
| 1256 | switch (builtin.mode) { | ||
| 1257 | .Debug => std.debug.panic("programmer bug caused syscall error: {t}", .{err}), | ||
| 1258 | else => return error.Unexpected, | ||
| 1259 | } | ||
| 1260 | } | ||
lib/std/Io/Threaded.zig created+1260
| ... | @@ -0,0 +1,1260 @@ | ||
| 1 | const Pool = @This(); | ||
| 2 | |||
| 3 | const builtin = @import("builtin"); | ||
| 4 | const native_os = builtin.os.tag; | ||
| 5 | const is_windows = native_os == .windows; | ||
| 6 | const windows = std.os.windows; | ||
| 7 | |||
| 8 | const std = @import("../std.zig"); | ||
| 9 | const Allocator = std.mem.Allocator; | ||
| 10 | const assert = std.debug.assert; | ||
| 11 | const WaitGroup = std.Thread.WaitGroup; | ||
| 12 | const posix = std.posix; | ||
| 13 | const Io = std.Io; | ||
| 14 | |||
| 15 | /// Thread-safe. | ||
| 16 | allocator: Allocator, | ||
| 17 | mutex: std.Thread.Mutex = .{}, | ||
| 18 | cond: std.Thread.Condition = .{}, | ||
| 19 | run_queue: std.SinglyLinkedList = .{}, | ||
| 20 | join_requested: bool = false, | ||
| 21 | threads: std.ArrayListUnmanaged(std.Thread), | ||
| 22 | stack_size: usize, | ||
| 23 | cpu_count: std.Thread.CpuCountError!usize, | ||
| 24 | parallel_count: usize, | ||
| 25 | |||
| 26 | threadlocal var current_closure: ?*AsyncClosure = null; | ||
| 27 | |||
| 28 | const max_iovecs_len = 8; | ||
| 29 | const splat_buffer_size = 64; | ||
| 30 | |||
| 31 | comptime { | ||
| 32 | assert(max_iovecs_len <= posix.IOV_MAX); | ||
| 33 | } | ||
| 34 | |||
| 35 | pub const Runnable = struct { | ||
| 36 | start: Start, | ||
| 37 | node: std.SinglyLinkedList.Node = .{}, | ||
| 38 | is_parallel: bool, | ||
| 39 | |||
| 40 | pub const Start = *const fn (*Runnable) void; | ||
| 41 | }; | ||
| 42 | |||
| 43 | pub const InitError = std.Thread.CpuCountError || Allocator.Error; | ||
| 44 | |||
| 45 | pub fn init(gpa: Allocator) Pool { | ||
| 46 | var pool: Pool = .{ | ||
| 47 | .allocator = gpa, | ||
| 48 | .threads = .empty, | ||
| 49 | .stack_size = std.Thread.SpawnConfig.default_stack_size, | ||
| 50 | .cpu_count = std.Thread.getCpuCount(), | ||
| 51 | .parallel_count = 0, | ||
| 52 | }; | ||
| 53 | if (pool.cpu_count) |n| { | ||
| 54 | pool.threads.ensureTotalCapacityPrecise(gpa, n - 1) catch {}; | ||
| 55 | } else |_| {} | ||
| 56 | return pool; | ||
| 57 | } | ||
| 58 | |||
| 59 | pub fn deinit(pool: *Pool) void { | ||
| 60 | const gpa = pool.allocator; | ||
| 61 | pool.join(); | ||
| 62 | pool.threads.deinit(gpa); | ||
| 63 | pool.* = undefined; | ||
| 64 | } | ||
| 65 | |||
| 66 | fn join(pool: *Pool) void { | ||
| 67 | if (builtin.single_threaded) return; | ||
| 68 | { | ||
| 69 | pool.mutex.lock(); | ||
| 70 | defer pool.mutex.unlock(); | ||
| 71 | pool.join_requested = true; | ||
| 72 | } | ||
| 73 | pool.cond.broadcast(); | ||
| 74 | for (pool.threads.items) |thread| thread.join(); | ||
| 75 | } | ||
| 76 | |||
| 77 | fn worker(pool: *Pool) void { | ||
| 78 | pool.mutex.lock(); | ||
| 79 | defer pool.mutex.unlock(); | ||
| 80 | |||
| 81 | while (true) { | ||
| 82 | while (pool.run_queue.popFirst()) |run_node| { | ||
| 83 | pool.mutex.unlock(); | ||
| 84 | const runnable: *Runnable = @fieldParentPtr("node", run_node); | ||
| 85 | runnable.start(runnable); | ||
| 86 | pool.mutex.lock(); | ||
| 87 | if (runnable.is_parallel) { | ||
| 88 | // TODO also pop thread and join sometimes | ||
| 89 | pool.parallel_count -= 1; | ||
| 90 | } | ||
| 91 | } | ||
| 92 | if (pool.join_requested) break; | ||
| 93 | pool.cond.wait(&pool.mutex); | ||
| 94 | } | ||
| 95 | } | ||
| 96 | |||
| 97 | pub fn io(pool: *Pool) Io { | ||
| 98 | return .{ | ||
| 99 | .userdata = pool, | ||
| 100 | .vtable = &.{ | ||
| 101 | .async = async, | ||
| 102 | .asyncConcurrent = asyncConcurrent, | ||
| 103 | .await = await, | ||
| 104 | .asyncDetached = asyncDetached, | ||
| 105 | .cancel = cancel, | ||
| 106 | .cancelRequested = cancelRequested, | ||
| 107 | .select = select, | ||
| 108 | |||
| 109 | .mutexLock = mutexLock, | ||
| 110 | .mutexUnlock = mutexUnlock, | ||
| 111 | |||
| 112 | .conditionWait = conditionWait, | ||
| 113 | .conditionWake = conditionWake, | ||
| 114 | |||
| 115 | .createFile = createFile, | ||
| 116 | .fileOpen = fileOpen, | ||
| 117 | .fileClose = fileClose, | ||
| 118 | .pwrite = pwrite, | ||
| 119 | .fileReadStreaming = fileReadStreaming, | ||
| 120 | .fileReadPositional = fileReadPositional, | ||
| 121 | .fileSeekBy = fileSeekBy, | ||
| 122 | .fileSeekTo = fileSeekTo, | ||
| 123 | |||
| 124 | .now = now, | ||
| 125 | .sleep = sleep, | ||
| 126 | |||
| 127 | .listen = listen, | ||
| 128 | .accept = accept, | ||
| 129 | .netRead = switch (builtin.os.tag) { | ||
| 130 | .windows => @panic("TODO"), | ||
| 131 | else => netReadPosix, | ||
| 132 | }, | ||
| 133 | .netWrite = switch (builtin.os.tag) { | ||
| 134 | .windows => @panic("TODO"), | ||
| 135 | else => netWritePosix, | ||
| 136 | }, | ||
| 137 | .netClose = netClose, | ||
| 138 | .netInterfaceIndex = netInterfaceIndex, | ||
| 139 | }, | ||
| 140 | }; | ||
| 141 | } | ||
| 142 | |||
| 143 | const AsyncClosure = struct { | ||
| 144 | func: *const fn (context: *anyopaque, result: *anyopaque) void, | ||
| 145 | runnable: Runnable, | ||
| 146 | reset_event: std.Thread.ResetEvent, | ||
| 147 | select_condition: ?*std.Thread.ResetEvent, | ||
| 148 | cancel_tid: std.Thread.Id, | ||
| 149 | context_offset: usize, | ||
| 150 | result_offset: usize, | ||
| 151 | |||
| 152 | const done_reset_event: *std.Thread.ResetEvent = @ptrFromInt(@alignOf(std.Thread.ResetEvent)); | ||
| 153 | |||
| 154 | const canceling_tid: std.Thread.Id = switch (@typeInfo(std.Thread.Id)) { | ||
| 155 | .int => |int_info| switch (int_info.signedness) { | ||
| 156 | .signed => -1, | ||
| 157 | .unsigned => std.math.maxInt(std.Thread.Id), | ||
| 158 | }, | ||
| 159 | .pointer => @ptrFromInt(std.math.maxInt(usize)), | ||
| 160 | else => @compileError("unsupported std.Thread.Id: " ++ @typeName(std.Thread.Id)), | ||
| 161 | }; | ||
| 162 | |||
| 163 | fn start(runnable: *Runnable) void { | ||
| 164 | const closure: *AsyncClosure = @alignCast(@fieldParentPtr("runnable", runnable)); | ||
| 165 | const tid = std.Thread.getCurrentId(); | ||
| 166 | if (@cmpxchgStrong( | ||
| 167 | std.Thread.Id, | ||
| 168 | &closure.cancel_tid, | ||
| 169 | 0, | ||
| 170 | tid, | ||
| 171 | .acq_rel, | ||
| 172 | .acquire, | ||
| 173 | )) |cancel_tid| { | ||
| 174 | assert(cancel_tid == canceling_tid); | ||
| 175 | closure.reset_event.set(); | ||
| 176 | return; | ||
| 177 | } | ||
| 178 | current_closure = closure; | ||
| 179 | closure.func(closure.contextPointer(), closure.resultPointer()); | ||
| 180 | current_closure = null; | ||
| 181 | if (@cmpxchgStrong( | ||
| 182 | std.Thread.Id, | ||
| 183 | &closure.cancel_tid, | ||
| 184 | tid, | ||
| 185 | 0, | ||
| 186 | .acq_rel, | ||
| 187 | .acquire, | ||
| 188 | )) |cancel_tid| assert(cancel_tid == canceling_tid); | ||
| 189 | |||
| 190 | if (@atomicRmw( | ||
| 191 | ?*std.Thread.ResetEvent, | ||
| 192 | &closure.select_condition, | ||
| 193 | .Xchg, | ||
| 194 | done_reset_event, | ||
| 195 | .release, | ||
| 196 | )) |select_reset| { | ||
| 197 | assert(select_reset != done_reset_event); | ||
| 198 | select_reset.set(); | ||
| 199 | } | ||
| 200 | closure.reset_event.set(); | ||
| 201 | } | ||
| 202 | |||
| 203 | fn contextOffset(context_alignment: std.mem.Alignment) usize { | ||
| 204 | return context_alignment.forward(@sizeOf(AsyncClosure)); | ||
| 205 | } | ||
| 206 | |||
| 207 | fn resultOffset( | ||
| 208 | context_alignment: std.mem.Alignment, | ||
| 209 | context_len: usize, | ||
| 210 | result_alignment: std.mem.Alignment, | ||
| 211 | ) usize { | ||
| 212 | return result_alignment.forward(contextOffset(context_alignment) + context_len); | ||
| 213 | } | ||
| 214 | |||
| 215 | fn resultPointer(closure: *AsyncClosure) [*]u8 { | ||
| 216 | const base: [*]u8 = @ptrCast(closure); | ||
| 217 | return base + closure.result_offset; | ||
| 218 | } | ||
| 219 | |||
| 220 | fn contextPointer(closure: *AsyncClosure) [*]u8 { | ||
| 221 | const base: [*]u8 = @ptrCast(closure); | ||
| 222 | return base + closure.context_offset; | ||
| 223 | } | ||
| 224 | |||
| 225 | fn waitAndFree(closure: *AsyncClosure, gpa: Allocator, result: []u8) void { | ||
| 226 | closure.reset_event.wait(); | ||
| 227 | @memcpy(result, closure.resultPointer()[0..result.len]); | ||
| 228 | free(closure, gpa, result.len); | ||
| 229 | } | ||
| 230 | |||
| 231 | fn free(closure: *AsyncClosure, gpa: Allocator, result_len: usize) void { | ||
| 232 | const base: [*]align(@alignOf(AsyncClosure)) u8 = @ptrCast(closure); | ||
| 233 | gpa.free(base[0 .. closure.result_offset + result_len]); | ||
| 234 | } | ||
| 235 | }; | ||
| 236 | |||
| 237 | fn async( | ||
| 238 | userdata: ?*anyopaque, | ||
| 239 | result: []u8, | ||
| 240 | result_alignment: std.mem.Alignment, | ||
| 241 | context: []const u8, | ||
| 242 | context_alignment: std.mem.Alignment, | ||
| 243 | start: *const fn (context: *const anyopaque, result: *anyopaque) void, | ||
| 244 | ) ?*Io.AnyFuture { | ||
| 245 | if (builtin.single_threaded) { | ||
| 246 | start(context.ptr, result.ptr); | ||
| 247 | return null; | ||
| 248 | } | ||
| 249 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 250 | const cpu_count = pool.cpu_count catch { | ||
| 251 | return asyncConcurrent(userdata, result.len, result_alignment, context, context_alignment, start) catch { | ||
| 252 | start(context.ptr, result.ptr); | ||
| 253 | return null; | ||
| 254 | }; | ||
| 255 | }; | ||
| 256 | const gpa = pool.allocator; | ||
| 257 | const context_offset = context_alignment.forward(@sizeOf(AsyncClosure)); | ||
| 258 | const result_offset = result_alignment.forward(context_offset + context.len); | ||
| 259 | const n = result_offset + result.len; | ||
| 260 | const closure: *AsyncClosure = @ptrCast(@alignCast(gpa.alignedAlloc(u8, .of(AsyncClosure), n) catch { | ||
| 261 | start(context.ptr, result.ptr); | ||
| 262 | return null; | ||
| 263 | })); | ||
| 264 | |||
| 265 | closure.* = .{ | ||
| 266 | .func = start, | ||
| 267 | .context_offset = context_offset, | ||
| 268 | .result_offset = result_offset, | ||
| 269 | .reset_event = .{}, | ||
| 270 | .cancel_tid = 0, | ||
| 271 | .select_condition = null, | ||
| 272 | .runnable = .{ | ||
| 273 | .start = AsyncClosure.start, | ||
| 274 | .is_parallel = false, | ||
| 275 | }, | ||
| 276 | }; | ||
| 277 | |||
| 278 | @memcpy(closure.contextPointer()[0..context.len], context); | ||
| 279 | |||
| 280 | pool.mutex.lock(); | ||
| 281 | |||
| 282 | const thread_capacity = cpu_count - 1 + pool.parallel_count; | ||
| 283 | |||
| 284 | pool.threads.ensureTotalCapacityPrecise(gpa, thread_capacity) catch { | ||
| 285 | pool.mutex.unlock(); | ||
| 286 | closure.free(gpa, result.len); | ||
| 287 | start(context.ptr, result.ptr); | ||
| 288 | return null; | ||
| 289 | }; | ||
| 290 | |||
| 291 | pool.run_queue.prepend(&closure.runnable.node); | ||
| 292 | |||
| 293 | if (pool.threads.items.len < thread_capacity) { | ||
| 294 | const thread = std.Thread.spawn(.{ .stack_size = pool.stack_size }, worker, .{pool}) catch { | ||
| 295 | if (pool.threads.items.len == 0) { | ||
| 296 | assert(pool.run_queue.popFirst() == &closure.runnable.node); | ||
| 297 | pool.mutex.unlock(); | ||
| 298 | closure.free(gpa, result.len); | ||
| 299 | start(context.ptr, result.ptr); | ||
| 300 | return null; | ||
| 301 | } | ||
| 302 | // Rely on other workers to do it. | ||
| 303 | pool.mutex.unlock(); | ||
| 304 | pool.cond.signal(); | ||
| 305 | return @ptrCast(closure); | ||
| 306 | }; | ||
| 307 | pool.threads.appendAssumeCapacity(thread); | ||
| 308 | } | ||
| 309 | |||
| 310 | pool.mutex.unlock(); | ||
| 311 | pool.cond.signal(); | ||
| 312 | return @ptrCast(closure); | ||
| 313 | } | ||
| 314 | |||
| 315 | fn asyncConcurrent( | ||
| 316 | userdata: ?*anyopaque, | ||
| 317 | result_len: usize, | ||
| 318 | result_alignment: std.mem.Alignment, | ||
| 319 | context: []const u8, | ||
| 320 | context_alignment: std.mem.Alignment, | ||
| 321 | start: *const fn (context: *const anyopaque, result: *anyopaque) void, | ||
| 322 | ) error{OutOfMemory}!*Io.AnyFuture { | ||
| 323 | if (builtin.single_threaded) unreachable; | ||
| 324 | |||
| 325 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 326 | const cpu_count = pool.cpu_count catch 1; | ||
| 327 | const gpa = pool.allocator; | ||
| 328 | const context_offset = context_alignment.forward(@sizeOf(AsyncClosure)); | ||
| 329 | const result_offset = result_alignment.forward(context_offset + context.len); | ||
| 330 | const n = result_offset + result_len; | ||
| 331 | const closure: *AsyncClosure = @ptrCast(@alignCast(try gpa.alignedAlloc(u8, .of(AsyncClosure), n))); | ||
| 332 | |||
| 333 | closure.* = .{ | ||
| 334 | .func = start, | ||
| 335 | .context_offset = context_offset, | ||
| 336 | .result_offset = result_offset, | ||
| 337 | .reset_event = .{}, | ||
| 338 | .cancel_tid = 0, | ||
| 339 | .select_condition = null, | ||
| 340 | .runnable = .{ | ||
| 341 | .start = AsyncClosure.start, | ||
| 342 | .is_parallel = true, | ||
| 343 | }, | ||
| 344 | }; | ||
| 345 | @memcpy(closure.contextPointer()[0..context.len], context); | ||
| 346 | |||
| 347 | pool.mutex.lock(); | ||
| 348 | |||
| 349 | pool.parallel_count += 1; | ||
| 350 | const thread_capacity = cpu_count - 1 + pool.parallel_count; | ||
| 351 | |||
| 352 | pool.threads.ensureTotalCapacity(gpa, thread_capacity) catch { | ||
| 353 | pool.mutex.unlock(); | ||
| 354 | closure.free(gpa, result_len); | ||
| 355 | return error.OutOfMemory; | ||
| 356 | }; | ||
| 357 | |||
| 358 | pool.run_queue.prepend(&closure.runnable.node); | ||
| 359 | |||
| 360 | if (pool.threads.items.len < thread_capacity) { | ||
| 361 | const thread = std.Thread.spawn(.{ .stack_size = pool.stack_size }, worker, .{pool}) catch { | ||
| 362 | assert(pool.run_queue.popFirst() == &closure.runnable.node); | ||
| 363 | pool.mutex.unlock(); | ||
| 364 | closure.free(gpa, result_len); | ||
| 365 | return error.OutOfMemory; | ||
| 366 | }; | ||
| 367 | pool.threads.appendAssumeCapacity(thread); | ||
| 368 | } | ||
| 369 | |||
| 370 | pool.mutex.unlock(); | ||
| 371 | pool.cond.signal(); | ||
| 372 | return @ptrCast(closure); | ||
| 373 | } | ||
| 374 | |||
| 375 | const DetachedClosure = struct { | ||
| 376 | pool: *Pool, | ||
| 377 | func: *const fn (context: *anyopaque) void, | ||
| 378 | runnable: Runnable, | ||
| 379 | context_alignment: std.mem.Alignment, | ||
| 380 | context_len: usize, | ||
| 381 | |||
| 382 | fn start(runnable: *Runnable) void { | ||
| 383 | const closure: *DetachedClosure = @alignCast(@fieldParentPtr("runnable", runnable)); | ||
| 384 | closure.func(closure.contextPointer()); | ||
| 385 | const gpa = closure.pool.allocator; | ||
| 386 | free(closure, gpa); | ||
| 387 | } | ||
| 388 | |||
| 389 | fn free(closure: *DetachedClosure, gpa: Allocator) void { | ||
| 390 | const base: [*]align(@alignOf(DetachedClosure)) u8 = @ptrCast(closure); | ||
| 391 | gpa.free(base[0..contextEnd(closure.context_alignment, closure.context_len)]); | ||
| 392 | } | ||
| 393 | |||
| 394 | fn contextOffset(context_alignment: std.mem.Alignment) usize { | ||
| 395 | return context_alignment.forward(@sizeOf(DetachedClosure)); | ||
| 396 | } | ||
| 397 | |||
| 398 | fn contextEnd(context_alignment: std.mem.Alignment, context_len: usize) usize { | ||
| 399 | return contextOffset(context_alignment) + context_len; | ||
| 400 | } | ||
| 401 | |||
| 402 | fn contextPointer(closure: *DetachedClosure) [*]u8 { | ||
| 403 | const base: [*]u8 = @ptrCast(closure); | ||
| 404 | return base + contextOffset(closure.context_alignment); | ||
| 405 | } | ||
| 406 | }; | ||
| 407 | |||
| 408 | fn asyncDetached( | ||
| 409 | userdata: ?*anyopaque, | ||
| 410 | context: []const u8, | ||
| 411 | context_alignment: std.mem.Alignment, | ||
| 412 | start: *const fn (context: *const anyopaque) void, | ||
| 413 | ) void { | ||
| 414 | if (builtin.single_threaded) return start(context.ptr); | ||
| 415 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 416 | const cpu_count = pool.cpu_count catch 1; | ||
| 417 | const gpa = pool.allocator; | ||
| 418 | const n = DetachedClosure.contextEnd(context_alignment, context.len); | ||
| 419 | const closure: *DetachedClosure = @ptrCast(@alignCast(gpa.alignedAlloc(u8, .of(DetachedClosure), n) catch { | ||
| 420 | return start(context.ptr); | ||
| 421 | })); | ||
| 422 | closure.* = .{ | ||
| 423 | .pool = pool, | ||
| 424 | .func = start, | ||
| 425 | .context_alignment = context_alignment, | ||
| 426 | .context_len = context.len, | ||
| 427 | .runnable = .{ | ||
| 428 | .start = DetachedClosure.start, | ||
| 429 | .is_parallel = false, | ||
| 430 | }, | ||
| 431 | }; | ||
| 432 | @memcpy(closure.contextPointer()[0..context.len], context); | ||
| 433 | |||
| 434 | pool.mutex.lock(); | ||
| 435 | |||
| 436 | const thread_capacity = cpu_count - 1 + pool.parallel_count; | ||
| 437 | |||
| 438 | pool.threads.ensureTotalCapacityPrecise(gpa, thread_capacity) catch { | ||
| 439 | pool.mutex.unlock(); | ||
| 440 | closure.free(gpa); | ||
| 441 | return start(context.ptr); | ||
| 442 | }; | ||
| 443 | |||
| 444 | pool.run_queue.prepend(&closure.runnable.node); | ||
| 445 | |||
| 446 | if (pool.threads.items.len < thread_capacity) { | ||
| 447 | const thread = std.Thread.spawn(.{ .stack_size = pool.stack_size }, worker, .{pool}) catch { | ||
| 448 | assert(pool.run_queue.popFirst() == &closure.runnable.node); | ||
| 449 | pool.mutex.unlock(); | ||
| 450 | closure.free(gpa); | ||
| 451 | return start(context.ptr); | ||
| 452 | }; | ||
| 453 | pool.threads.appendAssumeCapacity(thread); | ||
| 454 | } | ||
| 455 | |||
| 456 | pool.mutex.unlock(); | ||
| 457 | pool.cond.signal(); | ||
| 458 | } | ||
| 459 | |||
| 460 | fn await( | ||
| 461 | userdata: ?*anyopaque, | ||
| 462 | any_future: *std.Io.AnyFuture, | ||
| 463 | result: []u8, | ||
| 464 | result_alignment: std.mem.Alignment, | ||
| 465 | ) void { | ||
| 466 | _ = result_alignment; | ||
| 467 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 468 | const closure: *AsyncClosure = @ptrCast(@alignCast(any_future)); | ||
| 469 | closure.waitAndFree(pool.allocator, result); | ||
| 470 | } | ||
| 471 | |||
| 472 | fn cancel( | ||
| 473 | userdata: ?*anyopaque, | ||
| 474 | any_future: *Io.AnyFuture, | ||
| 475 | result: []u8, | ||
| 476 | result_alignment: std.mem.Alignment, | ||
| 477 | ) void { | ||
| 478 | _ = result_alignment; | ||
| 479 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 480 | const closure: *AsyncClosure = @ptrCast(@alignCast(any_future)); | ||
| 481 | switch (@atomicRmw( | ||
| 482 | std.Thread.Id, | ||
| 483 | &closure.cancel_tid, | ||
| 484 | .Xchg, | ||
| 485 | AsyncClosure.canceling_tid, | ||
| 486 | .acq_rel, | ||
| 487 | )) { | ||
| 488 | 0, AsyncClosure.canceling_tid => {}, | ||
| 489 | else => |cancel_tid| switch (builtin.os.tag) { | ||
| 490 | .linux => _ = std.os.linux.tgkill( | ||
| 491 | std.os.linux.getpid(), | ||
| 492 | @bitCast(cancel_tid), | ||
| 493 | posix.SIG.IO, | ||
| 494 | ), | ||
| 495 | else => {}, | ||
| 496 | }, | ||
| 497 | } | ||
| 498 | closure.waitAndFree(pool.allocator, result); | ||
| 499 | } | ||
| 500 | |||
| 501 | fn cancelRequested(userdata: ?*anyopaque) bool { | ||
| 502 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 503 | _ = pool; | ||
| 504 | const closure = current_closure orelse return false; | ||
| 505 | return @atomicLoad(std.Thread.Id, &closure.cancel_tid, .acquire) == AsyncClosure.canceling_tid; | ||
| 506 | } | ||
| 507 | |||
| 508 | fn checkCancel(pool: *Pool) error{Canceled}!void { | ||
| 509 | if (cancelRequested(pool)) return error.Canceled; | ||
| 510 | } | ||
| 511 | |||
| 512 | fn mutexLock(userdata: ?*anyopaque, prev_state: Io.Mutex.State, mutex: *Io.Mutex) error{Canceled}!void { | ||
| 513 | _ = userdata; | ||
| 514 | if (prev_state == .contended) { | ||
| 515 | std.Thread.Futex.wait(@ptrCast(&mutex.state), @intFromEnum(Io.Mutex.State.contended)); | ||
| 516 | } | ||
| 517 | while (@atomicRmw( | ||
| 518 | Io.Mutex.State, | ||
| 519 | &mutex.state, | ||
| 520 | .Xchg, | ||
| 521 | .contended, | ||
| 522 | .acquire, | ||
| 523 | ) != .unlocked) { | ||
| 524 | std.Thread.Futex.wait(@ptrCast(&mutex.state), @intFromEnum(Io.Mutex.State.contended)); | ||
| 525 | } | ||
| 526 | } | ||
| 527 | fn mutexUnlock(userdata: ?*anyopaque, prev_state: Io.Mutex.State, mutex: *Io.Mutex) void { | ||
| 528 | _ = userdata; | ||
| 529 | _ = prev_state; | ||
| 530 | if (@atomicRmw(Io.Mutex.State, &mutex.state, .Xchg, .unlocked, .release) == .contended) { | ||
| 531 | std.Thread.Futex.wake(@ptrCast(&mutex.state), 1); | ||
| 532 | } | ||
| 533 | } | ||
| 534 | |||
| 535 | fn conditionWait(userdata: ?*anyopaque, cond: *Io.Condition, mutex: *Io.Mutex) Io.Cancelable!void { | ||
| 536 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 537 | comptime assert(@TypeOf(cond.state) == u64); | ||
| 538 | const ints: *[2]std.atomic.Value(u32) = @ptrCast(&cond.state); | ||
| 539 | const cond_state = &ints[0]; | ||
| 540 | const cond_epoch = &ints[1]; | ||
| 541 | const one_waiter = 1; | ||
| 542 | const waiter_mask = 0xffff; | ||
| 543 | const one_signal = 1 << 16; | ||
| 544 | const signal_mask = 0xffff << 16; | ||
| 545 | // Observe the epoch, then check the state again to see if we should wake up. | ||
| 546 | // The epoch must be observed before we check the state or we could potentially miss a wake() and deadlock: | ||
| 547 | // | ||
| 548 | // - T1: s = LOAD(&state) | ||
| 549 | // - T2: UPDATE(&s, signal) | ||
| 550 | // - T2: UPDATE(&epoch, 1) + FUTEX_WAKE(&epoch) | ||
| 551 | // - T1: e = LOAD(&epoch) (was reordered after the state load) | ||
| 552 | // - T1: s & signals == 0 -> FUTEX_WAIT(&epoch, e) (missed the state update + the epoch change) | ||
| 553 | // | ||
| 554 | // Acquire barrier to ensure the epoch load happens before the state load. | ||
| 555 | var epoch = cond_epoch.load(.acquire); | ||
| 556 | var state = cond_state.fetchAdd(one_waiter, .monotonic); | ||
| 557 | assert(state & waiter_mask != waiter_mask); | ||
| 558 | state += one_waiter; | ||
| 559 | |||
| 560 | mutex.unlock(pool.io()); | ||
| 561 | defer mutex.lock(pool.io()) catch @panic("TODO"); | ||
| 562 | |||
| 563 | var futex_deadline = std.Thread.Futex.Deadline.init(null); | ||
| 564 | |||
| 565 | while (true) { | ||
| 566 | futex_deadline.wait(cond_epoch, epoch) catch |err| switch (err) { | ||
| 567 | error.Timeout => unreachable, | ||
| 568 | }; | ||
| 569 | |||
| 570 | epoch = cond_epoch.load(.acquire); | ||
| 571 | state = cond_state.load(.monotonic); | ||
| 572 | |||
| 573 | // Try to wake up by consuming a signal and decremented the waiter we added previously. | ||
| 574 | // Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return. | ||
| 575 | while (state & signal_mask != 0) { | ||
| 576 | const new_state = state - one_waiter - one_signal; | ||
| 577 | state = cond_state.cmpxchgWeak(state, new_state, .acquire, .monotonic) orelse return; | ||
| 578 | } | ||
| 579 | } | ||
| 580 | } | ||
| 581 | |||
| 582 | fn conditionWake(userdata: ?*anyopaque, cond: *Io.Condition, wake: Io.Condition.Wake) void { | ||
| 583 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 584 | _ = pool; | ||
| 585 | comptime assert(@TypeOf(cond.state) == u64); | ||
| 586 | const ints: *[2]std.atomic.Value(u32) = @ptrCast(&cond.state); | ||
| 587 | const cond_state = &ints[0]; | ||
| 588 | const cond_epoch = &ints[1]; | ||
| 589 | const one_waiter = 1; | ||
| 590 | const waiter_mask = 0xffff; | ||
| 591 | const one_signal = 1 << 16; | ||
| 592 | const signal_mask = 0xffff << 16; | ||
| 593 | var state = cond_state.load(.monotonic); | ||
| 594 | while (true) { | ||
| 595 | const waiters = (state & waiter_mask) / one_waiter; | ||
| 596 | const signals = (state & signal_mask) / one_signal; | ||
| 597 | |||
| 598 | // Reserves which waiters to wake up by incrementing the signals count. | ||
| 599 | // Therefore, the signals count is always less than or equal to the waiters count. | ||
| 600 | // We don't need to Futex.wake if there's nothing to wake up or if other wake() threads have reserved to wake up the current waiters. | ||
| 601 | const wakeable = waiters - signals; | ||
| 602 | if (wakeable == 0) { | ||
| 603 | return; | ||
| 604 | } | ||
| 605 | |||
| 606 | const to_wake = switch (wake) { | ||
| 607 | .one => 1, | ||
| 608 | .all => wakeable, | ||
| 609 | }; | ||
| 610 | |||
| 611 | // Reserve the amount of waiters to wake by incrementing the signals count. | ||
| 612 | // Release barrier ensures code before the wake() happens before the signal it posted and consumed by the wait() threads. | ||
| 613 | const new_state = state + (one_signal * to_wake); | ||
| 614 | state = cond_state.cmpxchgWeak(state, new_state, .release, .monotonic) orelse { | ||
| 615 | // Wake up the waiting threads we reserved above by changing the epoch value. | ||
| 616 | // NOTE: a waiting thread could miss a wake up if *exactly* ((1<<32)-1) wake()s happen between it observing the epoch and sleeping on it. | ||
| 617 | // This is very unlikely due to how many precise amount of Futex.wake() calls that would be between the waiting thread's potential preemption. | ||
| 618 | // | ||
| 619 | // Release barrier ensures the signal being added to the state happens before the epoch is changed. | ||
| 620 | // If not, the waiting thread could potentially deadlock from missing both the state and epoch change: | ||
| 621 | // | ||
| 622 | // - T2: UPDATE(&epoch, 1) (reordered before the state change) | ||
| 623 | // - T1: e = LOAD(&epoch) | ||
| 624 | // - T1: s = LOAD(&state) | ||
| 625 | // - T2: UPDATE(&state, signal) + FUTEX_WAKE(&epoch) | ||
| 626 | // - T1: s & signals == 0 -> FUTEX_WAIT(&epoch, e) (missed both epoch change and state change) | ||
| 627 | _ = cond_epoch.fetchAdd(1, .release); | ||
| 628 | std.Thread.Futex.wake(cond_epoch, to_wake); | ||
| 629 | return; | ||
| 630 | }; | ||
| 631 | } | ||
| 632 | } | ||
| 633 | |||
| 634 | fn createFile( | ||
| 635 | userdata: ?*anyopaque, | ||
| 636 | dir: Io.Dir, | ||
| 637 | sub_path: []const u8, | ||
| 638 | flags: Io.File.CreateFlags, | ||
| 639 | ) Io.File.OpenError!Io.File { | ||
| 640 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 641 | try pool.checkCancel(); | ||
| 642 | const fs_dir: std.fs.Dir = .{ .fd = dir.handle }; | ||
| 643 | const fs_file = try fs_dir.createFile(sub_path, flags); | ||
| 644 | return .{ .handle = fs_file.handle }; | ||
| 645 | } | ||
| 646 | |||
| 647 | fn fileOpen( | ||
| 648 | userdata: ?*anyopaque, | ||
| 649 | dir: Io.Dir, | ||
| 650 | sub_path: []const u8, | ||
| 651 | flags: Io.File.OpenFlags, | ||
| 652 | ) Io.File.OpenError!Io.File { | ||
| 653 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 654 | try pool.checkCancel(); | ||
| 655 | const fs_dir: std.fs.Dir = .{ .fd = dir.handle }; | ||
| 656 | const fs_file = try fs_dir.openFile(sub_path, flags); | ||
| 657 | return .{ .handle = fs_file.handle }; | ||
| 658 | } | ||
| 659 | |||
| 660 | fn fileClose(userdata: ?*anyopaque, file: Io.File) void { | ||
| 661 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 662 | _ = pool; | ||
| 663 | const fs_file: std.fs.File = .{ .handle = file.handle }; | ||
| 664 | return fs_file.close(); | ||
| 665 | } | ||
| 666 | |||
| 667 | fn fileReadStreaming(userdata: ?*anyopaque, file: Io.File, data: [][]u8) Io.File.ReadStreamingError!usize { | ||
| 668 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 669 | |||
| 670 | if (is_windows) { | ||
| 671 | const DWORD = windows.DWORD; | ||
| 672 | var index: usize = 0; | ||
| 673 | var truncate: usize = 0; | ||
| 674 | var total: usize = 0; | ||
| 675 | while (index < data.len) { | ||
| 676 | try pool.checkCancel(); | ||
| 677 | { | ||
| 678 | const untruncated = data[index]; | ||
| 679 | data[index] = untruncated[truncate..]; | ||
| 680 | defer data[index] = untruncated; | ||
| 681 | const buffer = data[index..]; | ||
| 682 | const want_read_count: DWORD = @min(std.math.maxInt(DWORD), buffer.len); | ||
| 683 | var n: DWORD = undefined; | ||
| 684 | if (windows.kernel32.ReadFile(file.handle, buffer.ptr, want_read_count, &n, null) == 0) { | ||
| 685 | switch (windows.GetLastError()) { | ||
| 686 | .IO_PENDING => unreachable, | ||
| 687 | .OPERATION_ABORTED => continue, | ||
| 688 | .BROKEN_PIPE => return 0, | ||
| 689 | .HANDLE_EOF => return 0, | ||
| 690 | .NETNAME_DELETED => return error.ConnectionResetByPeer, | ||
| 691 | .LOCK_VIOLATION => return error.LockViolation, | ||
| 692 | .ACCESS_DENIED => return error.AccessDenied, | ||
| 693 | .INVALID_HANDLE => return error.NotOpenForReading, | ||
| 694 | else => |err| return windows.unexpectedError(err), | ||
| 695 | } | ||
| 696 | } | ||
| 697 | total += n; | ||
| 698 | truncate += n; | ||
| 699 | } | ||
| 700 | while (index < data.len and truncate >= data[index].len) { | ||
| 701 | truncate -= data[index].len; | ||
| 702 | index += 1; | ||
| 703 | } | ||
| 704 | } | ||
| 705 | return total; | ||
| 706 | } | ||
| 707 | |||
| 708 | var iovecs_buffer: [max_iovecs_len]posix.iovec = undefined; | ||
| 709 | var i: usize = 0; | ||
| 710 | for (data) |buf| { | ||
| 711 | if (iovecs_buffer.len - i == 0) break; | ||
| 712 | if (buf.len != 0) { | ||
| 713 | iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len }; | ||
| 714 | i += 1; | ||
| 715 | } | ||
| 716 | } | ||
| 717 | const dest = iovecs_buffer[0..i]; | ||
| 718 | assert(dest[0].len > 0); | ||
| 719 | |||
| 720 | if (native_os == .wasi and !builtin.link_libc) { | ||
| 721 | try pool.checkCancel(); | ||
| 722 | var nread: usize = undefined; | ||
| 723 | switch (std.os.wasi.fd_read(file.handle, dest.ptr, dest.len, &nread)) { | ||
| 724 | .SUCCESS => return nread, | ||
| 725 | .INTR => unreachable, | ||
| 726 | .INVAL => unreachable, | ||
| 727 | .FAULT => unreachable, | ||
| 728 | .AGAIN => unreachable, // currently not support in WASI | ||
| 729 | .BADF => return error.NotOpenForReading, // can be a race condition | ||
| 730 | .IO => return error.InputOutput, | ||
| 731 | .ISDIR => return error.IsDir, | ||
| 732 | .NOBUFS => return error.SystemResources, | ||
| 733 | .NOMEM => return error.SystemResources, | ||
| 734 | .NOTCONN => return error.SocketNotConnected, | ||
| 735 | .CONNRESET => return error.ConnectionResetByPeer, | ||
| 736 | .TIMEDOUT => return error.ConnectionTimedOut, | ||
| 737 | .NOTCAPABLE => return error.AccessDenied, | ||
| 738 | else => |err| return posix.unexpectedErrno(err), | ||
| 739 | } | ||
| 740 | } | ||
| 741 | |||
| 742 | while (true) { | ||
| 743 | try pool.checkCancel(); | ||
| 744 | const rc = posix.system.readv(file.handle, dest.ptr, dest.len); | ||
| 745 | switch (posix.errno(rc)) { | ||
| 746 | .SUCCESS => return @intCast(rc), | ||
| 747 | .INTR => continue, | ||
| 748 | .INVAL => unreachable, | ||
| 749 | .FAULT => unreachable, | ||
| 750 | .SRCH => return error.ProcessNotFound, | ||
| 751 | .AGAIN => return error.WouldBlock, | ||
| 752 | .BADF => return error.NotOpenForReading, // can be a race condition | ||
| 753 | .IO => return error.InputOutput, | ||
| 754 | .ISDIR => return error.IsDir, | ||
| 755 | .NOBUFS => return error.SystemResources, | ||
| 756 | .NOMEM => return error.SystemResources, | ||
| 757 | .NOTCONN => return error.SocketNotConnected, | ||
| 758 | .CONNRESET => return error.ConnectionResetByPeer, | ||
| 759 | .TIMEDOUT => return error.ConnectionTimedOut, | ||
| 760 | else => |err| return posix.unexpectedErrno(err), | ||
| 761 | } | ||
| 762 | } | ||
| 763 | } | ||
| 764 | |||
| 765 | fn fileReadPositional(userdata: ?*anyopaque, file: Io.File, data: [][]u8, offset: u64) Io.File.ReadPositionalError!usize { | ||
| 766 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 767 | |||
| 768 | const have_pread_but_not_preadv = switch (native_os) { | ||
| 769 | .windows, .macos, .ios, .watchos, .tvos, .visionos, .haiku, .serenity => true, | ||
| 770 | else => false, | ||
| 771 | }; | ||
| 772 | if (have_pread_but_not_preadv) { | ||
| 773 | @compileError("TODO"); | ||
| 774 | } | ||
| 775 | |||
| 776 | if (is_windows) { | ||
| 777 | const DWORD = windows.DWORD; | ||
| 778 | const OVERLAPPED = windows.OVERLAPPED; | ||
| 779 | var index: usize = 0; | ||
| 780 | var truncate: usize = 0; | ||
| 781 | var total: usize = 0; | ||
| 782 | while (true) { | ||
| 783 | try pool.checkCancel(); | ||
| 784 | { | ||
| 785 | const untruncated = data[index]; | ||
| 786 | data[index] = untruncated[truncate..]; | ||
| 787 | defer data[index] = untruncated; | ||
| 788 | const buffer = data[index..]; | ||
| 789 | const want_read_count: DWORD = @min(std.math.maxInt(DWORD), buffer.len); | ||
| 790 | var n: DWORD = undefined; | ||
| 791 | var overlapped_data: OVERLAPPED = undefined; | ||
| 792 | const overlapped: ?*OVERLAPPED = if (offset) |off| blk: { | ||
| 793 | overlapped_data = .{ | ||
| 794 | .Internal = 0, | ||
| 795 | .InternalHigh = 0, | ||
| 796 | .DUMMYUNIONNAME = .{ | ||
| 797 | .DUMMYSTRUCTNAME = .{ | ||
| 798 | .Offset = @as(u32, @truncate(off)), | ||
| 799 | .OffsetHigh = @as(u32, @truncate(off >> 32)), | ||
| 800 | }, | ||
| 801 | }, | ||
| 802 | .hEvent = null, | ||
| 803 | }; | ||
| 804 | break :blk &overlapped_data; | ||
| 805 | } else null; | ||
| 806 | if (windows.kernel32.ReadFile(file.handle, buffer.ptr, want_read_count, &n, overlapped) == 0) { | ||
| 807 | switch (windows.GetLastError()) { | ||
| 808 | .IO_PENDING => unreachable, | ||
| 809 | .OPERATION_ABORTED => continue, | ||
| 810 | .BROKEN_PIPE => return 0, | ||
| 811 | .HANDLE_EOF => return 0, | ||
| 812 | .NETNAME_DELETED => return error.ConnectionResetByPeer, | ||
| 813 | .LOCK_VIOLATION => return error.LockViolation, | ||
| 814 | .ACCESS_DENIED => return error.AccessDenied, | ||
| 815 | .INVALID_HANDLE => return error.NotOpenForReading, | ||
| 816 | else => |err| return windows.unexpectedError(err), | ||
| 817 | } | ||
| 818 | } | ||
| 819 | total += n; | ||
| 820 | truncate += n; | ||
| 821 | } | ||
| 822 | while (index < data.len and truncate >= data[index].len) { | ||
| 823 | truncate -= data[index].len; | ||
| 824 | index += 1; | ||
| 825 | } | ||
| 826 | } | ||
| 827 | return total; | ||
| 828 | } | ||
| 829 | |||
| 830 | var iovecs_buffer: [max_iovecs_len]posix.iovec = undefined; | ||
| 831 | var i: usize = 0; | ||
| 832 | for (data) |buf| { | ||
| 833 | if (iovecs_buffer.len - i == 0) break; | ||
| 834 | if (buf.len != 0) { | ||
| 835 | iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len }; | ||
| 836 | i += 1; | ||
| 837 | } | ||
| 838 | } | ||
| 839 | const dest = iovecs_buffer[0..i]; | ||
| 840 | assert(dest[0].len > 0); | ||
| 841 | |||
| 842 | if (native_os == .wasi and !builtin.link_libc) { | ||
| 843 | try pool.checkCancel(); | ||
| 844 | var nread: usize = undefined; | ||
| 845 | switch (std.os.wasi.fd_pread(file.handle, dest.ptr, dest.len, offset, &nread)) { | ||
| 846 | .SUCCESS => return nread, | ||
| 847 | .INTR => unreachable, | ||
| 848 | .INVAL => unreachable, | ||
| 849 | .FAULT => unreachable, | ||
| 850 | .AGAIN => unreachable, | ||
| 851 | .BADF => return error.NotOpenForReading, // can be a race condition | ||
| 852 | .IO => return error.InputOutput, | ||
| 853 | .ISDIR => return error.IsDir, | ||
| 854 | .NOBUFS => return error.SystemResources, | ||
| 855 | .NOMEM => return error.SystemResources, | ||
| 856 | .NOTCONN => return error.SocketNotConnected, | ||
| 857 | .CONNRESET => return error.ConnectionResetByPeer, | ||
| 858 | .TIMEDOUT => return error.ConnectionTimedOut, | ||
| 859 | .NXIO => return error.Unseekable, | ||
| 860 | .SPIPE => return error.Unseekable, | ||
| 861 | .OVERFLOW => return error.Unseekable, | ||
| 862 | .NOTCAPABLE => return error.AccessDenied, | ||
| 863 | else => |err| return posix.unexpectedErrno(err), | ||
| 864 | } | ||
| 865 | } | ||
| 866 | |||
| 867 | const preadv_sym = if (posix.lfs64_abi) posix.system.preadv64 else posix.system.preadv; | ||
| 868 | while (true) { | ||
| 869 | try pool.checkCancel(); | ||
| 870 | const rc = preadv_sym(file.handle, dest.ptr, dest.len, @bitCast(offset)); | ||
| 871 | switch (posix.errno(rc)) { | ||
| 872 | .SUCCESS => return @bitCast(rc), | ||
| 873 | .INTR => continue, | ||
| 874 | .INVAL => unreachable, | ||
| 875 | .FAULT => unreachable, | ||
| 876 | .SRCH => return error.ProcessNotFound, | ||
| 877 | .AGAIN => return error.WouldBlock, | ||
| 878 | .BADF => return error.NotOpenForReading, // can be a race condition | ||
| 879 | .IO => return error.InputOutput, | ||
| 880 | .ISDIR => return error.IsDir, | ||
| 881 | .NOBUFS => return error.SystemResources, | ||
| 882 | .NOMEM => return error.SystemResources, | ||
| 883 | .NOTCONN => return error.SocketNotConnected, | ||
| 884 | .CONNRESET => return error.ConnectionResetByPeer, | ||
| 885 | .TIMEDOUT => return error.ConnectionTimedOut, | ||
| 886 | .NXIO => return error.Unseekable, | ||
| 887 | .SPIPE => return error.Unseekable, | ||
| 888 | .OVERFLOW => return error.Unseekable, | ||
| 889 | else => |err| return posix.unexpectedErrno(err), | ||
| 890 | } | ||
| 891 | } | ||
| 892 | } | ||
| 893 | |||
| 894 | fn fileSeekBy(userdata: ?*anyopaque, file: Io.File, offset: i64) Io.File.SeekError!void { | ||
| 895 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 896 | try pool.checkCancel(); | ||
| 897 | |||
| 898 | _ = file; | ||
| 899 | _ = offset; | ||
| 900 | @panic("TODO"); | ||
| 901 | } | ||
| 902 | |||
| 903 | fn fileSeekTo(userdata: ?*anyopaque, file: Io.File, offset: u64) Io.File.SeekError!void { | ||
| 904 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 905 | try pool.checkCancel(); | ||
| 906 | |||
| 907 | _ = file; | ||
| 908 | _ = offset; | ||
| 909 | @panic("TODO"); | ||
| 910 | } | ||
| 911 | |||
| 912 | fn pwrite(userdata: ?*anyopaque, file: Io.File, buffer: []const u8, offset: posix.off_t) Io.File.PWriteError!usize { | ||
| 913 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 914 | try pool.checkCancel(); | ||
| 915 | const fs_file: std.fs.File = .{ .handle = file.handle }; | ||
| 916 | return switch (offset) { | ||
| 917 | -1 => fs_file.write(buffer), | ||
| 918 | else => fs_file.pwrite(buffer, @bitCast(offset)), | ||
| 919 | }; | ||
| 920 | } | ||
| 921 | |||
| 922 | fn now(userdata: ?*anyopaque, clockid: posix.clockid_t) Io.ClockGetTimeError!Io.Timestamp { | ||
| 923 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 924 | try pool.checkCancel(); | ||
| 925 | const timespec = try posix.clock_gettime(clockid); | ||
| 926 | return @enumFromInt(@as(i128, timespec.sec) * std.time.ns_per_s + timespec.nsec); | ||
| 927 | } | ||
| 928 | |||
| 929 | fn sleep(userdata: ?*anyopaque, clockid: posix.clockid_t, deadline: Io.Deadline) Io.SleepError!void { | ||
| 930 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 931 | const deadline_nanoseconds: i96 = switch (deadline) { | ||
| 932 | .duration => |duration| duration.nanoseconds, | ||
| 933 | .timestamp => |timestamp| @intFromEnum(timestamp), | ||
| 934 | }; | ||
| 935 | var timespec: posix.timespec = .{ | ||
| 936 | .sec = @intCast(@divFloor(deadline_nanoseconds, std.time.ns_per_s)), | ||
| 937 | .nsec = @intCast(@mod(deadline_nanoseconds, std.time.ns_per_s)), | ||
| 938 | }; | ||
| 939 | while (true) { | ||
| 940 | try pool.checkCancel(); | ||
| 941 | switch (std.os.linux.E.init(std.os.linux.clock_nanosleep(clockid, .{ .ABSTIME = switch (deadline) { | ||
| 942 | .duration => false, | ||
| 943 | .timestamp => true, | ||
| 944 | } }, ×pec, ×pec))) { | ||
| 945 | .SUCCESS => return, | ||
| 946 | .FAULT => unreachable, | ||
| 947 | .INTR => {}, | ||
| 948 | .INVAL => return error.UnsupportedClock, | ||
| 949 | else => |err| return posix.unexpectedErrno(err), | ||
| 950 | } | ||
| 951 | } | ||
| 952 | } | ||
| 953 | |||
| 954 | fn select(userdata: ?*anyopaque, futures: []const *Io.AnyFuture) usize { | ||
| 955 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 956 | _ = pool; | ||
| 957 | |||
| 958 | var reset_event: std.Thread.ResetEvent = .{}; | ||
| 959 | |||
| 960 | for (futures, 0..) |future, i| { | ||
| 961 | const closure: *AsyncClosure = @ptrCast(@alignCast(future)); | ||
| 962 | if (@atomicRmw(?*std.Thread.ResetEvent, &closure.select_condition, .Xchg, &reset_event, .seq_cst) == AsyncClosure.done_reset_event) { | ||
| 963 | for (futures[0..i]) |cleanup_future| { | ||
| 964 | const cleanup_closure: *AsyncClosure = @ptrCast(@alignCast(cleanup_future)); | ||
| 965 | if (@atomicRmw(?*std.Thread.ResetEvent, &cleanup_closure.select_condition, .Xchg, null, .seq_cst) == AsyncClosure.done_reset_event) { | ||
| 966 | cleanup_closure.reset_event.wait(); // Ensure no reference to our stack-allocated reset_event. | ||
| 967 | } | ||
| 968 | } | ||
| 969 | return i; | ||
| 970 | } | ||
| 971 | } | ||
| 972 | |||
| 973 | reset_event.wait(); | ||
| 974 | |||
| 975 | var result: ?usize = null; | ||
| 976 | for (futures, 0..) |future, i| { | ||
| 977 | const closure: *AsyncClosure = @ptrCast(@alignCast(future)); | ||
| 978 | if (@atomicRmw(?*std.Thread.ResetEvent, &closure.select_condition, .Xchg, null, .seq_cst) == AsyncClosure.done_reset_event) { | ||
| 979 | closure.reset_event.wait(); // Ensure no reference to our stack-allocated reset_event. | ||
| 980 | if (result == null) result = i; // In case multiple are ready, return first. | ||
| 981 | } | ||
| 982 | } | ||
| 983 | return result.?; | ||
| 984 | } | ||
| 985 | |||
| 986 | fn listen(userdata: ?*anyopaque, address: Io.net.IpAddress, options: Io.net.ListenOptions) Io.net.ListenError!Io.net.Server { | ||
| 987 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 988 | try pool.checkCancel(); | ||
| 989 | |||
| 990 | const nonblock: u32 = if (options.force_nonblocking) posix.SOCK.NONBLOCK else 0; | ||
| 991 | const sock_flags = posix.SOCK.STREAM | posix.SOCK.CLOEXEC | nonblock; | ||
| 992 | const proto: u32 = posix.IPPROTO.TCP; | ||
| 993 | const family = posixAddressFamily(address); | ||
| 994 | const sockfd = try posix.socket(family, sock_flags, proto); | ||
| 995 | const stream: std.net.Stream = .{ .handle = sockfd }; | ||
| 996 | errdefer stream.close(); | ||
| 997 | |||
| 998 | if (options.reuse_address) { | ||
| 999 | try posix.setsockopt( | ||
| 1000 | sockfd, | ||
| 1001 | posix.SOL.SOCKET, | ||
| 1002 | posix.SO.REUSEADDR, | ||
| 1003 | &std.mem.toBytes(@as(c_int, 1)), | ||
| 1004 | ); | ||
| 1005 | if (@hasDecl(posix.SO, "REUSEPORT") and family != posix.AF.UNIX) { | ||
| 1006 | try posix.setsockopt( | ||
| 1007 | sockfd, | ||
| 1008 | posix.SOL.SOCKET, | ||
| 1009 | posix.SO.REUSEPORT, | ||
| 1010 | &std.mem.toBytes(@as(c_int, 1)), | ||
| 1011 | ); | ||
| 1012 | } | ||
| 1013 | } | ||
| 1014 | |||
| 1015 | var storage: PosixAddress = undefined; | ||
| 1016 | var socklen = addressToPosix(address, &storage); | ||
| 1017 | try posix.bind(sockfd, &storage.any, socklen); | ||
| 1018 | try posix.listen(sockfd, options.kernel_backlog); | ||
| 1019 | try posix.getsockname(sockfd, &storage.any, &socklen); | ||
| 1020 | return .{ | ||
| 1021 | .listen_address = addressFromPosix(&storage), | ||
| 1022 | .stream = .{ .handle = stream.handle }, | ||
| 1023 | }; | ||
| 1024 | } | ||
| 1025 | |||
| 1026 | fn accept(userdata: ?*anyopaque, server: *Io.net.Server) Io.net.Server.AcceptError!Io.net.Server.Connection { | ||
| 1027 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 1028 | try pool.checkCancel(); | ||
| 1029 | |||
| 1030 | var storage: PosixAddress = undefined; | ||
| 1031 | var addr_len: posix.socklen_t = @sizeOf(PosixAddress); | ||
| 1032 | const fd = try posix.accept(server.stream.handle, &storage.any, &addr_len, posix.SOCK.CLOEXEC); | ||
| 1033 | return .{ | ||
| 1034 | .stream = .{ .handle = fd }, | ||
| 1035 | .address = addressFromPosix(&storage), | ||
| 1036 | }; | ||
| 1037 | } | ||
| 1038 | |||
| 1039 | fn netReadPosix(userdata: ?*anyopaque, stream: Io.net.Stream, data: [][]u8) Io.net.Stream.Reader.Error!usize { | ||
| 1040 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 1041 | try pool.checkCancel(); | ||
| 1042 | |||
| 1043 | var iovecs_buffer: [max_iovecs_len]posix.iovec = undefined; | ||
| 1044 | var i: usize = 0; | ||
| 1045 | for (data) |buf| { | ||
| 1046 | if (iovecs_buffer.len - i == 0) break; | ||
| 1047 | if (buf.len != 0) { | ||
| 1048 | iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len }; | ||
| 1049 | i += 1; | ||
| 1050 | } | ||
| 1051 | } | ||
| 1052 | const dest = iovecs_buffer[0..i]; | ||
| 1053 | assert(dest[0].len > 0); | ||
| 1054 | const n = try posix.readv(stream.handle, dest); | ||
| 1055 | if (n == 0) return error.EndOfStream; | ||
| 1056 | return n; | ||
| 1057 | } | ||
| 1058 | |||
| 1059 | fn netWritePosix( | ||
| 1060 | userdata: ?*anyopaque, | ||
| 1061 | stream: Io.net.Stream, | ||
| 1062 | header: []const u8, | ||
| 1063 | data: []const []const u8, | ||
| 1064 | splat: usize, | ||
| 1065 | ) Io.net.Stream.Writer.Error!usize { | ||
| 1066 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 1067 | try pool.checkCancel(); | ||
| 1068 | |||
| 1069 | var iovecs: [max_iovecs_len]posix.iovec_const = undefined; | ||
| 1070 | var msg: posix.msghdr_const = .{ | ||
| 1071 | .name = null, | ||
| 1072 | .namelen = 0, | ||
| 1073 | .iov = &iovecs, | ||
| 1074 | .iovlen = 0, | ||
| 1075 | .control = null, | ||
| 1076 | .controllen = 0, | ||
| 1077 | .flags = 0, | ||
| 1078 | }; | ||
| 1079 | addBuf(&iovecs, &msg.iovlen, header); | ||
| 1080 | for (data[0 .. data.len - 1]) |bytes| addBuf(&iovecs, &msg.iovlen, bytes); | ||
| 1081 | const pattern = data[data.len - 1]; | ||
| 1082 | if (iovecs.len - msg.iovlen != 0) switch (splat) { | ||
| 1083 | 0 => {}, | ||
| 1084 | 1 => addBuf(&iovecs, &msg.iovlen, pattern), | ||
| 1085 | else => switch (pattern.len) { | ||
| 1086 | 0 => {}, | ||
| 1087 | 1 => { | ||
| 1088 | var backup_buffer: [splat_buffer_size]u8 = undefined; | ||
| 1089 | const splat_buffer = &backup_buffer; | ||
| 1090 | const memset_len = @min(splat_buffer.len, splat); | ||
| 1091 | const buf = splat_buffer[0..memset_len]; | ||
| 1092 | @memset(buf, pattern[0]); | ||
| 1093 | addBuf(&iovecs, &msg.iovlen, buf); | ||
| 1094 | var remaining_splat = splat - buf.len; | ||
| 1095 | while (remaining_splat > splat_buffer.len and iovecs.len - msg.iovlen != 0) { | ||
| 1096 | assert(buf.len == splat_buffer.len); | ||
| 1097 | addBuf(&iovecs, &msg.iovlen, splat_buffer); | ||
| 1098 | remaining_splat -= splat_buffer.len; | ||
| 1099 | } | ||
| 1100 | addBuf(&iovecs, &msg.iovlen, splat_buffer[0..remaining_splat]); | ||
| 1101 | }, | ||
| 1102 | else => for (0..@min(splat, iovecs.len - msg.iovlen)) |_| { | ||
| 1103 | addBuf(&iovecs, &msg.iovlen, pattern); | ||
| 1104 | }, | ||
| 1105 | }, | ||
| 1106 | }; | ||
| 1107 | const flags = posix.MSG.NOSIGNAL; | ||
| 1108 | return posix.sendmsg(stream.handle, &msg, flags); | ||
| 1109 | } | ||
| 1110 | |||
| 1111 | fn addBuf(v: []posix.iovec_const, i: *@FieldType(posix.msghdr_const, "iovlen"), bytes: []const u8) void { | ||
| 1112 | // OS checks ptr addr before length so zero length vectors must be omitted. | ||
| 1113 | if (bytes.len == 0) return; | ||
| 1114 | if (v.len - i.* == 0) return; | ||
| 1115 | v[i.*] = .{ .base = bytes.ptr, .len = bytes.len }; | ||
| 1116 | i.* += 1; | ||
| 1117 | } | ||
| 1118 | |||
| 1119 | fn netClose(userdata: ?*anyopaque, stream: Io.net.Stream) void { | ||
| 1120 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 1121 | _ = pool; | ||
| 1122 | const net_stream: std.net.Stream = .{ .handle = stream.handle }; | ||
| 1123 | return net_stream.close(); | ||
| 1124 | } | ||
| 1125 | |||
| 1126 | fn netInterfaceIndex(userdata: ?*anyopaque, name: []const u8) Io.net.InterfaceIndexError!u32 { | ||
| 1127 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | ||
| 1128 | try pool.checkCancel(); | ||
| 1129 | |||
| 1130 | if (native_os == .linux) { | ||
| 1131 | if (name.len >= posix.IFNAMESIZE) return error.InterfaceNotFound; | ||
| 1132 | var ifr: posix.ifreq = undefined; | ||
| 1133 | @memcpy(ifr.ifrn.name[0..name.len], name); | ||
| 1134 | ifr.ifrn.name[name.len] = 0; | ||
| 1135 | |||
| 1136 | const rc = posix.system.socket(posix.AF.UNIX, posix.SOCK.DGRAM | posix.SOCK.CLOEXEC, 0); | ||
| 1137 | const sock_fd: posix.fd_t = switch (posix.errno(rc)) { | ||
| 1138 | .SUCCESS => @intCast(rc), | ||
| 1139 | .ACCES => return error.AccessDenied, | ||
| 1140 | .MFILE => return error.SystemResources, | ||
| 1141 | .NFILE => return error.SystemResources, | ||
| 1142 | .NOBUFS => return error.SystemResources, | ||
| 1143 | .NOMEM => return error.SystemResources, | ||
| 1144 | else => |err| return posix.unexpectedErrno(err), | ||
| 1145 | }; | ||
| 1146 | defer posix.close(sock_fd); | ||
| 1147 | |||
| 1148 | while (true) { | ||
| 1149 | try pool.checkCancel(); | ||
| 1150 | switch (posix.errno(posix.system.ioctl(sock_fd, posix.SIOCGIFINDEX, @intFromPtr(&ifr)))) { | ||
| 1151 | .SUCCESS => return @bitCast(ifr.ifru.ivalue), | ||
| 1152 | .INVAL => |err| return badErrno(err), // Bad parameters. | ||
| 1153 | .NOTTY => |err| return badErrno(err), | ||
| 1154 | .NXIO => |err| return badErrno(err), | ||
| 1155 | .BADF => |err| return badErrno(err), // Always a race condition. | ||
| 1156 | .FAULT => |err| return badErrno(err), // Bad pointer parameter. | ||
| 1157 | .INTR => continue, | ||
| 1158 | .IO => |err| return badErrno(err), // sock_fd is not a file descriptor | ||
| 1159 | .NODEV => return error.InterfaceNotFound, | ||
| 1160 | else => |err| return posix.unexpectedErrno(err), | ||
| 1161 | } | ||
| 1162 | } | ||
| 1163 | } | ||
| 1164 | |||
| 1165 | if (native_os.isDarwin()) { | ||
| 1166 | if (name.len >= posix.IFNAMESIZE) return error.InterfaceNotFound; | ||
| 1167 | var if_name: [posix.IFNAMESIZE:0]u8 = undefined; | ||
| 1168 | @memcpy(if_name[0..name.len], name); | ||
| 1169 | if_name[name.len] = 0; | ||
| 1170 | const if_slice = if_name[0..name.len :0]; | ||
| 1171 | const index = std.c.if_nametoindex(if_slice); | ||
| 1172 | if (index == 0) return error.InterfaceNotFound; | ||
| 1173 | return @bitCast(index); | ||
| 1174 | } | ||
| 1175 | |||
| 1176 | if (native_os == .windows) { | ||
| 1177 | if (name.len >= posix.IFNAMESIZE) return error.InterfaceNotFound; | ||
| 1178 | var interface_name: [posix.IFNAMESIZE:0]u8 = undefined; | ||
| 1179 | @memcpy(interface_name[0..name.len], name); | ||
| 1180 | interface_name[name.len] = 0; | ||
| 1181 | const index = std.os.windows.ws2_32.if_nametoindex(@as([*:0]const u8, &interface_name)); | ||
| 1182 | if (index == 0) return error.InterfaceNotFound; | ||
| 1183 | return index; | ||
| 1184 | } | ||
| 1185 | |||
| 1186 | @compileError("std.net.if_nametoindex unimplemented for this OS"); | ||
| 1187 | } | ||
| 1188 | |||
| 1189 | const PosixAddress = extern union { | ||
| 1190 | any: posix.sockaddr, | ||
| 1191 | in: posix.sockaddr.in, | ||
| 1192 | in6: posix.sockaddr.in6, | ||
| 1193 | }; | ||
| 1194 | |||
| 1195 | fn posixAddressFamily(a: Io.net.IpAddress) posix.sa_family_t { | ||
| 1196 | return switch (a) { | ||
| 1197 | .ip4 => posix.AF.INET, | ||
| 1198 | .ip6 => posix.AF.INET6, | ||
| 1199 | }; | ||
| 1200 | } | ||
| 1201 | |||
| 1202 | fn addressFromPosix(posix_address: *PosixAddress) Io.net.IpAddress { | ||
| 1203 | return switch (posix_address.any.family) { | ||
| 1204 | posix.AF.INET => .{ .ip4 = address4FromPosix(&posix_address.in) }, | ||
| 1205 | posix.AF.INET6 => .{ .ip6 = address6FromPosix(&posix_address.in6) }, | ||
| 1206 | else => unreachable, | ||
| 1207 | }; | ||
| 1208 | } | ||
| 1209 | |||
| 1210 | fn addressToPosix(a: Io.net.IpAddress, storage: *PosixAddress) posix.socklen_t { | ||
| 1211 | return switch (a) { | ||
| 1212 | .ip4 => |ip4| { | ||
| 1213 | storage.in = address4ToPosix(ip4); | ||
| 1214 | return @sizeOf(posix.sockaddr.in); | ||
| 1215 | }, | ||
| 1216 | .ip6 => |ip6| { | ||
| 1217 | storage.in6 = address6ToPosix(ip6); | ||
| 1218 | return @sizeOf(posix.sockaddr.in6); | ||
| 1219 | }, | ||
| 1220 | }; | ||
| 1221 | } | ||
| 1222 | |||
| 1223 | fn address4FromPosix(in: *posix.sockaddr.in) Io.net.Ip4Address { | ||
| 1224 | return .{ | ||
| 1225 | .port = std.mem.bigToNative(u16, in.port), | ||
| 1226 | .bytes = @bitCast(in.addr), | ||
| 1227 | }; | ||
| 1228 | } | ||
| 1229 | |||
| 1230 | fn address6FromPosix(in6: *posix.sockaddr.in6) Io.net.Ip6Address { | ||
| 1231 | return .{ | ||
| 1232 | .port = std.mem.bigToNative(u16, in6.port), | ||
| 1233 | .bytes = in6.addr, | ||
| 1234 | .flowinfo = in6.flowinfo, | ||
| 1235 | .scope_id = in6.scope_id, | ||
| 1236 | }; | ||
| 1237 | } | ||
| 1238 | |||
| 1239 | fn address4ToPosix(a: Io.net.Ip4Address) posix.sockaddr.in { | ||
| 1240 | return .{ | ||
| 1241 | .port = std.mem.nativeToBig(u16, a.port), | ||
| 1242 | .addr = @bitCast(a.bytes), | ||
| 1243 | }; | ||
| 1244 | } | ||
| 1245 | |||
| 1246 | fn address6ToPosix(a: Io.net.Ip6Address) posix.sockaddr.in6 { | ||
| 1247 | return .{ | ||
| 1248 | .port = std.mem.nativeToBig(u16, a.port), | ||
| 1249 | .flowinfo = a.flowinfo, | ||
| 1250 | .addr = a.bytes, | ||
| 1251 | .scope_id = a.scope_id, | ||
| 1252 | }; | ||
| 1253 | } | ||
| 1254 | |||
| 1255 | fn badErrno(err: posix.E) Io.UnexpectedError { | ||
| 1256 | switch (builtin.mode) { | ||
| 1257 | .Debug => std.debug.panic("programmer bug caused syscall error: {t}", .{err}), | ||
| 1258 | else => return error.Unexpected, | ||
| 1259 | } | ||
| 1260 | } | ||