Commit 2fa1a784914c536e45fdfaba9e357d0ef4f4f133 | zig

author	Jacob Young <jacobly@ziglang.org> 2026-02-09 14:19:27-05:00
committer	Jacob Young <jacobly@ziglang.org> 2026-02-13 12:29:40-05:00
log	2fa1a784914c536e45fdfaba9e357d0ef4f4f133
tree	658e98008e20195a7a00ac1a3affb1d82ce18d43
parent	469bf6af07ad16fc5ac1421a302178a3fd508343

Io.Dispatch: introduce grand central dispatch io impl

15 files changed, 11980 insertions(+), 6762 deletions(-)

lib
- std
  - Build
    Watch
    FsEvents.zig
  - Io.zig
  - Io
    Dispatch.zig
    IoUring.zig
    Kqueue.zig
    Reader.zig
    Threaded.zig
    Uring.zig
    fiber.zig
  - c.zig
  - c
    darwin.zig
    darwin
    dispatch.zig
  - debug.zig
  - process
    Environ.zig
src
- crash_report.zig

lib/std/Build/Watch/FsEvents.zig+13-31

 @@ -34,11 +34,11 @@ watch_paths: std.StringArrayHashMapUnmanaged([]const *std.Build.Step),
 /// The semaphore we use to block the thread calling `wait` until the callback determines a relevant
 /// event has occurred. This is retained across `wait` calls for simplicity and efficiency.
-waiting_semaphore: dispatch_semaphore_t,
+waiting_semaphore: dispatch.semaphore_t,
 /// This dispatch queue is created by us and executes serially. It exists exclusively to trigger the
 /// callbacks of the FSEventStream we create. This is not in use outside of `wait`, but is retained
 /// across `wait` calls for simplicity and efficiency.
-dispatch_queue: dispatch_queue_t,
+dispatch_queue: dispatch.queue_t,
 /// In theory, this field avoids race conditions. In practice, it is essentially unused at the time
 /// of writing. See the comment at the start of `wait` for details.
 since_event: FSEventStreamEventId,
 @@ -57,7 +57,7 @@ const ResolvedSymbols = struct {
         latency: CFTimeInterval,
         flags: FSEventStreamCreateFlags,
     ) callconv(.c) FSEventStreamRef,
-    FSEventStreamSetDispatchQueue: *const fn (stream: FSEventStreamRef, queue: dispatch_queue_t) callconv(.c) void,
+    FSEventStreamSetDispatchQueue: *const fn (stream: FSEventStreamRef, queue: dispatch.queue_t) callconv(.c) void,
     FSEventStreamStart: *const fn (stream: FSEventStreamRef) callconv(.c) bool,
     FSEventStreamStop: *const fn (stream: FSEventStreamRef) callconv(.c) void,
     FSEventStreamInvalidate: *const fn (stream: FSEventStreamRef) callconv(.c) void,
 @@ -80,7 +80,7 @@ const ResolvedSymbols = struct {
     kCFAllocatorUseContext: *const CFAllocatorRef,
 };
-pub fn init(cwd_path: []const u8) error{ OpenFrameworkFailed, MissingCoreServicesSymbol }!FsEvents {
+pub fn init(cwd_path: []const u8) error{ OpenFrameworkFailed, MissingCoreServicesSymbol, SystemResources }!FsEvents {
     var core_services = std.DynLib.open("/System/Library/Frameworks/CoreServices.framework/CoreServices") catch
         return error.OpenFrameworkFailed;
     errdefer core_services.close();
 @@ -96,8 +96,8 @@ pub fn init(cwd_path: []const u8) error{ OpenFrameworkFailed, MissingCoreService
         .paths_arena = .{},
         .watch_roots = &.{},
         .watch_paths = .empty,
-        .waiting_semaphore = dispatch_semaphore_create(0),
+        .waiting_semaphore = dispatch.semaphore_create(0) orelse return error.SystemResources,
-        .dispatch_queue = dispatch_queue_create("zig-watch", .SERIAL),
+        .dispatch_queue = dispatch.queue_create("zig-watch", .SERIAL()) orelse return error.SystemResources,
         // Not `.since_now`, because this means we can init `FsEvents` *before* we do work in order
         // to notice any changes which happened during said work.
         .since_event = resolved_symbols.FSEventsGetCurrentEventId(),
 @@ -106,8 +106,8 @@ pub fn init(cwd_path: []const u8) error{ OpenFrameworkFailed, MissingCoreService
 }
 pub fn deinit(fse: *FsEvents, gpa: Allocator, io: Io) void {
-    dispatch_release(fse.waiting_semaphore);
+    fse.waiting_semaphore.as_object().release();
-    dispatch_release(fse.dispatch_queue);
+    fse.dispatch_queue.as_object().release();
     fse.core_services.close(io);
     gpa.free(fse.watch_roots);
 @@ -275,9 +275,9 @@ pub fn wait(fse: *FsEvents, gpa: Allocator, timeout_ns: ?u64) error{ OutOfMemory
     defer rs.FSEventStreamInvalidate(event_stream);
     if (!rs.FSEventStreamStart(event_stream)) return error.StartFailed;
     defer rs.FSEventStreamStop(event_stream);
-    const result = dispatch_semaphore_wait(fse.waiting_semaphore, timeout: {
+    const result = fse.waiting_semaphore.wait(timeout: {
-        const ns = timeout_ns orelse break :timeout .forever;
+        const ns = timeout_ns orelse break :timeout .FOREVER;
-        break :timeout dispatch_time(.now, @intCast(ns));
+        break :timeout .time(.NOW, @intCast(ns));
     });
     return switch (result) {
         0 => .dirty,
 @@ -382,7 +382,7 @@ fn eventCallback(
     }
     if (any_dirty) {
         fse.since_event = rs.FSEventStreamGetLatestEventId(stream);
-        _ = dispatch_semaphore_signal(fse.waiting_semaphore);
+        _ = fse.waiting_semaphore.signal();
     }
 }
 fn dirStartsWith(path: []const u8, prefix: []const u8) bool {
 @@ -392,25 +392,6 @@ fn dirStartsWith(path: []const u8, prefix: []const u8) bool {
     return true; // `path` is `/foo/bar/...`, `prefix` is `/foo/bar`
 }
-const dispatch_time_t = enum(u64) {
-    now = 0,
-    forever = std.math.maxInt(u64),
-    _,
-};
-extern fn dispatch_time(base: dispatch_time_t, delta_ns: i64) dispatch_time_t;
-const dispatch_semaphore_t = *opaque {};
-extern fn dispatch_semaphore_create(value: isize) dispatch_semaphore_t;
-extern fn dispatch_semaphore_wait(dsema: dispatch_semaphore_t, timeout: dispatch_time_t) isize;
-extern fn dispatch_semaphore_signal(dsema: dispatch_semaphore_t) isize;
-const dispatch_queue_t = *opaque {};
-const dispatch_queue_attr_t = ?*opaque {
-    const SERIAL: dispatch_queue_attr_t = null;
-};
-extern fn dispatch_queue_create(label: [*:0]const u8, attr: dispatch_queue_attr_t) dispatch_queue_t;
-extern fn dispatch_release(object: *anyopaque) void;
 const CFAllocatorRef = ?*const opaque {};
 const CFArrayRef = *const opaque {};
 const CFStringRef = *const opaque {};
 @@ -489,6 +470,7 @@ const FSEventStreamEventFlags = packed struct(u32) {
     _: u24 = 0,
 };
+const dispatch = std.c.dispatch;
 const std = @import("std");
 const Io = std.Io;
 const assert = std.debug.assert;

lib/std/Io.zig+56-56

...	@@ -26,19 +26,17 @@ userdata: ?*anyopaque,	...	@@ -26,19 +26,17 @@ userdata: ?*anyopaque,
26	vtable: *const VTable,	26	vtable: *const VTable,
27		27
28	pub const Threaded = @import("Io/Threaded.zig");	28	pub const Threaded = @import("Io/Threaded.zig");
29	pub const Evented = switch (builtin.os.tag) {	29
30	.linux => switch (builtin.cpu.arch) {	30	pub const fiber = @import("Io/fiber.zig");
31	.x86_64, .aarch64 => IoUring,	31	pub const Evented = if (fiber.supported) switch (builtin.os.tag) {
32	else => void, // context-switching code not implemented yet	32	.linux => Uring,
33	},	33	.dragonfly, .freebsd, .netbsd, .openbsd => Kqueue,
34	.dragonfly, .freebsd, .netbsd, .openbsd, .driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => switch (builtin.cpu.arch) {	34	.driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => Dispatch,
35	.x86_64, .aarch64 => Kqueue,
36	else => void, // context-switching code not implemented yet
37	},
38	else => void,	35	else => void,
39	};	36	} else void; // context-switching code not implemented yet
		37	pub const Dispatch = @import("Io/Dispatch.zig");
40	pub const Kqueue = @import("Io/Kqueue.zig");	38	pub const Kqueue = @import("Io/Kqueue.zig");
41	pub const IoUring = @import("Io/IoUring.zig");	39	pub const Uring = @import("Io/Uring.zig");
42		40
43	pub const Reader = @import("Io/Reader.zig");	41	pub const Reader = @import("Io/Reader.zig");
44	pub const Writer = @import("Io/Writer.zig");	42	pub const Writer = @import("Io/Writer.zig");
...	@@ -51,6 +49,8 @@ pub const RwLock = @import("Io/RwLock.zig");	...	@@ -51,6 +49,8 @@ pub const RwLock = @import("Io/RwLock.zig");
51	pub const Semaphore = @import("Io/Semaphore.zig");	49	pub const Semaphore = @import("Io/Semaphore.zig");
52		50
53	pub const VTable = struct {	51	pub const VTable = struct {
		52	crashHandler: const fn (?anyopaque) void,
		53
54	/// If it returns `null` it means `result` has been already populated and	54	/// If it returns `null` it means `result` has been already populated and
55	/// `await` will be a no-op.	55	/// `await` will be a no-op.
56	///	56	///
...	@@ -378,9 +378,9 @@ pub const Operation = union(enum) {	...	@@ -378,9 +378,9 @@ pub const Operation = union(enum) {
378	pub const Pending = struct {	378	pub const Pending = struct {
379	node: List.DoubleNode,	379	node: List.DoubleNode,
380	tag: Tag,	380	tag: Tag,
381	context: Context align(@max(@alignOf(usize), 4)),	381	userdata: Userdata align(@max(@alignOf(usize), 4)),
382		382
383	pub const Context = [3]usize;	383	pub const Userdata = [7]usize;
384	};	384	};
385		385
386	pub const Completion = struct {	386	pub const Completion = struct {
...	@@ -431,7 +431,7 @@ pub const Batch = struct {	...	@@ -431,7 +431,7 @@ pub const Batch = struct {
431	submitted: Operation.List,	431	submitted: Operation.List,
432	pending: Operation.List,	432	pending: Operation.List,
433	completed: Operation.List,	433	completed: Operation.List,
434	context: ?anyopaque align(@max(@alignOf(?anyopaque), 4)),	434	userdata: ?anyopaque align(@max(@alignOf(?anyopaque), 4)),
435		435
436	/// After calling this, it is safe to unconditionally defer a call to	436	/// After calling this, it is safe to unconditionally defer a call to
437	/// `cancel`. `storage` is a pre-allocated buffer of undefined memory that	437	/// `cancel`. `storage` is a pre-allocated buffer of undefined memory that
...	@@ -453,40 +453,40 @@ pub const Batch = struct {	...	@@ -453,40 +453,40 @@ pub const Batch = struct {
453	.submitted = .empty,	453	.submitted = .empty,
454	.pending = .empty,	454	.pending = .empty,
455	.completed = .empty,	455	.completed = .empty,
456	.context = null,	456	.userdata = null,
457	};	457	};
458	}	458	}
459		459
460	/// Adds an operation to be performed at the next await call.	460	/// Adds an operation to be performed at the next await call.
461	/// Returns the index that will be returned by `next` after the operation completes.	461	/// Returns the index that will be returned by `next` after the operation completes.
462	/// Asserts that no more than `storage.len` operations are active at a time.	462	/// Asserts that no more than `storage.len` operations are active at a time.
463	pub fn add(b: *Batch, operation: Operation) u32 {	463	pub fn add(batch: *Batch, operation: Operation) u32 {
464	const index = b.unused.next;	464	const index = batch.unused.next;
465	b.addAt(index.toIndex(), operation);	465	batch.addAt(index.toIndex(), operation);
466	return index;	466	return index;
467	}	467	}
468		468
469	/// Adds an operation to be performed at the next await call.	469	/// Adds an operation to be performed at the next await call.
470	/// After the operation completes, `next` will return `index`.	470	/// After the operation completes, `next` will return `index`.
471	/// Asserts that the operation at `index` is not active.	471	/// Asserts that the operation at `index` is not active.
472	pub fn addAt(b: *Batch, index: u32, operation: Operation) void {	472	pub fn addAt(batch: *Batch, index: u32, operation: Operation) void {
473	const storage = &b.storage[index];	473	const storage = &batch.storage[index];
474	const unused = storage.unused;	474	const unused = storage.unused;
475	switch (unused.prev) {	475	switch (unused.prev) {
476	.none => b.unused.head = unused.next,	476	.none => batch.unused.head = unused.next,
477	else => \|prev_index\| b.storage[prev_index.toIndex()].unused.next = unused.next,	477	else => \|prev_index\| batch.storage[prev_index.toIndex()].unused.next = unused.next,
478	}	478	}
479	switch (unused.next) {	479	switch (unused.next) {
480	.none => b.unused.tail = unused.prev,	480	.none => batch.unused.tail = unused.prev,
481	else => \|next_index\| b.storage[next_index.toIndex()].unused.prev = unused.prev,	481	else => \|next_index\| batch.storage[next_index.toIndex()].unused.prev = unused.prev,
482	}	482	}
483		483
484	switch (b.submitted.tail) {	484	switch (batch.submitted.tail) {
485	.none => b.submitted.head = .fromIndex(index),	485	.none => batch.submitted.head = .fromIndex(index),
486	else => \|tail_index\| b.storage[tail_index.toIndex()].submission.node.next = .fromIndex(index),	486	else => \|tail_index\| batch.storage[tail_index.toIndex()].submission.node.next = .fromIndex(index),
487	}	487	}
488	storage.* = .{ .submission = .{ .node = .{ .next = .none }, .operation = operation } };	488	storage.* = .{ .submission = .{ .node = .{ .next = .none }, .operation = operation } };
489	b.submitted.tail = .fromIndex(index);	489	batch.submitted.tail = .fromIndex(index);
490	}	490	}
491		491
492	pub const Completion = struct {	492	pub const Completion = struct {
...	@@ -502,22 +502,22 @@ pub const Batch = struct {	...	@@ -502,22 +502,22 @@ pub const Batch = struct {
502	///	502	///
503	/// Each completion returned from this function dequeues from the `Batch`.	503	/// Each completion returned from this function dequeues from the `Batch`.
504	/// It is not required to dequeue all completions before awaiting again.	504	/// It is not required to dequeue all completions before awaiting again.
505	pub fn next(b: *Batch) ?Completion {	505	pub fn next(batch: *Batch) ?Completion {
506	const index = b.completed.head;	506	const index = batch.completed.head;
507	if (index == .none) return null;	507	if (index == .none) return null;
508	const storage = &b.storage[index.toIndex()];	508	const storage = &batch.storage[index.toIndex()];
509	const completion = storage.completion;	509	const completion = storage.completion;
510	const next_index = completion.node.next;	510	const next_index = completion.node.next;
511	b.completed.head = next_index;	511	batch.completed.head = next_index;
512	if (next_index == .none) b.completed.tail = .none;	512	if (next_index == .none) batch.completed.tail = .none;
513		513
514	const tail_index = b.unused.tail;	514	const tail_index = batch.unused.tail;
515	switch (tail_index) {	515	switch (tail_index) {
516	.none => b.unused.head = index,	516	.none => batch.unused.head = index,
517	else => b.storage[tail_index.toIndex()].unused.next = index,	517	else => batch.storage[tail_index.toIndex()].unused.next = index,
518	}	518	}
519	storage.* = .{ .unused = .{ .prev = tail_index, .next = .none } };	519	storage.* = .{ .unused = .{ .prev = tail_index, .next = .none } };
520	b.unused.tail = index;	520	batch.unused.tail = index;
521	return .{ .index = index.toIndex(), .result = completion.result };	521	return .{ .index = index.toIndex(), .result = completion.result };
522	}	522	}
523		523
...	@@ -529,8 +529,8 @@ pub const Batch = struct {	...	@@ -529,8 +529,8 @@ pub const Batch = struct {
529	/// concurrency into the batched operations, but unlike `awaitConcurrent`,	529	/// concurrency into the batched operations, but unlike `awaitConcurrent`,
530	/// does not require it, and therefore cannot fail with	530	/// does not require it, and therefore cannot fail with
531	/// `error.ConcurrencyUnavailable`.	531	/// `error.ConcurrencyUnavailable`.
532	pub fn awaitAsync(b: *Batch, io: Io) Cancelable!void {	532	pub fn awaitAsync(batch: *Batch, io: Io) Cancelable!void {
533	return io.vtable.batchAwaitAsync(io.userdata, b);	533	return io.vtable.batchAwaitAsync(io.userdata, batch);
534	}	534	}
535		535
536	pub const AwaitConcurrentError = ConcurrentError \|\| Cancelable \|\| Timeout.Error;	536	pub const AwaitConcurrentError = ConcurrentError \|\| Cancelable \|\| Timeout.Error;
...	@@ -542,8 +542,8 @@ pub const Batch = struct {	...	@@ -542,8 +542,8 @@ pub const Batch = struct {
542	/// Unlike `awaitAsync`, this function requires the implementation to	542	/// Unlike `awaitAsync`, this function requires the implementation to
543	/// perform the operations concurrently and therefore can fail with	543	/// perform the operations concurrently and therefore can fail with
544	/// `error.ConcurrencyUnavailable`.	544	/// `error.ConcurrencyUnavailable`.
545	pub fn awaitConcurrent(b: *Batch, io: Io, timeout: Timeout) AwaitConcurrentError!void {	545	pub fn awaitConcurrent(batch: *Batch, io: Io, timeout: Timeout) AwaitConcurrentError!void {
546	return io.vtable.batchAwaitConcurrent(io.userdata, b, timeout);	546	return io.vtable.batchAwaitConcurrent(io.userdata, batch, timeout);
547	}	547	}
548		548
549	/// Requests all pending operations to be interrupted, then waits for all	549	/// Requests all pending operations to be interrupted, then waits for all
...	@@ -552,28 +552,28 @@ pub const Batch = struct {	...	@@ -552,28 +552,28 @@ pub const Batch = struct {
552	/// canceled operations will be absent from the iteration. Some operations	552	/// canceled operations will be absent from the iteration. Some operations
553	/// may have successfully completed regardless of the cancel request and	553	/// may have successfully completed regardless of the cancel request and
554	/// will appear in the iteration.	554	/// will appear in the iteration.
555	pub fn cancel(b: *Batch, io: Io) void {	555	pub fn cancel(batch: *Batch, io: Io) void {
556	{ // abort pending submissions	556	{ // abort pending submissions
557	var tail_index = b.unused.tail;	557	var tail_index = batch.unused.tail;
558	defer b.unused.tail = tail_index;	558	defer batch.unused.tail = tail_index;
559	var index = b.submitted.head;	559	var index = batch.submitted.head;
560	errdefer b.submissions.head = index;	560	errdefer batch.submissions.head = index;
561	while (index != .none) {	561	while (index != .none) {
562	const next_index = b.storage[index.toIndex()].submission.node.next;	562	const next_index = batch.storage[index.toIndex()].submission.node.next;
563	switch (tail_index) {	563	switch (tail_index) {
564	.none => b.unused.head = index,	564	.none => batch.unused.head = index,
565	else => b.storage[tail_index.toIndex()].unused.next = index,	565	else => batch.storage[tail_index.toIndex()].unused.next = index,
566	}	566	}
567	b.storage[index.toIndex()] = .{ .unused = .{ .prev = tail_index, .next = .none } };	567	batch.storage[index.toIndex()] = .{ .unused = .{ .prev = tail_index, .next = .none } };
568	tail_index = index;	568	tail_index = index;
569	index = next_index;	569	index = next_index;
570	}	570	}
571	b.submitted = .{ .head = .none, .tail = .none };	571	batch.submitted = .{ .head = .none, .tail = .none };
572	}	572	}
573	io.vtable.batchCancel(io.userdata, b);	573	io.vtable.batchCancel(io.userdata, batch);
574	assert(b.submitted.head == .none and b.submitted.tail == .none);	574	assert(batch.submitted.head == .none and batch.submitted.tail == .none);
575	assert(b.pending.head == .none and b.pending.tail == .none);	575	assert(batch.pending.head == .none and batch.pending.tail == .none);
576	assert(b.context == null); // that was the last chance to deallocate resources	576	assert(batch.userdata == null); // that was the last chance to deallocate resources
577	}	577	}
578	};	578	};
579		579
...	@@ -643,7 +643,7 @@ pub const Limit = enum(usize) {	...	@@ -643,7 +643,7 @@ pub const Limit = enum(usize) {
643	}	643	}
644		644
645	pub fn nonzero(l: Limit) bool {	645	pub fn nonzero(l: Limit) bool {
646	return @intFromEnum(l) > 0;	646	return l != .nothing;
647	}	647	}
648		648
649	/// Return a new limit reduced by `amount` or return `null` indicating	649	/// Return a new limit reduced by `amount` or return `null` indicating

lib/std/Io/Dispatch.zig created+5038

...	@@ -0,0 +1,5038 @@
1	const Alignment = std.mem.Alignment;
2	const Allocator = std.mem.Allocator;
3	const Argv0 = Io.Threaded.Argv0;
4	const assert = std.debug.assert;
5	const builtin = @import("builtin");
6	const c = std.c;
7	const ChdirError = Io.Threaded.ChdirError;
8	const clockToPosix = Io.Threaded.clockToPosix;
9	const closeFd = Io.Threaded.closeFd;
10	const Csprng = Io.Threaded.Csprng;
11	const default_PATH = Io.Threaded.default_PATH;
12	const Dir = Io.Dir;
13	const Environ = Io.Threaded.Environ;
14	const errnoBug = Io.Threaded.errnoBug;
15	const Evented = @This();
16	const fallbackSeed = Io.Threaded.fallbackSeed;
17	const File = Io.File;
18	const Io = std.Io;
19	const iovec = std.posix.iovec;
20	const iovec_const = std.posix.iovec_const;
21	const log = std.log.scoped(.dispatch);
22	const max_iovecs_len = Io.Threaded.max_iovecs_len;
23	const nanosecondsFromPosix = Io.Threaded.nanosecondsFromPosix;
24	const net = Io.net;
25	const pathToPosix = Io.Threaded.pathToPosix;
26	const process = std.process;
27	const recoverableOsBugDetected = Io.Threaded.recoverableOsBugDetected;
28	const setTimestampToPosix = Io.Threaded.setTimestampToPosix;
29	const splat_buffer_size = Io.Threaded.splat_buffer_size;
30	const statFromPosix = Io.Threaded.statFromPosix;
31	const statusToTerm = Io.Threaded.statusToTerm;
32	const std = @import("std");
33	const timestampFromPosix = Io.Threaded.timestampFromPosix;
34	const unexpectedErrno = std.posix.unexpectedErrno;
35	const UseSendfile = Io.Threaded.UseSendfile;
36	const UseFcopyfile = Io.Threaded.UseFcopyfile;
37
38	/// Empirically saw >4KB being used by the llvm aarch64 backend.
39	const main_loop_stack_size = 8 * 1024;
40
41	queue: c.dispatch.queue_t,
42	backing_allocator_needs_mutex: bool,
43	backing_allocator_mutex: Mutex,
44	/// Does not need to be thread-safe if not used elsewhere.
45	backing_allocator: Allocator,
46	main_fiber: Fiber,
47	main_loop_stack: [*]align(builtin.target.stackAlignment()) u8,
48	exit_semaphore: c.dispatch.semaphore_t,
49
50	use_sendfile: UseSendfile,
51	use_fcopyfile: UseFcopyfile,
52	leeway: u64,
53
54	futexes: [1 << 8]Futex,
55
56	init_stderr_writer: c.dispatch.once_t,
57	stderr_mutex: Mutex,
58	stderr_writer: File.Writer,
59	stderr_mode: Io.Terminal.Mode,
60
61	scan_environ: c.dispatch.once_t,
62	environ: Environ,
63
64	open_dev_null: c.dispatch.once_t,
65	dev_null_file: File.OpenError!File,
66
67	csprng_mutex: Mutex,
68	csprng: Csprng,
69
70	const Thread = struct {
71	main_context: Io.fiber.Context,
72	current_context: ?*Io.fiber.Context,
73	seed_csprng: c.dispatch.once_t,
74	csprng: Csprng,
75
76	threadlocal var self: Thread = .{
77	.main_context = undefined,
78	.current_context = null,
79	.seed_csprng = .init,
80	.csprng = undefined,
81	};
82
83	noinline fn current() *Thread {
84	return &self;
85	}
86
87	fn currentFiber(thread: Thread) Fiber {
88	assert(thread.current_context != &thread.main_context);
89	return @fieldParentPtr("context", thread.current_context.?);
90	}
91
92	const List = struct {
93	allocated: []Thread,
94	reserved: u32,
95	active: u32,
96	};
97	};
98
99	const Fiber = struct {
100	required_align: void align(4),
101	evented: *Evented,
102	context: Io.fiber.Context,
103	await_count: i32,
104	link: union {
105	awaiter: ?*Fiber,
106	group: struct { prev: ?Fiber, next: ?Fiber },
107	},
108	status: union(enum) {
109	queue_next: ?*Fiber,
110	awaiting_group: Group,
111	},
112	cancel_status: CancelStatus,
113	cancel_protection: CancelProtection,
114
115	var next_name: u64 = 0;
116
117	const CancelStatus = packed struct(usize) {
118	requested: bool,
119	awaiting: Awaiting,
120
121	const unrequested: CancelStatus = .{ .requested = false, .awaiting = .nothing };
122
123	const Awaiting = enum(@Int(.unsigned, @bitSizeOf(usize) - shift)) {
124	nothing = 0,
125	group = 1,
126	select = 2,
127	_,
128
129	const shift = 1;
130
131	fn subWrap(lhs: Awaiting, rhs: Awaiting) Awaiting {
132	return @enumFromInt(@intFromEnum(lhs) -% @intFromEnum(rhs));
133	}
134
135	fn fromCancelable(cancelable: *Cancelable) Awaiting {
136	return @enumFromInt(@shrExact(@intFromPtr(cancelable), shift));
137	}
138
139	fn toCancelable(awaiting: Awaiting) *Cancelable {
140	return @ptrFromInt(@shlExact(@as(usize, @intFromEnum(awaiting)), shift));
141	}
142	};
143
144	fn changeAwaiting(
145	cancel_status: *CancelStatus,
146	old_awaiting: Awaiting,
147	new_awaiting: Awaiting,
148	) bool {
149	const old_cancel_status = @atomicRmw(CancelStatus, cancel_status, .Add, .{
150	.requested = false,
151	.awaiting = new_awaiting.subWrap(old_awaiting),
152	}, .release);
153	assert(old_cancel_status.awaiting == old_awaiting);
154	return old_cancel_status.requested;
155	}
156	};
157
158	const CancelProtection = packed struct {
159	user: Io.CancelProtection,
160	acknowledged: bool,
161
162	const unblocked: CancelProtection = .{ .user = .unblocked, .acknowledged = false };
163
164	fn check(cancel_protection: CancelProtection) Io.CancelProtection {
165	return @enumFromInt(@intFromBool(cancel_protection != unblocked));
166	}
167
168	fn acknowledge(cancel_protection: *CancelProtection) void {
169	assert(!cancel_protection.acknowledged);
170	cancel_protection.acknowledged = true;
171	}
172
173	fn recancel(cancel_protection: *CancelProtection) void {
174	assert(cancel_protection.acknowledged);
175	cancel_protection.acknowledged = false;
176	}
177
178	test check {
179	try std.testing.expectEqual(Io.CancelProtection.unblocked, check(.unblocked));
180	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
181	.user = .unblocked,
182	.acknowledged = true,
183	}));
184	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
185	.user = .blocked,
186	.acknowledged = false,
187	}));
188	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
189	.user = .blocked,
190	.acknowledged = true,
191	}));
192	}
193	};
194
195	const finished: ?*Fiber = @ptrFromInt(@alignOf(Fiber));
196
197	const max_result_align: Alignment = .@"16";
198	const max_result_size = max_result_align.forward(512);
199	/// This includes any stack realignments that need to happen, and also the
200	/// initial frame return address slot and argument frame, depending on target.
201	const min_stack_size = 60 * 1024 * 1024;
202	const max_context_align: Alignment = .@"16";
203	const max_context_size = max_context_align.forward(1024);
204	const max_closure_size: usize = @sizeOf(AsyncClosure);
205	const max_closure_align: Alignment = .of(AsyncClosure);
206	const allocation_size = std.mem.alignForward(
207	usize,
208	max_closure_align.max(max_context_align).forward(
209	max_result_align.forward(@sizeOf(Fiber)) + max_result_size + min_stack_size,
210	) + max_closure_size + max_context_size,
211	std.heap.page_size_max,
212	);
213
214	fn create(ev: Evented) error{OutOfMemory}!Fiber {
215	return @ptrCast(try ev.allocator().alignedAlloc(u8, .of(Fiber), allocation_size));
216	}
217
218	fn destroy(fiber: Fiber, ev: Evented) void {
219	assert(fiber.status.queue_next == null);
220	ev.allocator().free(fiber.allocatedSlice());
221	}
222
223	fn allocatedSlice(f: *Fiber) []align(@alignOf(Fiber)) u8 {
224	return @as([*]align(@alignOf(Fiber)) u8, @ptrCast(f))[0..allocation_size];
225	}
226
227	fn allocatedEnd(f: Fiber) []u8 {
228	const allocated_slice = f.allocatedSlice();
229	return allocated_slice[allocated_slice.len..].ptr;
230	}
231
232	fn resultPointer(f: Fiber, comptime Result: type) Result {
233	return @ptrCast(@alignCast(f.resultBytes(.of(Result))));
234	}
235
236	fn resultBytes(f: Fiber, alignment: Alignment) []u8 {
237	return @ptrFromInt(alignment.forward(@intFromPtr(f) + @sizeOf(Fiber)));
238	}
239
240	const Queue = struct { head: Fiber, tail: Fiber };
241
242	/// Like a `*Fiber`, but 2 bits smaller than a pointer (because the LSBs are always 0 due to
243	/// alignment) so that those two bits can be used in a `packed struct`.
244	const PackedPtr = enum(@Int(.unsigned, @bitSizeOf(usize) - 2)) {
245	null = 0,
246	all_ones = std.math.maxInt(@Int(.unsigned, @bitSizeOf(usize) - 2)),
247	_,
248
249	const Split = packed struct(usize) { low: u2, high: PackedPtr };
250	fn pack(ptr: ?*Fiber) PackedPtr {
251	const split: Split = @bitCast(@intFromPtr(ptr));
252	assert(split.low == 0);
253	return split.high;
254	}
255	fn unpack(ptr: PackedPtr) ?*Fiber {
256	const split: Split = .{ .low = 0, .high = ptr };
257	return @ptrFromInt(@as(usize, @bitCast(split)));
258	}
259	};
260
261	fn requestCancel(fiber: Fiber, ev: Evented) void {
262	const cancel_status = @atomicRmw(
263	Fiber.CancelStatus,
264	&fiber.cancel_status,
265	.Or,
266	.{ .requested = true, .awaiting = .nothing },
267	.acquire,
268	);
269	assert(!cancel_status.requested);
270	switch (cancel_status.awaiting) {
271	.nothing => {},
272	.group => {
273	// The awaiter received a cancelation request while awaiting a group,
274	// so propagate the cancelation to the group.
275	if (fiber.status.awaiting_group.cancel(ev, null)) {
276	fiber.status = .{ .queue_next = null };
277	ev.queue.async(fiber, &Fiber.@"resume");
278	}
279	},
280	.select => if (@atomicRmw(i32, &fiber.await_count, .Add, 1, .monotonic) == -1) {
281	ev.queue.async(fiber, &Fiber.@"resume");
282	},
283	_ => \|awaiting\| awaiting.toCancelable().canceled(),
284	}
285	}
286
287	fn @"resume"(context: ?*anyopaque) callconv(.c) void {
288	const fiber: *Fiber = @ptrCast(@alignCast(context));
289	const thread: *Thread = .current();
290	const message: SwitchMessage = .{
291	.contexts = .{
292	.old = &thread.main_context,
293	.new = &fiber.context,
294	},
295	.pending_task = .nothing,
296	};
297	contextSwitch(&message).handle(fiber.evented);
298	}
299	};
300
301	pub fn allocator(ev: *Evented) std.mem.Allocator {
302	return if (ev.backing_allocator_needs_mutex) .{
303	.ptr = ev,
304	.vtable = &.{
305	.alloc = alloc,
306	.resize = resize,
307	.remap = remap,
308	.free = free,
309	},
310	} else ev.backing_allocator;
311	}
312
313	fn alloc(userdata: anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[]u8 {
314	const ev: *Evented = @ptrCast(@alignCast(userdata));
315	ev.backing_allocator_mutex.lockUncancelable(ev);
316	defer ev.backing_allocator_mutex.unlock();
317	return ev.backing_allocator.rawAlloc(len, alignment, ret_addr);
318	}
319
320	fn resize(
321	userdata: *anyopaque,
322	memory: []u8,
323	alignment: std.mem.Alignment,
324	new_len: usize,
325	ret_addr: usize,
326	) bool {
327	const ev: *Evented = @ptrCast(@alignCast(userdata));
328	ev.backing_allocator_mutex.lockUncancelable(ev);
329	defer ev.backing_allocator_mutex.unlock();
330	return ev.backing_allocator.rawResize(memory, alignment, new_len, ret_addr);
331	}
332
333	fn remap(
334	userdata: *anyopaque,
335	memory: []u8,
336	alignment: Alignment,
337	new_len: usize,
338	ret_addr: usize,
339	) ?[*]u8 {
340	const ev: *Evented = @ptrCast(@alignCast(userdata));
341	ev.backing_allocator_mutex.lockUncancelable(ev);
342	defer ev.backing_allocator_mutex.unlock();
343	return ev.backing_allocator.rawRemap(memory, alignment, new_len, ret_addr);
344	}
345
346	fn free(userdata: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void {
347	const ev: *Evented = @ptrCast(@alignCast(userdata));
348	ev.backing_allocator_mutex.lockUncancelable(ev);
349	defer ev.backing_allocator_mutex.unlock();
350	return ev.backing_allocator.rawFree(memory, alignment, ret_addr);
351	}
352
353	pub fn io(ev: *Evented) Io {
354	return .{
355	.userdata = ev,
356	.vtable = &.{
357	.crashHandler = crashHandler,
358
359	.async = async,
360	.concurrent = concurrent,
361	.await = await,
362	.cancel = cancel,
363
364	.groupAsync = groupAsync,
365	.groupConcurrent = groupConcurrent,
366	.groupAwait = groupAwait,
367	.groupCancel = groupCancel,
368
369	.recancel = recancel,
370	.swapCancelProtection = swapCancelProtection,
371	.checkCancel = checkCancel,
372
373	.select = select,
374
375	.futexWait = futexWait,
376	.futexWaitUncancelable = futexWaitUncancelable,
377	.futexWake = futexWake,
378
379	.operate = operate,
380	.batchAwaitAsync = batchAwaitAsync,
381	.batchAwaitConcurrent = batchAwaitConcurrent,
382	.batchCancel = batchCancel,
383
384	.dirCreateDir = dirCreateDir,
385	.dirCreateDirPath = dirCreateDirPath,
386	.dirCreateDirPathOpen = dirCreateDirPathOpen,
387	.dirOpenDir = dirOpenDir,
388	.dirStat = dirStat,
389	.dirStatFile = dirStatFile,
390	.dirAccess = dirAccess,
391	.dirCreateFile = dirCreateFile,
392	.dirCreateFileAtomic = dirCreateFileAtomic,
393	.dirOpenFile = dirOpenFile,
394	.dirClose = dirClose,
395	.dirRead = dirRead,
396	.dirRealPath = dirRealPath,
397	.dirRealPathFile = dirRealPathFile,
398	.dirDeleteFile = dirDeleteFile,
399	.dirDeleteDir = dirDeleteDir,
400	.dirRename = dirRename,
401	.dirRenamePreserve = dirRenamePreserve,
402	.dirSymLink = dirSymLink,
403	.dirReadLink = dirReadLink,
404	.dirSetOwner = dirSetOwner,
405	.dirSetFileOwner = dirSetFileOwner,
406	.dirSetPermissions = dirSetPermissions,
407	.dirSetFilePermissions = dirSetFilePermissions,
408	.dirSetTimestamps = dirSetTimestamps,
409	.dirHardLink = dirHardLink,
410
411	.fileStat = fileStat,
412	.fileLength = fileLength,
413	.fileClose = fileClose,
414	.fileWritePositional = fileWritePositional,
415	.fileWriteFileStreaming = fileWriteFileStreaming,
416	.fileWriteFilePositional = fileWriteFilePositional,
417	.fileReadPositional = fileReadPositional,
418	.fileSeekBy = fileSeekBy,
419	.fileSeekTo = fileSeekTo,
420	.fileSync = fileSync,
421	.fileIsTty = fileIsTty,
422	.fileEnableAnsiEscapeCodes = fileEnableAnsiEscapeCodes,
423	.fileSupportsAnsiEscapeCodes = fileIsTty,
424	.fileSetLength = fileSetLength,
425	.fileSetOwner = fileSetOwner,
426	.fileSetPermissions = fileSetPermissions,
427	.fileSetTimestamps = fileSetTimestamps,
428	.fileLock = fileLock,
429	.fileTryLock = fileTryLock,
430	.fileUnlock = fileUnlock,
431	.fileDowngradeLock = fileDowngradeLock,
432	.fileRealPath = fileRealPath,
433	.fileHardLink = fileHardLink,
434
435	.fileMemoryMapCreate = fileMemoryMapCreate,
436	.fileMemoryMapDestroy = fileMemoryMapDestroy,
437	.fileMemoryMapSetLength = fileMemoryMapSetLength,
438	.fileMemoryMapRead = fileMemoryMapRead,
439	.fileMemoryMapWrite = fileMemoryMapWrite,
440
441	.processExecutableOpen = processExecutableOpen,
442	.processExecutablePath = processExecutablePath,
443	.lockStderr = lockStderr,
444	.tryLockStderr = tryLockStderr,
445	.unlockStderr = unlockStderr,
446	.processCurrentPath = processCurrentPath,
447	.processSetCurrentDir = processSetCurrentDir,
448	.processReplace = processReplace,
449	.processReplacePath = processReplacePath,
450	.processSpawn = processSpawn,
451	.processSpawnPath = processSpawnPath,
452	.childWait = childWait,
453	.childKill = childKill,
454
455	.progressParentFile = progressParentFile,
456
457	.now = now,
458	.clockResolution = clockResolution,
459	.sleep = sleep,
460
461	.random = random,
462	.randomSecure = randomSecure,
463
464	.netListenIp = netListenIpUnavailable,
465	.netAccept = netAcceptUnavailable,
466	.netBindIp = netBindIpUnavailable,
467	.netConnectIp = netConnectIpUnavailable,
468	.netListenUnix = netListenUnixUnavailable,
469	.netConnectUnix = netConnectUnixUnavailable,
470	.netSocketCreatePair = netSocketCreatePairUnavailable,
471	.netSend = netSendUnavailable,
472	.netReceive = netReceiveUnavailable,
473	.netRead = netReadUnavailable,
474	.netWrite = netWriteUnavailable,
475	.netWriteFile = netWriteFileUnavailable,
476	.netClose = netClose,
477	.netShutdown = netShutdownUnavailable,
478	.netInterfaceNameResolve = netInterfaceNameResolveUnavailable,
479	.netInterfaceName = netInterfaceNameUnavailable,
480	.netLookup = netLookupUnavailable,
481	},
482	};
483	}
484
485	pub const InitOptions = struct {
486	backing_allocator_needs_mutex: bool = true,
487	queue: ?c.dispatch.queue_t = null,
488	/// Upper limit on the allowable delay in processing timeouts in order to improve power
489	/// consumption and system performance.
490	leeway: Io.Duration = .fromMilliseconds(10),
491
492	/// Affects the following operations:
493	/// * `processExecutablePath` on OpenBSD and Haiku.
494	argv0: Argv0 = .empty,
495	/// Affects the following operations:
496	/// * `fileIsTty`
497	/// * `processSpawn`, `processSpawnPath`, `processReplace`, `processReplacePath`
498	environ: process.Environ = .empty,
499	};
500
501	pub fn init(ev: *Evented, backing_allocator: Allocator, options: InitOptions) !void {
502	const queue = if (options.queue) \|queue\| queue: {
503	queue.as_object().retain();
504	break :queue queue;
505	} else c.dispatch.queue_create("org.ziglang.std.Io.Dispatch", .CONCURRENT()) orelse
506	return error.SystemResources;
507	errdefer queue.as_object().release();
508	const main_loop_stack = try backing_allocator.alignedAlloc(
509	u8,
510	.fromByteUnits(builtin.target.stackAlignment()),
511	main_loop_stack_size,
512	);
513	errdefer backing_allocator.free(main_loop_stack);
514	const exit_semaphore = c.dispatch.semaphore_create(0) orelse return error.SystemResources;
515	errdefer exit_semaphore.as_object().release();
516	ev.* = .{
517	.queue = queue,
518	.backing_allocator_needs_mutex = options.backing_allocator_needs_mutex,
519	.backing_allocator_mutex = undefined,
520	.backing_allocator = backing_allocator,
521	.main_fiber = .{
522	.required_align = {},
523	.evented = ev,
524	.context = undefined,
525	.await_count = 0,
526	.link = .{ .awaiter = null },
527	.status = .{ .queue_next = null },
528	.cancel_status = .unrequested,
529	.cancel_protection = .unblocked,
530	},
531	.main_loop_stack = main_loop_stack.ptr,
532	.exit_semaphore = exit_semaphore,
533
534	.use_fcopyfile = .default,
535	.use_sendfile = .default,
536	.leeway = std.math.lossyCast(u64, options.leeway.toNanoseconds()),
537
538	.futexes = undefined,
539
540	.init_stderr_writer = .init,
541	.stderr_mutex = undefined,
542	.stderr_writer = .{
543	.io = ev.io(),
544	.interface = Io.File.Writer.initInterface(&.{}),
545	.file = .stderr(),
546	.mode = .streaming,
547	},
548	.stderr_mode = .no_color,
549
550	.scan_environ = if (options.environ.block.isEmpty()) .done else .init,
551	.environ = .{ .process_environ = options.environ },
552
553	.open_dev_null = .init,
554	.dev_null_file = error.FileNotFound,
555
556	.csprng_mutex = undefined,
557	.csprng = .uninitialized,
558	};
559	try ev.backing_allocator_mutex.init(queue);
560	errdefer ev.backing_allocator_mutex.deinit();
561	var initialized_futexes: usize = 0;
562	errdefer for (ev.futexes[0..initialized_futexes]) \|*futex\| futex.deinit();
563	for (&ev.futexes) \|*futex\| {
564	try futex.init(queue);
565	initialized_futexes += 1;
566	}
567	try ev.stderr_mutex.init(queue);
568	errdefer ev.stderr_mutex.deinit();
569	try ev.csprng_mutex.init(queue);
570	errdefer ev.csprng_mutex.deinit();
571	const thread: *Thread = .current();
572	thread.main_context = switch (builtin.cpu.arch) {
573	.aarch64 => .{
574	.sp = @intFromPtr(main_loop_stack[main_loop_stack_size..].ptr),
575	.fp = @intFromPtr(ev),
576	.pc = @intFromPtr(&mainLoopEntry),
577	},
578	.x86_64 => .{
579	.rsp = @intFromPtr(main_loop_stack[main_loop_stack_size..].ptr) - 8,
580	.rbp = @intFromPtr(ev),
581	.rip = @intFromPtr(&mainLoopEntry),
582	},
583	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
584	};
585	thread.current_context = &ev.main_fiber.context;
586	}
587
588	pub fn deinit(ev: *Evented) void {
589	assert(Thread.current().currentFiber() == &ev.main_fiber);
590	ev.yield(.exit);
591	ev.csprng_mutex.deinit();
592	if (ev.dev_null_file) \|file\| fileClose(ev, &.{file}) else \|_\| {}
593	ev.stderr_mutex.deinit();
594	for (&ev.futexes) \|*futex\| futex.deinit();
595	ev.exit_semaphore.as_object().release();
596	ev.backing_allocator.free(ev.main_loop_stack[0..main_loop_stack_size]);
597	ev.queue.as_object().release();
598	}
599
600	fn yield(ev: *Evented, pending_task: SwitchMessage.PendingTask) void {
601	const thread: *Thread = .current();
602	const message: SwitchMessage = .{
603	.contexts = .{
604	.old = thread.current_context.?,
605	.new = &thread.main_context,
606	},
607	.pending_task = pending_task,
608	};
609	contextSwitch(&message).handle(ev);
610	}
611
612	fn mainLoopEntry() callconv(.naked) void {
613	switch (builtin.cpu.arch) {
614	.aarch64 => asm volatile (
615	\\ mov x0, fp
616	\\ mov fp, #0
617	\\ b %[mainLoop]
618	:
619	: [mainLoop] "X" (&mainLoop),
620	),
621	.x86_64 => asm volatile (
622	\\ movq %%rbp, %%rdi
623	\\ xor %%ebp, %%ebp
624	\\ jmp %[mainLoop:P]
625	:
626	: [mainLoop] "X" (&mainLoop),
627	),
628	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
629	}
630	}
631
632	fn mainLoop(ev: Evented, message: const SwitchMessage) callconv(.c) noreturn {
633	message.handle(ev);
634	assert(ev.exit_semaphore.wait(.FOREVER) == 0);
635	Fiber.@"resume"(&ev.main_fiber);
636	unreachable; // switched to dead fiber
637	}
638
639	const SwitchMessage = struct {
640	contexts: Io.fiber.Switch,
641	pending_task: PendingTask,
642
643	const PendingTask = union(enum) {
644	nothing,
645	await: u31,
646	activate: c.dispatch.object_t,
647	@"resume": c.dispatch.object_t,
648	group_await: Group,
649	group_cancel: Group,
650	mutex_wait: *Mutex.Waiter,
651	futex_wait: *Futex.Waiter,
652	futex_wake: *Futex.Waker,
653	sleep: c.dispatch.time_t,
654	destroy,
655	exit,
656	};
657
658	fn handle(message: const SwitchMessage, ev: Evented) void {
659	const thread: *Thread = .current();
660	thread.current_context = message.contexts.new;
661	switch (message.pending_task) {
662	.nothing => {},
663	.await => \|count\| {
664	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
665	if (@atomicRmw(i32, &fiber.await_count, .Sub, count, .monotonic) > 0)
666	ev.queue.async(fiber, &Fiber.@"resume");
667	},
668	.activate => \|object\| object.activate(),
669	.@"resume" => \|object\| object.@"resume"(),
670	.group_await => \|group\| {
671	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
672	if (group.await(ev, fiber)) ev.queue.async(fiber, &Fiber.@"resume");
673	},
674	.group_cancel => \|group\| {
675	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
676	if (group.cancel(ev, fiber)) ev.queue.async(fiber, &Fiber.@"resume");
677	},
678	.mutex_wait => \|waiter\| {
679	waiter.sleeper =
680	.init(ev.queue, @alignCast(@fieldParentPtr("context", message.contexts.old)));
681	switch (waiter.sleeper.fiber.cancel_protection.check()) {
682	.unblocked => {},
683	.blocked => waiter.cancelable = .blocked,
684	}
685	waiter.mutex.queue.async(waiter, &Mutex.Waiter.add);
686	},
687	.futex_wait => \|waiter\| {
688	waiter.sleeper =
689	.init(ev.queue, @alignCast(@fieldParentPtr("context", message.contexts.old)));
690	switch (waiter.sleeper.fiber.cancel_protection.check()) {
691	.unblocked => {},
692	.blocked => waiter.cancelable = .blocked,
693	}
694	waiter.futex.queue.async(waiter, &Futex.Waiter.add);
695	},
696	.futex_wake => \|waker\| {
697	waker.sleeper =
698	.init(ev.queue, @alignCast(@fieldParentPtr("context", message.contexts.old)));
699	waker.futex.queue.async(waker, &Futex.Waker.remove);
700	},
701	.sleep => \|when\| {
702	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
703	when.after(ev.queue, fiber, &Fiber.@"resume");
704	},
705	.destroy => {
706	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
707	fiber.destroy(ev);
708	},
709	.exit => _ = ev.exit_semaphore.signal(),
710	}
711	}
712	};
713
714	inline fn contextSwitch(message: const SwitchMessage) const SwitchMessage {
715	return @fieldParentPtr("contexts", Io.fiber.contextSwitch(&message.contexts));
716	}
717
718	const Cancelable = struct {
719	required_align: void align(2) = {},
720	queue: c.dispatch.queue_t,
721	cancel: c.dispatch.function_t,
722
723	const is_blocked: c.dispatch.function_t =
724	@ptrFromInt(@typeInfo(c.dispatch.function_t).pointer.alignment * 1);
725	const is_requested: c.dispatch.function_t =
726	@ptrFromInt(@typeInfo(c.dispatch.function_t).pointer.alignment * 2);
727
728	const blocked: Cancelable = .{ .queue = undefined, .cancel = is_blocked };
729
730	const AwaitError = error{CancelRequested};
731
732	fn await(cancelable: Cancelable, fiber: Fiber) AwaitError!void {
733	const function = cancelable.cancel;
734	assert(function != is_requested);
735	if (function == is_blocked) {
736	@branchHint(.unlikely);
737	return;
738	}
739	if (@cmpxchgStrong(
740	Fiber.CancelStatus,
741	&fiber.cancel_status,
742	.{ .requested = false, .awaiting = .nothing },
743	.{ .requested = false, .awaiting = .fromCancelable(cancelable) },
744	.release,
745	.monotonic,
746	)) \|cancel_status\| {
747	assert(cancel_status.requested and cancel_status.awaiting == .nothing);
748	cancelable.cancel = is_requested;
749	return error.CancelRequested;
750	}
751	}
752
753	fn canceled(cancelable: *Cancelable) void {
754	assert(cancelable.cancel != is_blocked);
755	assert(cancelable.cancel != is_requested);
756	cancelable.queue.async(cancelable, cancelable.cancel);
757	}
758
759	fn check(cancelable: Cancelable, fiber: Fiber) Io.Cancelable!void {
760	if (cancelable.cancel == is_requested) {
761	@branchHint(.unlikely);
762	fiber.cancel_protection.acknowledge();
763	return error.Canceled;
764	}
765	}
766	};
767
768	const Sleeper = struct {
769	queue: c.dispatch.queue_t,
770	fiber: *Fiber,
771
772	fn init(queue: c.dispatch.queue_t, fiber: *Fiber) Sleeper {
773	queue.as_object().retain();
774	return .{ .queue = queue, .fiber = fiber };
775	}
776
777	fn wake(context: ?*anyopaque) callconv(.c) void {
778	const sleeper: *Sleeper = @ptrCast(@alignCast(context));
779	const queue = sleeper.queue;
780	sleeper.queue = undefined;
781	queue.async(sleeper.fiber, &Fiber.@"resume");
782	queue.as_object().release();
783	}
784	};
785
786	const Mutex = struct {
787	/// including the locker
788	num_waiters: usize,
789	queue: c.dispatch.queue_t,
790	waiters: std.DoublyLinkedList,
791
792	const Waiter = struct {
793	sleeper: Sleeper = undefined,
794	cancelable: Cancelable,
795	mutex: *Mutex,
796	node: std.DoublyLinkedList.Node = .{},
797
798	fn add(context: ?*anyopaque) callconv(.c) void {
799	const waiter: *Waiter = @ptrCast(@alignCast(context));
800	waiter.tryAdd() catch \|err\| switch (err) {
801	error.CancelRequested => {
802	waiter.wake();
803	assert(@atomicRmw(usize, &waiter.mutex.num_waiters, .Sub, 1, .monotonic) >= 1);
804	},
805	};
806	}
807
808	fn tryAdd(waiter: *Waiter) Cancelable.AwaitError!void {
809	switch (@atomicLoad(usize, &waiter.mutex.num_waiters, .acquire)) {
810	0 => unreachable,
811	1 => return waiter.wake(), // already locked exclusively
812	else => try waiter.cancelable.await(waiter.sleeper.fiber),
813	}
814	waiter.mutex.waiters.append(&waiter.node);
815	}
816
817	fn canceled(context: ?*anyopaque) callconv(.c) void {
818	const cancelable: *Cancelable = @ptrCast(@alignCast(context));
819	cancelable.cancel = Cancelable.is_requested;
820	const waiter: *Waiter = @fieldParentPtr("cancelable", cancelable);
821	assert(@atomicRmw(
822	Fiber.CancelStatus,
823	&waiter.sleeper.fiber.cancel_status,
824	.Xchg,
825	.{ .requested = true, .awaiting = .nothing },
826	.monotonic,
827	) == Fiber.CancelStatus{ .requested = true, .awaiting = .fromCancelable(cancelable) });
828	const mutex = waiter.mutex;
829	mutex.waiters.remove(&waiter.node);
830	waiter.wake();
831	assert(@atomicRmw(usize, &mutex.num_waiters, .Sub, 1, .monotonic) >= 1);
832	}
833
834	fn remove(context: ?*anyopaque) callconv(.c) void {
835	const mutex: *Mutex = @ptrCast(@alignCast(context));
836	var stop_node: ?*std.DoublyLinkedList.Node = null;
837	while (mutex.waiters.first != stop_node) {
838	@branchHint(.likely);
839	const waiter: *Waiter = @fieldParentPtr("node", mutex.waiters.popFirst().?);
840	if (waiter.cancelable.cancel != Cancelable.is_blocked) {
841	@branchHint(.likely);
842	const cancel_status = @atomicRmw(
843	Fiber.CancelStatus,
844	&waiter.sleeper.fiber.cancel_status,
845	.And,
846	.{ .requested = true, .awaiting = .nothing },
847	.monotonic,
848	);
849	assert(cancel_status.awaiting.toCancelable() == &waiter.cancelable);
850	if (cancel_status.requested) {
851	@branchHint(.unlikely);
852	// carefully place the hot potato out of the way
853	mutex.waiters.append(&waiter.node);
854	if (stop_node == null) stop_node = &waiter.node;
855	continue;
856	}
857	}
858	waiter.wake();
859	return;
860	}
861	// everyone is about to die, nobody will wake up ;-(
862	}
863
864	fn wake(waiter: *Waiter) void {
865	Sleeper.wake(&waiter.sleeper);
866	}
867	};
868
869	fn init(mutex: *Mutex, queue: c.dispatch.queue_t) error{SystemResources}!void {
870	mutex.* = .{
871	.num_waiters = 0,
872	.queue = c.dispatch.queue_create_with_target(
873	"org.ziglang.std.Io.Dispatch.Mutex",
874	.SERIAL(),
875	queue,
876	) orelse return error.SystemResources,
877	.waiters = .{},
878	};
879	}
880
881	fn deinit(mutex: *Mutex) void {
882	assert(mutex.num_waiters == 0 and mutex.waiters.first == null and mutex.waiters.last == null);
883	mutex.queue.as_object().release();
884	mutex.* = undefined;
885	}
886
887	fn tryLock(mutex: *Mutex) bool {
888	if (@cmpxchgWeak(usize, &mutex.num_waiters, 0, 1, .acquire, .monotonic) == null) {
889	@branchHint(.likely);
890	return true;
891	}
892	return false;
893	}
894
895	fn lock(mutex: Mutex, ev: Evented) Io.Cancelable!void {
896	switch (@atomicRmw(usize, &mutex.num_waiters, .Add, 1, .acquire)) {
897	0 => {},
898	else => {
899	@branchHint(.unlikely);
900	var waiter: Waiter = .{
901	.cancelable = .{ .queue = mutex.queue, .cancel = &Mutex.Waiter.canceled },
902	.mutex = mutex,
903	};
904	ev.yield(.{ .mutex_wait = &waiter });
905	try waiter.cancelable.check(waiter.sleeper.fiber);
906	},
907	}
908	}
909
910	fn lockUncancelable(mutex: Mutex, ev: Evented) void {
911	switch (@atomicRmw(usize, &mutex.num_waiters, .Add, 1, .acquire)) {
912	0 => {},
913	else => {
914	@branchHint(.unlikely);
915	var waiter: Waiter = .{ .cancelable = .blocked, .mutex = mutex };
916	ev.yield(.{ .mutex_wait = &waiter });
917	waiter.cancelable.check(waiter.sleeper.fiber) catch \|err\| switch (err) {
918	error.Canceled => unreachable, // blocked
919	};
920	},
921	}
922	}
923
924	fn unlock(mutex: *Mutex) void {
925	switch (@atomicRmw(usize, &mutex.num_waiters, .Sub, 1, .release)) {
926	0 => unreachable,
927	1 => {},
928	else => {
929	@branchHint(.unlikely);
930	mutex.queue.async(mutex, &Waiter.remove);
931	},
932	}
933	}
934	};
935
936	fn crashHandler(userdata: ?*anyopaque) void {
937	const ev: *Evented = @ptrCast(@alignCast(userdata));
938	_ = ev;
939	const thread = &Thread.self;
940	if (thread.current_context == null) std.process.abort();
941	if (thread.current_context == &thread.main_context) std.process.abort();
942	const fiber = thread.currentFiber();
943	@atomicStore(
944	Fiber.CancelStatus,
945	&fiber.cancel_status,
946	.{ .requested = true, .awaiting = .nothing },
947	.monotonic,
948	);
949	fiber.cancel_protection = .{ .user = .blocked, .acknowledged = true };
950	}
951
952	const AsyncClosure = struct {
953	ev: *Evented,
954	fiber: *Fiber,
955	start: const fn (context: const anyopaque, result: *anyopaque) void,
956	result_align: Alignment,
957
958	fn fromFiber(fiber: Fiber) AsyncClosure {
959	return @ptrFromInt(Fiber.max_context_align.max(.of(AsyncClosure)).backward(
960	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
961	) - @sizeOf(AsyncClosure));
962	}
963
964	fn contextPointer(closure: AsyncClosure) []align(Fiber.max_context_align.toByteUnits()) u8 {
965	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(AsyncClosure));
966	}
967
968	fn entry() callconv(.naked) void {
969	switch (builtin.cpu.arch) {
970	.aarch64 => asm volatile (
971	\\ mov x0, sp
972	\\ b %[call]
973	:
974	: [call] "X" (&call),
975	),
976	.x86_64 => asm volatile (
977	\\ leaq 8(%%rsp), %%rdi
978	\\ jmp %[call:P]
979	:
980	: [call] "X" (&call),
981	),
982	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
983	}
984	}
985
986	fn call(
987	closure: *AsyncClosure,
988	message: *const SwitchMessage,
989	) callconv(.withStackAlign(.c, @alignOf(AsyncClosure))) noreturn {
990	message.handle(closure.ev);
991	const fiber = closure.fiber;
992	closure.start(closure.contextPointer(), fiber.resultBytes(closure.result_align));
993	if (@atomicRmw(?*Fiber, &fiber.link.awaiter, .Xchg, Fiber.finished, .acq_rel)) \|awaiter\|
994	if (@atomicRmw(i32, &awaiter.await_count, .Add, 1, .monotonic) == -1)
995	closure.ev.queue.async(awaiter, &Fiber.@"resume");
996	closure.ev.yield(.nothing);
997	unreachable; // switched to dead fiber
998	}
999	};
1000
1001	fn async(
1002	userdata: ?*anyopaque,
1003	result: []u8,
1004	result_alignment: Alignment,
1005	context: []const u8,
1006	context_alignment: Alignment,
1007	start: const fn (context: const anyopaque, result: *anyopaque) void,
1008	) ?*std.Io.AnyFuture {
1009	const ev: *Evented = @ptrCast(@alignCast(userdata));
1010	return concurrent(ev, result.len, result_alignment, context, context_alignment, start) catch {
1011	start(context.ptr, result.ptr);
1012	return null;
1013	};
1014	}
1015
1016	fn concurrent(
1017	userdata: ?*anyopaque,
1018	result_len: usize,
1019	result_alignment: Alignment,
1020	context: []const u8,
1021	context_alignment: Alignment,
1022	start: const fn (context: const anyopaque, result: *anyopaque) void,
1023	) Io.ConcurrentError!*std.Io.AnyFuture {
1024	assert(result_alignment.compare(.lte, Fiber.max_result_align)); // TODO
1025	assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
1026	assert(result_len <= Fiber.max_result_size); // TODO
1027	assert(context.len <= Fiber.max_context_size); // TODO
1028
1029	const ev: *Evented = @ptrCast(@alignCast(userdata));
1030	const fiber = Fiber.create(ev) catch \|err\| switch (err) {
1031	error.OutOfMemory => return error.ConcurrencyUnavailable,
1032	};
1033
1034	const closure: *AsyncClosure = .fromFiber(fiber);
1035	fiber.* = .{
1036	.required_align = {},
1037	.evented = ev,
1038	.context = switch (builtin.cpu.arch) {
1039	.aarch64 => .{
1040	.sp = @intFromPtr(closure),
1041	.fp = 0,
1042	.pc = @intFromPtr(&AsyncClosure.entry),
1043	},
1044	.x86_64 => .{
1045	.rsp = @intFromPtr(closure) - 8,
1046	.rbp = 0,
1047	.rip = @intFromPtr(&AsyncClosure.entry),
1048	},
1049	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1050	},
1051	.await_count = 0,
1052	.link = .{ .awaiter = null },
1053	.status = .{ .queue_next = null },
1054	.cancel_status = .unrequested,
1055	.cancel_protection = .unblocked,
1056	};
1057	closure.* = .{
1058	.ev = ev,
1059	.fiber = fiber,
1060	.start = start,
1061	.result_align = result_alignment,
1062	};
1063	@memcpy(closure.contextPointer(), context);
1064
1065	ev.queue.async(fiber, &Fiber.@"resume");
1066	return @ptrCast(fiber);
1067	}
1068
1069	fn await(
1070	userdata: ?*anyopaque,
1071	future: *std.Io.AnyFuture,
1072	result: []u8,
1073	result_alignment: Alignment,
1074	) void {
1075	const ev: *Evented = @ptrCast(@alignCast(userdata));
1076	const fiber = Thread.current().currentFiber();
1077	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
1078	if (@atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, fiber, .acq_rel)) \|awaiter\| {
1079	assert(awaiter == Fiber.finished);
1080	} else while (true) {
1081	ev.yield(.{ .await = 1 });
1082	const awaiter = @atomicLoad(?*Fiber, &future_fiber.link.awaiter, .acquire);
1083	if (awaiter == Fiber.finished) break;
1084	assert(awaiter == fiber); // spurious wakeup
1085	}
1086	@memcpy(result, future_fiber.resultBytes(result_alignment));
1087	future_fiber.destroy(ev);
1088	}
1089
1090	fn cancel(
1091	userdata: ?*anyopaque,
1092	future: *std.Io.AnyFuture,
1093	result: []u8,
1094	result_alignment: Alignment,
1095	) void {
1096	const ev: *Evented = @ptrCast(@alignCast(userdata));
1097	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
1098	future_fiber.requestCancel(ev);
1099	await(ev, future, result, result_alignment);
1100	}
1101
1102	const Group = struct {
1103	ptr: *Io.Group,
1104
1105	const List = packed struct(usize) {
1106	cancel_requested: bool,
1107	awaiter_delayed: bool,
1108	fibers: Fiber.PackedPtr,
1109	};
1110	fn listPtr(group: Group) *List {
1111	return @ptrCast(&group.ptr.token);
1112	}
1113
1114	const Mutex = packed struct(u32) {
1115	locked: bool,
1116	contended: bool,
1117	shared2: u30,
1118	};
1119	fn mutexPtr(group: Group) *Group.Mutex {
1120	return switch (comptime builtin.cpu.arch.endian()) {
1121	.little => @ptrCast(&group.ptr.state),
1122	.big => @ptrCast(@alignCast(
1123	@as([*]u8, @ptrCast(&group.ptr.state)) + @sizeOf(usize) - @sizeOf(u32),
1124	)),
1125	};
1126	}
1127
1128	const Awaiter = packed struct(usize) {
1129	locked: bool,
1130	contended: bool,
1131	awaiter: Fiber.PackedPtr,
1132	};
1133	fn awaiterPtr(group: Group) *Awaiter {
1134	return @ptrCast(&group.ptr.state);
1135	}
1136
1137	fn lock(group: Group, ev: *Evented) void {
1138	const mutex = group.mutexPtr();
1139	{
1140	const old_state = @atomicRmw(
1141	Group.Mutex,
1142	mutex,
1143	.Or,
1144	.{ .locked = true, .contended = false, .shared2 = 0 },
1145	.acquire,
1146	);
1147	if (!old_state.locked) {
1148	@branchHint(.likely);
1149	return;
1150	}
1151	if (old_state.contended) {
1152	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
1153	}
1154	}
1155	while (true) {
1156	var old_state = @atomicRmw(
1157	Group.Mutex,
1158	mutex,
1159	.Or,
1160	.{ .locked = true, .contended = true, .shared2 = 0 },
1161	.acquire,
1162	);
1163	if (!old_state.locked) {
1164	@branchHint(.likely);
1165	return;
1166	}
1167	old_state.contended = true;
1168	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
1169	}
1170	}
1171
1172	fn unlock(group: Group, ev: *Evented) void {
1173	const mutex = group.mutexPtr();
1174	const old_state = @atomicRmw(
1175	Group.Mutex,
1176	mutex,
1177	.And,
1178	.{ .locked = false, .contended = false, .shared2 = std.math.maxInt(u30) },
1179	.release,
1180	);
1181	assert(old_state.locked);
1182	if (old_state.contended) futexWake(ev, @ptrCast(mutex), 1);
1183	}
1184
1185	fn addFiber(group: Group, ev: Evented, fiber: Fiber) void {
1186	group.lock(ev);
1187	defer group.unlock(ev);
1188	const list_ptr = group.listPtr();
1189	const list = @atomicLoad(List, list_ptr, .monotonic);
1190	if (list.cancel_requested) fiber.cancel_status = .{ .requested = true, .awaiting = .nothing };
1191	const old_head = list.fibers.unpack();
1192	if (old_head) \|head\| head.link.group.prev = fiber;
1193	fiber.link.group.next = old_head;
1194	@atomicStore(List, list_ptr, .{
1195	.cancel_requested = list.cancel_requested,
1196	.awaiter_delayed = list.awaiter_delayed,
1197	.fibers = .pack(fiber),
1198	}, .monotonic);
1199	}
1200
1201	fn removeFiber(group: Group, ev: Evented, fiber: Fiber) ?*Fiber {
1202	group.lock(ev);
1203	defer group.unlock(ev);
1204	const list_ptr = group.listPtr();
1205	const list = @atomicLoad(List, list_ptr, .monotonic);
1206	if (fiber.link.group.next) \|next\| next.link.group.prev = fiber.link.group.prev;
1207	if (fiber.link.group.prev) \|prev\| {
1208	prev.link.group.next = fiber.link.group.next;
1209	} else if (fiber.link.group.next) \|new_head\| {
1210	@atomicStore(List, list_ptr, .{
1211	.cancel_requested = list.cancel_requested,
1212	.awaiter_delayed = list.awaiter_delayed,
1213	.fibers = .pack(new_head),
1214	}, .monotonic);
1215	} else if (@atomicLoad(Awaiter, group.awaiterPtr(), .monotonic).awaiter.unpack()) \|awaiter\| {
1216	if (!awaiter.cancel_status.changeAwaiting(.group, .nothing) or list.cancel_requested) {
1217	@atomicStore(List, list_ptr, .{
1218	.cancel_requested = false,
1219	.awaiter_delayed = false,
1220	.fibers = .null,
1221	}, .release);
1222	assert(awaiter.status.awaiting_group.ptr == group.ptr);
1223	awaiter.status = .{ .queue_next = null };
1224	return awaiter;
1225	}
1226	// Race with `Fiber.requestCancel`
1227	@atomicStore(List, list_ptr, .{
1228	.cancel_requested = false,
1229	.awaiter_delayed = true,
1230	.fibers = .null,
1231	}, .monotonic);
1232	} else @atomicStore(List, list_ptr, .{
1233	.cancel_requested = false,
1234	.awaiter_delayed = false,
1235	.fibers = .null,
1236	}, .release);
1237	return null;
1238	}
1239
1240	fn await(group: Group, ev: Evented, awaiter: Fiber) bool {
1241	group.lock(ev);
1242	defer group.unlock(ev);
1243	if (@atomicLoad(List, group.listPtr(), .monotonic).fibers.unpack()) \|_\| {
1244	if (group.registerAwaiter(awaiter) and awaiter.cancel_protection.check() == .unblocked) {
1245	// The awaiter already had an unacknowledged cancelation request before
1246	// attempting to await a group, so propagate the cancelation to the group.
1247	assert(!group.cancelLocked(ev, null));
1248	}
1249	return false;
1250	}
1251	return true;
1252	}
1253
1254	fn cancel(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
1255	group.lock(ev);
1256	defer group.unlock(ev);
1257	return group.cancelLocked(ev, maybe_awaiter);
1258	}
1259
1260	/// Assumes the mutex is held.
1261	fn cancelLocked(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
1262	const list_ptr = group.listPtr();
1263	const list = @atomicRmw(
1264	List,
1265	list_ptr,
1266	.Add,
1267	.{ .cancel_requested = true, .awaiter_delayed = false, .fibers = .null },
1268	.monotonic,
1269	);
1270	assert(!list.cancel_requested);
1271	if (list.fibers.unpack()) \|head\| {
1272	var maybe_fiber: ?*Fiber = head;
1273	while (maybe_fiber) \|fiber\| {
1274	fiber.requestCancel(ev);
1275	maybe_fiber = fiber.link.group.next;
1276	}
1277	if (maybe_awaiter) \|awaiter\| _ = group.registerAwaiter(awaiter);
1278	return false;
1279	}
1280	@atomicStore(
1281	List,
1282	list_ptr,
1283	.{ .cancel_requested = false, .awaiter_delayed = false, .fibers = .null },
1284	.release,
1285	);
1286	return if (maybe_awaiter) \|_\| true else list.awaiter_delayed;
1287	}
1288
1289	/// Assumes the mutex is held.
1290	fn registerAwaiter(group: Group, awaiter: *Fiber) bool {
1291	assert(awaiter.status.queue_next == null);
1292	awaiter.status = .{ .awaiting_group = group };
1293	assert(@atomicRmw(
1294	Awaiter,
1295	group.awaiterPtr(),
1296	.Add,
1297	.{ .locked = false, .contended = false, .awaiter = .pack(awaiter) },
1298	.monotonic,
1299	).awaiter == .null);
1300	return awaiter.cancel_status.changeAwaiting(.nothing, .group);
1301	}
1302
1303	const AsyncClosure = struct {
1304	ev: *Evented,
1305	group: Group,
1306	fiber: *Fiber,
1307	start: const fn (context: const anyopaque) Io.Cancelable!void,
1308
1309	fn fromFiber(fiber: Fiber) Group.AsyncClosure {
1310	return @ptrFromInt(Fiber.max_context_align.max(.of(Group.AsyncClosure)).backward(
1311	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
1312	) - @sizeOf(Group.AsyncClosure));
1313	}
1314
1315	fn contextPointer(
1316	closure: *Group.AsyncClosure,
1317	) [*]align(Fiber.max_context_align.toByteUnits()) u8 {
1318	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(Group.AsyncClosure));
1319	}
1320
1321	fn entry() callconv(.naked) void {
1322	switch (builtin.cpu.arch) {
1323	.aarch64 => asm volatile (
1324	\\ mov x0, sp
1325	\\ b %[call]
1326	:
1327	: [call] "X" (&call),
1328	),
1329	.x86_64 => asm volatile (
1330	\\ leaq 8(%%rsp), %%rdi
1331	\\ jmp %[call:P]
1332	:
1333	: [call] "X" (&call),
1334	),
1335	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1336	}
1337	}
1338
1339	fn call(
1340	closure: *Group.AsyncClosure,
1341	message: *const SwitchMessage,
1342	) callconv(.withStackAlign(.c, @alignOf(Group.AsyncClosure))) noreturn {
1343	message.handle(closure.ev);
1344	assert(closure.fiber.status.queue_next == null);
1345	const result = closure.start(closure.contextPointer());
1346	const ev = closure.ev;
1347	const group = closure.group;
1348	const fiber = closure.fiber;
1349	const cancel_acknowledged = fiber.cancel_protection.acknowledged;
1350	if (result) {
1351	assert(!cancel_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
1352	} else \|err\| switch (err) {
1353	error.Canceled => assert(cancel_acknowledged), // group task returned `error.Canceled` but was never canceled
1354	}
1355	if (group.removeFiber(ev, fiber)) \|awaiter\| ev.queue.async(awaiter, &Fiber.@"resume");
1356	ev.yield(.destroy);
1357	unreachable; // switched to dead fiber
1358	}
1359	};
1360	};
1361
1362	fn groupAsync(
1363	userdata: ?*anyopaque,
1364	type_erased: *Io.Group,
1365	context: []const u8,
1366	context_alignment: Alignment,
1367	start: const fn (context: const anyopaque) Io.Cancelable!void,
1368	) void {
1369	const ev: *Evented = @ptrCast(@alignCast(userdata));
1370	return groupConcurrent(ev, type_erased, context, context_alignment, start) catch {
1371	const fiber = Thread.current().currentFiber();
1372	const pre_acknowledged = fiber.cancel_protection.acknowledged;
1373	const result = start(context.ptr);
1374	const post_acknowledged = fiber.cancel_protection.acknowledged;
1375	if (result) {
1376	if (pre_acknowledged) {
1377	assert(post_acknowledged); // group task called `recancel` but was not canceled
1378	} else {
1379	assert(!post_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
1380	}
1381	} else \|err\| switch (err) {
1382	// Don't swallow the cancelation: make it visible to the `Group.async` caller.
1383	error.Canceled => {
1384	assert(!pre_acknowledged); // group task called `recancel` but was not canceled
1385	assert(post_acknowledged); // group task returned `error.Canceled` but was never canceled
1386	fiber.cancel_protection.recancel();
1387	},
1388	}
1389	};
1390	}
1391
1392	fn groupConcurrent(
1393	userdata: ?*anyopaque,
1394	type_erased: *Io.Group,
1395	context: []const u8,
1396	context_alignment: Alignment,
1397	start: const fn (context: const anyopaque) Io.Cancelable!void,
1398	) Io.ConcurrentError!void {
1399	assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
1400	assert(context.len <= Fiber.max_context_size); // TODO
1401
1402	const ev: *Evented = @ptrCast(@alignCast(userdata));
1403	const group: Group = .{ .ptr = type_erased };
1404	const fiber = Fiber.create(ev) catch \|err\| switch (err) {
1405	error.OutOfMemory => return error.ConcurrencyUnavailable,
1406	};
1407
1408	const closure: *Group.AsyncClosure = .fromFiber(fiber);
1409	fiber.* = .{
1410	.required_align = {},
1411	.evented = ev,
1412	.context = switch (builtin.cpu.arch) {
1413	.aarch64 => .{
1414	.sp = @intFromPtr(closure),
1415	.fp = 0,
1416	.pc = @intFromPtr(&Group.AsyncClosure.entry),
1417	},
1418	.x86_64 => .{
1419	.rsp = @intFromPtr(closure) - 8,
1420	.rbp = 0,
1421	.rip = @intFromPtr(&Group.AsyncClosure.entry),
1422	},
1423	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1424	},
1425	.await_count = 0,
1426	.link = .{ .group = .{ .prev = null, .next = null } },
1427	.status = .{ .queue_next = null },
1428	.cancel_status = .unrequested,
1429	.cancel_protection = .unblocked,
1430	};
1431	closure.* = .{
1432	.ev = ev,
1433	.group = group,
1434	.fiber = fiber,
1435	.start = start,
1436	};
1437	@memcpy(closure.contextPointer(), context);
1438	group.addFiber(ev, fiber);
1439	ev.queue.async(fiber, &Fiber.@"resume");
1440	}
1441
1442	fn groupAwait(
1443	userdata: ?*anyopaque,
1444	type_erased: *Io.Group,
1445	initial_token: *anyopaque,
1446	) Io.Cancelable!void {
1447	const ev: *Evented = @ptrCast(@alignCast(userdata));
1448	_ = initial_token;
1449	ev.yield(.{ .group_await = .{ .ptr = type_erased } });
1450	}
1451
1452	fn groupCancel(userdata: ?anyopaque, type_erased: Io.Group, initial_token: *anyopaque) void {
1453	const ev: *Evented = @ptrCast(@alignCast(userdata));
1454	_ = initial_token;
1455	ev.yield(.{ .group_cancel = .{ .ptr = type_erased } });
1456	}
1457
1458	fn recancel(userdata: ?*anyopaque) void {
1459	const ev: *Evented = @ptrCast(@alignCast(userdata));
1460	_ = ev;
1461	Thread.current().currentFiber().cancel_protection.recancel();
1462	}
1463
1464	fn swapCancelProtection(userdata: ?*anyopaque, new: Io.CancelProtection) Io.CancelProtection {
1465	const ev: *Evented = @ptrCast(@alignCast(userdata));
1466	_ = ev;
1467	const cancel_protection = &Thread.current().currentFiber().cancel_protection;
1468	defer cancel_protection.user = new;
1469	return cancel_protection.user;
1470	}
1471
1472	fn checkCancel(userdata: ?*anyopaque) Io.Cancelable!void {
1473	const ev: *Evented = @ptrCast(@alignCast(userdata));
1474	_ = ev;
1475	const fiber = Thread.current().currentFiber();
1476	switch (fiber.cancel_protection.check()) {
1477	.unblocked => {
1478	const cancel_status = @atomicLoad(Fiber.CancelStatus, &fiber.cancel_status, .monotonic);
1479	assert(cancel_status.awaiting == .nothing);
1480	if (cancel_status.requested) {
1481	@branchHint(.unlikely);
1482	fiber.cancel_protection.acknowledge();
1483	return error.Canceled;
1484	}
1485	},
1486	.blocked => {},
1487	}
1488	}
1489
1490	const Futex = struct {
1491	num_waiters: usize,
1492	queue: c.dispatch.queue_t,
1493	waiters: std.DoublyLinkedList,
1494
1495	const Waiter = struct {
1496	sleeper: Sleeper = undefined,
1497	cancelable: Cancelable,
1498	futex: *Futex,
1499	node: std.DoublyLinkedList.Node = .{},
1500	ptr: *const u32,
1501	expected: u32,
1502	timeout: c.dispatch.time_t = .FOREVER,
1503	leeway: u64,
1504	timer: ?c.dispatch.source_t = null,
1505
1506	const already_signaled: c.dispatch.source_t = @ptrFromInt(1);
1507
1508	fn add(context: ?*anyopaque) callconv(.c) void {
1509	const waiter: *Waiter = @ptrCast(@alignCast(context));
1510	const futex = waiter.futex;
1511	_ = @atomicRmw(usize, &futex.num_waiters, .Add, 1, .acquire);
1512	waiter.tryAdd() catch \|err\| switch (err) {
1513	error.CancelRequested => {
1514	wake(waiter);
1515	assert(@atomicRmw(usize, &futex.num_waiters, .Sub, 1, .monotonic) >= 1);
1516	},
1517	};
1518	}
1519
1520	fn tryAdd(waiter: *Waiter) Cancelable.AwaitError!void {
1521	if (@atomicLoad(u32, waiter.ptr, .monotonic) != waiter.expected)
1522	return error.CancelRequested;
1523	try waiter.cancelable.await(waiter.sleeper.fiber);
1524	const futex = waiter.futex;
1525	switch (waiter.timeout) {
1526	.FOREVER => {},
1527	else => \|timeout\| {
1528	const timer = c.dispatch.source_create(.TIMER, 0, .none, futex.queue) orelse {
1529	log.warn("unable to create timer for futex timeout", .{});
1530	return error.CancelRequested;
1531	};
1532	timer.as_object().set_context(waiter);
1533	timer.set_event_handler(&timedOut);
1534	timer.set_cancel_handler(&wake);
1535	timer.set_timer(timeout, c.dispatch.TIME_FOREVER, waiter.leeway);
1536	timer.as_object().activate();
1537	waiter.timer = timer;
1538	},
1539	}
1540	futex.waiters.append(&waiter.node);
1541	}
1542
1543	fn canceled(context: ?*anyopaque) callconv(.c) void {
1544	const cancelable: *Cancelable = @ptrCast(@alignCast(context));
1545	cancelable.cancel = Cancelable.is_requested;
1546	const waiter: *Waiter = @fieldParentPtr("cancelable", cancelable);
1547	assert(@atomicRmw(
1548	Fiber.CancelStatus,
1549	&waiter.sleeper.fiber.cancel_status,
1550	.Xchg,
1551	.{ .requested = true, .awaiting = .nothing },
1552	.monotonic,
1553	) == Fiber.CancelStatus{ .requested = true, .awaiting = .fromCancelable(cancelable) });
1554	const futex = waiter.futex;
1555	waiter.removeUncancelable();
1556	assert(@atomicRmw(usize, &futex.num_waiters, .Sub, 1, .monotonic) >= 1);
1557	}
1558
1559	fn timedOut(context: ?*anyopaque) callconv(.c) void {
1560	const waiter: *Waiter = @ptrCast(@alignCast(context));
1561	const futex = waiter.futex;
1562	waiter.remove() catch \|err\| switch (err) {
1563	error.CancelRequested => return,
1564	};
1565	assert(@atomicRmw(usize, &futex.num_waiters, .Sub, 1, .monotonic) >= 1);
1566	}
1567
1568	fn remove(waiter: *Waiter) Cancelable.AwaitError!void {
1569	if (waiter.cancelable.cancel != Cancelable.is_blocked) {
1570	@branchHint(.likely);
1571	const cancel_status = @atomicRmw(
1572	Fiber.CancelStatus,
1573	&waiter.sleeper.fiber.cancel_status,
1574	.And,
1575	.{ .requested = true, .awaiting = .nothing },
1576	.monotonic,
1577	);
1578	assert(cancel_status.awaiting.toCancelable() == &waiter.cancelable);
1579	if (cancel_status.requested) return error.CancelRequested;
1580	}
1581	waiter.removeUncancelable();
1582	}
1583
1584	fn removeUncancelable(waiter: *Waiter) void {
1585	waiter.futex.waiters.remove(&waiter.node);
1586	if (waiter.timer) \|timer\| timer.cancel() else wake(waiter);
1587	}
1588
1589	fn wake(context: ?*anyopaque) callconv(.c) void {
1590	const waiter: *Waiter = @ptrCast(@alignCast(context));
1591	if (waiter.timer) \|timer\| timer.as_object().release();
1592	Sleeper.wake(&waiter.sleeper);
1593	}
1594	};
1595
1596	const Waker = struct {
1597	sleeper: Sleeper = undefined,
1598	futex: *Futex,
1599	ptr: *const u32,
1600	max_waiters: u32,
1601
1602	fn remove(context: ?*anyopaque) callconv(.c) void {
1603	const waker: *Waker = @ptrCast(@alignCast(context));
1604	const futex = waker.futex;
1605	const ptr = waker.ptr;
1606	const max_waiters = waker.max_waiters;
1607
1608	var num_removed: usize = 0;
1609	var next_node = futex.waiters.first;
1610	while (num_removed < max_waiters) {
1611	const waiter: *Waiter = @fieldParentPtr("node", next_node orelse break);
1612	next_node = waiter.node.next;
1613	if (waiter.ptr != ptr) {
1614	@branchHint(.unlikely);
1615	continue;
1616	}
1617	waiter.remove() catch \|err\| switch (err) {
1618	error.CancelRequested => continue,
1619	};
1620	num_removed += 1;
1621	}
1622	assert(@atomicRmw(usize, &futex.num_waiters, .Sub, num_removed, .monotonic) >= num_removed);
1623
1624	var sleeper = waker.sleeper;
1625	waker.* = undefined;
1626	Sleeper.wake(&sleeper);
1627	}
1628	};
1629
1630	fn init(futex: *Futex, queue: c.dispatch.queue_t) error{SystemResources}!void {
1631	futex.* = .{
1632	.num_waiters = 0,
1633	.queue = c.dispatch.queue_create_with_target(
1634	"org.ziglang.std.Io.Dispatch.Futex",
1635	.SERIAL(),
1636	queue,
1637	) orelse return error.SystemResources,
1638	.waiters = .{},
1639	};
1640	}
1641
1642	fn deinit(futex: *Futex) void {
1643	assert(futex.num_waiters == 0 and futex.waiters.first == null and futex.waiters.last == null);
1644	futex.queue.as_object().release();
1645	futex.* = undefined;
1646	}
1647	};
1648
1649	fn futexForAddress(ev: Evented, address: usize) Futex {
1650	// Here we use Fibonacci hashing: the golden ratio can be used to evenly redistribute input
1651	// values across a range, giving a poor, but extremely quick to compute, hash.
1652
1653	// This literal is the rounded value of '2^64 / phi' (where 'phi' is the golden ratio). The
1654	// shift then converts it to '2^b / phi', where 'b' is the pointer bit width.
1655	const fibonacci_multiplier = 0x9E3779B97F4A7C15 >> (64 - @bitSizeOf(usize));
1656	const hashed = address *% fibonacci_multiplier;
1657	comptime assert(std.math.isPowerOfTwo(ev.futexes.len));
1658	// The high bits of `hashed` have better entropy than the low bits.
1659	return &ev.futexes[hashed >> @clz(ev.futexes.len - 1)];
1660	}
1661
1662	fn select(userdata: ?anyopaque, futures: []const Io.AnyFuture) Io.Cancelable!usize {
1663	const ev: *Evented = @ptrCast(@alignCast(userdata));
1664	const fiber = Thread.current().currentFiber();
1665	var await_count: u31, var result = for (futures, 0..) \|future, future_index\| {
1666	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
1667	if (@atomicRmw(
1668	?*Fiber,
1669	&future_fiber.link.awaiter,
1670	.Xchg,
1671	fiber,
1672	.acq_rel,
1673	)) \|awaiter\| {
1674	assert(awaiter == Fiber.finished);
1675	break .{ @intCast(future_index), future_index };
1676	}
1677	} else result: {
1678	const await_count: u31 = @intCast(futures.len);
1679	ev.yield(.{ .await = 1 });
1680	break :result .{ await_count - 1, futures.len };
1681	};
1682	for (futures[0..result], 0..) \|future, future_index\| {
1683	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
1684	const awaiter = @atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, null, .monotonic);
1685	if (awaiter == Fiber.finished) {
1686	@atomicStore(?*Fiber, &future_fiber.link.awaiter, Fiber.finished, .monotonic);
1687	result = @min(future_index, result);
1688	} else {
1689	assert(awaiter == fiber);
1690	await_count -= 1;
1691	}
1692	}
1693	// Equivalent to `ev.yield(null, .{ .await = await_count });`,
1694	// but avoiding a context switch in the common case.
1695	switch (std.math.order(
1696	@atomicRmw(i32, &fiber.await_count, .Sub, await_count, .monotonic),
1697	await_count,
1698	)) {
1699	.lt => ev.yield(.{ .await = 0 }),
1700	.eq => {},
1701	.gt => unreachable,
1702	}
1703	return result;
1704	}
1705
1706	fn futexWait(
1707	userdata: ?*anyopaque,
1708	ptr: *const u32,
1709	expected: u32,
1710	timeout: Io.Timeout,
1711	) Io.Cancelable!void {
1712	const ev: *Evented = @ptrCast(@alignCast(userdata));
1713	const futex = ev.futexForAddress(@intFromPtr(ptr));
1714	var waiter: Futex.Waiter = .{
1715	.cancelable = .{ .queue = futex.queue, .cancel = &Futex.Waiter.canceled },
1716	.futex = futex,
1717	.ptr = ptr,
1718	.expected = expected,
1719	.timeout = ev.timeFromTimeout(timeout),
1720	.leeway = ev.leeway,
1721	};
1722	ev.yield(.{ .futex_wait = &waiter });
1723	try waiter.cancelable.check(waiter.sleeper.fiber);
1724	}
1725
1726	fn futexWaitUncancelable(userdata: ?anyopaque, ptr: const u32, expected: u32) void {
1727	const ev: *Evented = @ptrCast(@alignCast(userdata));
1728	const futex = ev.futexForAddress(@intFromPtr(ptr));
1729	var waiter: Futex.Waiter = .{
1730	.cancelable = .blocked,
1731	.futex = futex,
1732	.ptr = ptr,
1733	.expected = expected,
1734	.leeway = ev.leeway,
1735	};
1736	ev.yield(.{ .futex_wait = &waiter });
1737	waiter.cancelable.check(waiter.sleeper.fiber) catch \|err\| switch (err) {
1738	error.Canceled => unreachable, // blocked
1739	};
1740	}
1741
1742	fn futexWake(userdata: ?anyopaque, ptr: const u32, max_waiters: u32) void {
1743	const ev: *Evented = @ptrCast(@alignCast(userdata));
1744	if (max_waiters == 0) return;
1745	const futex = ev.futexForAddress(@intFromPtr(ptr));
1746	switch (@atomicRmw(usize, &futex.num_waiters, .Add, 0, .release)) {
1747	0 => return,
1748	else => {
1749	@branchHint(.unlikely);
1750	var waker: Futex.Waker = .{ .futex = futex, .ptr = ptr, .max_waiters = max_waiters };
1751	ev.yield(.{ .futex_wake = &waker });
1752	},
1753	}
1754	}
1755
1756	fn operate(userdata: ?*anyopaque, operation: Io.Operation) Io.Cancelable!Io.Operation.Result {
1757	const ev: *Evented = @ptrCast(@alignCast(userdata));
1758	switch (operation) {
1759	.file_read_streaming => \|o\| return .{
1760	.file_read_streaming = ev.fileReadStreaming(o.file, o.data) catch \|err\| switch (err) {
1761	error.Canceled => \|e\| return e,
1762	else => \|e\| e,
1763	},
1764	},
1765	.file_write_streaming => \|o\| return .{
1766	.file_write_streaming = ev.fileWriteStreaming(
1767	o.file,
1768	o.header,
1769	o.data,
1770	o.splat,
1771	) catch \|err\| switch (err) {
1772	error.Canceled => \|e\| return e,
1773	else => \|e\| e,
1774	},
1775	},
1776	.device_io_control => \|*o\| return .{ .device_io_control = try deviceIoControl(o) },
1777	}
1778	}
1779
1780	fn fileReadStreaming(ev: *Evented, file: File, data: []const []u8) File.ReadStreamingError!usize {
1781	if (file.flags.nonblocking) nonblocking: {
1782	return fileReadStreamingLimit(file.handle, data, .unlimited) catch \|err\| switch (err) {
1783	error.WouldBlock => break :nonblocking,
1784	else => \|e\| return e,
1785	};
1786	}
1787	const source = c.dispatch.source_create(
1788	.READ,
1789	@bitCast(@as(isize, file.handle)),
1790	.none,
1791	ev.queue,
1792	) orelse return error.SystemResources;
1793	source.as_object().set_context(Thread.current().currentFiber());
1794	source.set_event_handler(&Fiber.@"resume");
1795	ev.yield(.{ .activate = source.as_object() });
1796	const limit = source.get_data();
1797	source.as_object().release();
1798	while (true) return fileReadStreamingLimit(
1799	file.handle,
1800	data,
1801	.limited(limit),
1802	) catch \|err\| switch (err) {
1803	error.WouldBlock => {
1804	ev.yield(.nothing);
1805	continue;
1806	},
1807	else => \|e\| return e,
1808	};
1809	}
1810	fn fileReadStreamingLimit(
1811	handle: File.Handle,
1812	data: []const []u8,
1813	limit: Io.Limit,
1814	) File.ReadStreamingError!usize {
1815	var iovecs: [max_iovecs_len]iovec = undefined;
1816	var iovlen: iovlen_t = 0;
1817	// .nothing can mean that the write side has been closed,
1818	// in which case the buffer still needs to be drained
1819	var remaining = if (limit == .nothing) .unlimited else limit;
1820	for (data) \|buf\| addBuf(false, &iovecs, &iovlen, &remaining, buf);
1821	if (iovlen == 0) return 0;
1822	while (true) {
1823	const rc = c.readv(handle, &iovecs, iovlen);
1824	switch (c.errno(rc)) {
1825	.SUCCESS => return if (rc == 0) error.EndOfStream else @intCast(rc),
1826	.INTR => continue,
1827	.INVAL => \|err\| return errnoBug(err),
1828	.FAULT => \|err\| return errnoBug(err),
1829	.AGAIN => return error.WouldBlock,
1830	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed
1831	.IO => return error.InputOutput,
1832	.ISDIR => return error.IsDir,
1833	.NOBUFS => return error.SystemResources,
1834	.NOMEM => return error.SystemResources,
1835	.NOTCONN => return error.SocketUnconnected,
1836	.CONNRESET => return error.ConnectionResetByPeer,
1837	else => \|err\| return unexpectedErrno(err),
1838	}
1839	}
1840	}
1841
1842	fn fileWriteStreaming(
1843	ev: *Evented,
1844	file: File,
1845	header: []const u8,
1846	data: []const []const u8,
1847	splat: usize,
1848	) File.Writer.Error!usize {
1849	if (file.flags.nonblocking) nonblocking: {
1850	return fileWriteStreamingLimit(
1851	file.handle,
1852	header,
1853	data,
1854	splat,
1855	.unlimited,
1856	) catch \|err\| switch (err) {
1857	error.WouldBlock => break :nonblocking,
1858	else => \|e\| return e,
1859	};
1860	}
1861	const source = c.dispatch.source_create(
1862	.WRITE,
1863	@bitCast(@as(isize, file.handle)),
1864	.none,
1865	ev.queue,
1866	) orelse return error.SystemResources;
1867	source.as_object().set_context(Thread.current().currentFiber());
1868	source.set_event_handler(&Fiber.@"resume");
1869	ev.yield(.{ .activate = source.as_object() });
1870	const limit = source.get_data();
1871	source.as_object().release();
1872	while (true) return fileWriteStreamingLimit(
1873	file.handle,
1874	header,
1875	data,
1876	splat,
1877	.limited(limit),
1878	) catch \|err\| switch (err) {
1879	error.WouldBlock => {
1880	ev.yield(.nothing);
1881	continue;
1882	},
1883	else => \|e\| return e,
1884	};
1885	}
1886	fn fileWriteStreamingLimit(
1887	handle: File.Handle,
1888	header: []const u8,
1889	data: []const []const u8,
1890	splat: usize,
1891	limit: Io.Limit,
1892	) File.Writer.Error!usize {
1893	if (limit == .nothing) return 0;
1894	var iovecs: [max_iovecs_len]iovec_const = undefined;
1895	var iovlen: iovlen_t = 0;
1896	var remaining = limit;
1897	addBuf(true, &iovecs, &iovlen, &remaining, header);
1898	for (data[0 .. data.len - 1]) \|bytes\| addBuf(true, &iovecs, &iovlen, &remaining, bytes);
1899	const pattern = data[data.len - 1];
1900	var backup_buffer: [splat_buffer_size]u8 = undefined;
1901	if (iovecs.len - iovlen != 0 and remaining != .nothing) switch (splat) {
1902	0 => {},
1903	1 => addBuf(true, &iovecs, &iovlen, &remaining, pattern),
1904	else => switch (pattern.len) {
1905	0 => {},
1906	1 => {
1907	const splat_buffer = &backup_buffer;
1908	const memset_len = @min(splat_buffer.len, splat);
1909	const buf = splat_buffer[0..memset_len];
1910	@memset(buf, pattern[0]);
1911	addBuf(true, &iovecs, &iovlen, &remaining, buf);
1912	var remaining_splat = splat - buf.len;
1913	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0 and remaining != .nothing) {
1914	assert(buf.len == splat_buffer.len);
1915	addBuf(true, &iovecs, &iovlen, &remaining, splat_buffer);
1916	remaining_splat -= splat_buffer.len;
1917	}
1918	addBuf(true, &iovecs, &iovlen, &remaining, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
1919	},
1920	else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
1921	if (remaining == .nothing) break;
1922	addBuf(true, &iovecs, &iovlen, &remaining, pattern);
1923	},
1924	},
1925	};
1926	if (iovlen == 0) return 0;
1927	while (true) {
1928	const rc = c.writev(handle, &iovecs, iovlen);
1929	switch (c.errno(rc)) {
1930	.SUCCESS => return @intCast(rc),
1931	.INTR => continue,
1932	.INVAL => \|err\| return errnoBug(err),
1933	.FAULT => \|err\| return errnoBug(err),
1934	.AGAIN => return error.WouldBlock,
1935	.BADF => return error.NotOpenForWriting, // Can be a race condition.
1936	.DESTADDRREQ => \|err\| return errnoBug(err), // `connect` was never called.
1937	.DQUOT => return error.DiskQuota,
1938	.FBIG => return error.FileTooBig,
1939	.IO => return error.InputOutput,
1940	.NOSPC => return error.NoSpaceLeft,
1941	.PERM => return error.PermissionDenied,
1942	.PIPE => return error.BrokenPipe,
1943	.CONNRESET => \|err\| return errnoBug(err), // Not a socket handle.
1944	.BUSY => return error.DeviceBusy,
1945	else => \|err\| return unexpectedErrno(err),
1946	}
1947	}
1948	}
1949
1950	fn deviceIoControl(o: *const Io.Operation.DeviceIoControl) Io.Cancelable!i32 {
1951	while (true) {
1952	const rc = c.ioctl(o.file.handle, @bitCast(o.code), @intFromPtr(o.arg));
1953	switch (c.errno(rc)) {
1954	.SUCCESS => return rc,
1955	.INTR => {},
1956	else => \|err\| return -@as(i32, @intFromEnum(err)),
1957	}
1958	}
1959	}
1960
1961	const BatchWaiter = struct {
1962	sleeper: Sleeper,
1963	queue: c.dispatch.queue_t,
1964	timer: ?c.dispatch.source_t = null,
1965
1966	const already_signaled: c.dispatch.source_t = @ptrFromInt(1);
1967
1968	fn signal(context: ?*anyopaque) callconv(.c) void {
1969	const waiter: *BatchWaiter = @ptrCast(@alignCast(context));
1970	if (waiter.timer) \|timer\| {
1971	if (timer != already_signaled) timer.cancel();
1972	} else {
1973	waiter.timer = already_signaled;
1974	waiter.queue.async(waiter, &@"suspend");
1975	}
1976	}
1977
1978	fn @"suspend"(context: ?*anyopaque) callconv(.c) void {
1979	const waiter: *BatchWaiter = @ptrCast(@alignCast(context));
1980	if (waiter.timer) \|timer\| if (timer != already_signaled) timer.as_object().release();
1981	waiter.queue.as_object().@"suspend"();
1982	waiter.wake();
1983	}
1984
1985	fn wake(waiter: *BatchWaiter) void {
1986	var sleeper = waiter.sleeper;
1987	waiter.* = undefined;
1988	Sleeper.wake(&sleeper);
1989	}
1990	};
1991
1992	fn batchAwaitAsync(userdata: ?anyopaque, batch: Io.Batch) Io.Cancelable!void {
1993	const ev: *Evented = @ptrCast(@alignCast(userdata));
1994	const queue = ev.batchDrainSubmitted(batch, false) catch \|err\| switch (err) {
1995	error.ConcurrencyUnavailable => unreachable, // passed concurrency=false
1996	error.Canceled => \|e\| return e,
1997	} orelse return;
1998	if (batch.pending.head == .none) return;
1999	var waiter: BatchWaiter = .{
2000	.sleeper = .init(ev.queue, Thread.current().currentFiber()),
2001	.queue = queue,
2002	};
2003	if (batch.completed.head != .none) BatchWaiter.signal(&waiter);
2004	queue.as_object().set_context(&waiter);
2005	ev.yield(.{ .@"resume" = queue.as_object() });
2006	}
2007
2008	fn batchAwaitConcurrent(
2009	userdata: ?*anyopaque,
2010	batch: *Io.Batch,
2011	timeout: Io.Timeout,
2012	) Io.Batch.AwaitConcurrentError!void {
2013	const ev: *Evented = @ptrCast(@alignCast(userdata));
2014	const queue = try ev.batchDrainSubmitted(batch, true) orelse return;
2015	if (batch.pending.head == .none) return;
2016	var waiter: BatchWaiter = .{
2017	.sleeper = .init(ev.queue, Thread.current().currentFiber()),
2018	.queue = queue,
2019	};
2020	if (batch.completed.head == .none) switch (timeout) {
2021	.none => {},
2022	else => {
2023	const timer = c.dispatch.source_create(.TIMER, 0, .none, queue) orelse
2024	return error.ConcurrencyUnavailable;
2025	assert(timer != BatchWaiter.already_signaled);
2026	timer.as_object().set_context(&waiter);
2027	timer.set_event_handler(&BatchWaiter.signal);
2028	timer.set_cancel_handler(&BatchWaiter.@"suspend");
2029	timer.set_timer(ev.timeFromTimeout(timeout), c.dispatch.TIME_FOREVER, ev.leeway);
2030	timer.as_object().activate();
2031	waiter.timer = timer;
2032	},
2033	} else BatchWaiter.signal(&waiter);
2034	queue.as_object().set_context(&waiter);
2035	ev.yield(.{ .@"resume" = queue.as_object() });
2036	}
2037
2038	fn batchCancel(userdata: ?anyopaque, batch: Io.Batch) void {
2039	const ev: *Evented = @ptrCast(@alignCast(userdata));
2040	var index = batch.pending.head;
2041	while (index != .none) {
2042	const storage = &batch.storage[index.toIndex()];
2043	const pending = &storage.pending;
2044	const operation_userdata: *BatchOperationUserdata = .fromErased(&pending.userdata);
2045	assert(operation_userdata.batch == batch);
2046	operation_userdata.source.cancel();
2047	}
2048	const queue: c.dispatch.queue_t = @ptrCast(batch.userdata orelse return);
2049	if (batch.pending.head != .none) {
2050	var waiter: BatchWaiter = .{
2051	.sleeper = .init(ev.queue, Thread.current().currentFiber()),
2052	.queue = queue,
2053	.timer = BatchWaiter.already_signaled,
2054	};
2055	if (batch.pending.head == .none) queue.async(&waiter, &BatchWaiter.signal);
2056	queue.as_object().set_context(&waiter);
2057	ev.yield(.{ .@"resume" = queue.as_object() });
2058	}
2059	batch.userdata = null;
2060	}
2061
2062	const BatchOperationUserdata = extern struct {
2063	batch: *Io.Batch,
2064	source: c.dispatch.source_t,
2065	operation: extern union {
2066	file_read_streaming: extern struct {
2067	data_ptr: [*]const []u8,
2068	data_len: usize,
2069	},
2070	file_write_streaming: extern struct {
2071	header_ptr: [*]const u8,
2072	header_len: usize,
2073	data_ptr: [*]const []const u8,
2074	data_len: usize,
2075	splat: usize,
2076
2077	fn header(operation: *const @This()) []const u8 {
2078	return operation.header_ptr[0..operation.header_len];
2079	}
2080
2081	fn data(operation: *const @This()) []const []const u8 {
2082	return operation.data_ptr[0..operation.data_len];
2083	}
2084	},
2085	},
2086
2087	const Erased = Io.Operation.Storage.Pending.Userdata;
2088
2089	comptime {
2090	assert(@sizeOf(BatchOperationUserdata) <= @sizeOf(Erased));
2091	}
2092
2093	fn toErased(userdata: BatchOperationUserdata) Erased {
2094	return @ptrCast(userdata);
2095	}
2096
2097	fn fromErased(erased: Erased) BatchOperationUserdata {
2098	return @ptrCast(erased);
2099	}
2100	};
2101
2102	/// If `concurrency` is false, `error.ConcurrencyUnavailable` is unreachable.
2103	fn batchDrainSubmitted(
2104	ev: *Evented,
2105	batch: *Io.Batch,
2106	concurrency: bool,
2107	) (Io.ConcurrentError \|\| Io.Cancelable)!?c.dispatch.queue_t {
2108	var index = batch.submitted.head;
2109	if (index == .none) return @ptrCast(batch.userdata);
2110	errdefer batch.submitted.head = index;
2111	const maybe_queue: ?c.dispatch.queue_t = if (batch.userdata) \|batch_userdata\|
2112	@ptrCast(batch_userdata)
2113	else maybe_queue: {
2114	const queue = c.dispatch.queue_create_with_target(
2115	"org.ziglang.std.Io.Dispatch.Batch",
2116	.SERIAL(),
2117	ev.queue,
2118	) orelse if (concurrency) return error.ConcurrencyUnavailable else break :maybe_queue null;
2119	queue.as_object().@"suspend"();
2120	batch.userdata = queue;
2121	break :maybe_queue queue;
2122	};
2123	while (index != .none) {
2124	const storage = &batch.storage[index.toIndex()];
2125	const next_index = storage.submission.node.next;
2126	if (@as(?Io.Operation.Result, result: {
2127	if (maybe_queue) \|queue\| switch (storage.submission.operation) {
2128	.file_read_streaming => \|operation\| {
2129	const data = for (operation.data, 0..) \|buffer, data_index\| {
2130	if (buffer.len > 0) break operation.data[data_index..];
2131	} else break :result .{ .file_read_streaming = 0 };
2132	const source = c.dispatch.source_create(
2133	.READ,
2134	@bitCast(@as(isize, operation.file.handle)),
2135	.none,
2136	queue,
2137	) orelse break :result .{ .file_read_streaming = error.SystemResources };
2138	storage.* = .{ .pending = .{
2139	.node = .{ .prev = batch.pending.tail, .next = .none },
2140	.tag = .file_read_streaming,
2141	.userdata = undefined,
2142	} };
2143	const operation_userdata: *BatchOperationUserdata =
2144	.fromErased(&storage.pending.userdata);
2145	operation_userdata.* = .{
2146	.batch = batch,
2147	.source = source,
2148	.operation = .{ .file_read_streaming = .{
2149	.data_ptr = data.ptr,
2150	.data_len = data.len,
2151	} },
2152	};
2153	source.as_object().set_context(storage);
2154	source.set_event_handler(&batchSourceEvent);
2155	source.set_cancel_handler(&batchSourceCancel);
2156	source.as_object().activate();
2157	break :result null;
2158	},
2159	.file_write_streaming => \|operation\| {
2160	const data = for (operation.data, 0..) \|buffer, data_index\| {
2161	if (buffer.len > 0) break operation.data[data_index..];
2162	} else if (operation.header.len > 0)
2163	operation.data[0..1]
2164	else
2165	break :result .{ .file_write_streaming = 0 };
2166	const source = c.dispatch.source_create(
2167	.WRITE,
2168	@bitCast(@as(isize, operation.file.handle)),
2169	.none,
2170	queue,
2171	) orelse break :result .{ .file_write_streaming = error.SystemResources };
2172	storage.* = .{ .pending = .{
2173	.node = .{ .prev = batch.pending.tail, .next = .none },
2174	.tag = .file_write_streaming,
2175	.userdata = undefined,
2176	} };
2177	const operation_userdata: *BatchOperationUserdata =
2178	.fromErased(&storage.pending.userdata);
2179	operation_userdata.* = .{
2180	.batch = batch,
2181	.source = source,
2182	.operation = .{ .file_write_streaming = .{
2183	.header_ptr = operation.header.ptr,
2184	.header_len = operation.header.len,
2185	.data_ptr = data.ptr,
2186	.data_len = data.len,
2187	.splat = operation.splat,
2188	} },
2189	};
2190	source.as_object().set_context(storage);
2191	source.set_event_handler(&batchSourceEvent);
2192	source.set_cancel_handler(&batchSourceCancel);
2193	source.as_object().activate();
2194	break :result null;
2195	},
2196	.device_io_control => {},
2197	};
2198	if (concurrency) return error.ConcurrencyUnavailable;
2199	break :result try operate(ev, storage.submission.operation);
2200	})) \|result\| {
2201	switch (batch.completed.tail) {
2202	.none => batch.completed.head = index,
2203	else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next = index,
2204	}
2205	batch.completed.tail = index;
2206	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2207	} else {
2208	switch (batch.pending.tail) {
2209	.none => batch.pending.head = index,
2210	else => \|tail_index\| batch.storage[tail_index.toIndex()].pending.node.next = index,
2211	}
2212	batch.pending.tail = index;
2213	}
2214	index = next_index;
2215	}
2216	batch.submitted = .{ .head = .none, .tail = .none };
2217	return maybe_queue;
2218	}
2219
2220	fn batchSourceEvent(context: ?*anyopaque) callconv(.c) void {
2221	const storage: *Io.Operation.Storage = @ptrCast(@alignCast(context));
2222	const pending = &storage.pending;
2223	const operation_userdata: *BatchOperationUserdata = .fromErased(&pending.userdata);
2224	const batch = operation_userdata.batch;
2225	const source = operation_userdata.source;
2226	const index: Io.Operation.OptionalIndex = .fromIndex(storage - batch.storage.ptr);
2227	const result: Io.Operation.Result = result: switch (pending.tag) {
2228	.file_read_streaming => {
2229	const operation = &operation_userdata.operation.file_read_streaming;
2230	break :result .{ .file_read_streaming = fileReadStreamingLimit(
2231	@intCast(source.get_handle()),
2232	operation.data_ptr[0..operation.data_len],
2233	.limited(source.get_data()),
2234	) catch \|err\| switch (err) {
2235	error.Canceled => return Thread.current().currentFiber().cancel_protection.recancel(),
2236	error.WouldBlock => return,
2237	else => \|e\| e,
2238	} };
2239	},
2240	.file_write_streaming => {
2241	const operation = &operation_userdata.operation.file_write_streaming;
2242	break :result .{ .file_write_streaming = fileWriteStreamingLimit(
2243	@intCast(source.get_handle()),
2244	operation.header_ptr[0..operation.header_len],
2245	operation.data_ptr[0..operation.data_len],
2246	operation.splat,
2247	.limited(source.get_data()),
2248	) catch \|err\| switch (err) {
2249	error.Canceled => return Thread.current().currentFiber().cancel_protection.recancel(),
2250	error.WouldBlock => return,
2251	else => \|e\| e,
2252	} };
2253	},
2254	.device_io_control => unreachable,
2255	};
2256
2257	switch (pending.node.prev) {
2258	.none => batch.pending.head = pending.node.next,
2259	else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.next = pending.node.next,
2260	}
2261	switch (pending.node.next) {
2262	.none => batch.pending.tail = pending.node.prev,
2263	else => \|next_index\| batch.storage[next_index.toIndex()].pending.node.prev = pending.node.prev,
2264	}
2265
2266	switch (batch.completed.tail) {
2267	.none => batch.completed.head = index,
2268	else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next = index,
2269	}
2270	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2271	batch.completed.tail = index;
2272
2273	source.as_object().release();
2274	const queue: c.dispatch.queue_t = @ptrCast(batch.userdata);
2275	const waiter: *BatchWaiter = @ptrCast(@alignCast(queue.as_object().get_context()));
2276	BatchWaiter.signal(waiter);
2277	}
2278
2279	fn batchSourceCancel(context: ?*anyopaque) callconv(.c) void {
2280	const storage: *Io.Operation.Storage = @ptrCast(@alignCast(context));
2281	const pending = &storage.pending;
2282	const operation_userdata: *BatchOperationUserdata = .fromErased(&pending.userdata);
2283	const batch = operation_userdata.batch;
2284	const source = operation_userdata.source;
2285	const index: Io.Operation.OptionalIndex = .fromIndex(storage - batch.storage.ptr);
2286
2287	switch (pending.node.prev) {
2288	.none => batch.pending.head = pending.node.next,
2289	else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.next = pending.node.next,
2290	}
2291	switch (pending.node.next) {
2292	.none => batch.pending.tail = pending.node.prev,
2293	else => \|next_index\| batch.storage[next_index.toIndex()].pending.node.prev = pending.node.prev,
2294	}
2295
2296	const tail_index = batch.unused.tail;
2297	switch (tail_index) {
2298	.none => batch.unused.head = index,
2299	else => batch.storage[tail_index.toIndex()].unused.next = index,
2300	}
2301	storage.* = .{ .unused = .{ .prev = tail_index, .next = .none } };
2302	batch.unused.tail = index;
2303
2304	source.as_object().release();
2305	if (batch.pending.head != .none) return;
2306	const queue: c.dispatch.queue_t = @ptrCast(batch.userdata);
2307	const waiter: *BatchWaiter = @ptrCast(@alignCast(queue.as_object().get_context()));
2308	queue.as_object().release();
2309	waiter.wake();
2310	}
2311
2312	fn dirCreateDir(
2313	userdata: ?*anyopaque,
2314	dir: Dir,
2315	sub_path: []const u8,
2316	permissions: Dir.Permissions,
2317	) Dir.CreateDirError!void {
2318	const ev: *Evented = @ptrCast(@alignCast(userdata));
2319	_ = ev;
2320
2321	var path_buffer: [c.PATH_MAX]u8 = undefined;
2322	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2323
2324	while (true) {
2325	switch (c.errno(c.mkdirat(dir.handle, sub_path_posix, permissions.toMode()))) {
2326	.SUCCESS => return,
2327	.INTR => {},
2328	.ACCES => return error.AccessDenied,
2329	.PERM => return error.PermissionDenied,
2330	.DQUOT => return error.DiskQuota,
2331	.EXIST => return error.PathAlreadyExists,
2332	.LOOP => return error.SymLinkLoop,
2333	.MLINK => return error.LinkQuotaExceeded,
2334	.NAMETOOLONG => return error.NameTooLong,
2335	.NOENT => return error.FileNotFound,
2336	.NOMEM => return error.SystemResources,
2337	.NOSPC => return error.NoSpaceLeft,
2338	.NOTDIR => return error.NotDir,
2339	.ROFS => return error.ReadOnlyFileSystem,
2340	.ILSEQ => return error.BadPathName,
2341	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
2342	.FAULT => \|err\| return errnoBug(err),
2343	else => \|err\| return unexpectedErrno(err),
2344	}
2345	}
2346	}
2347
2348	fn dirCreateDirPath(
2349	userdata: ?*anyopaque,
2350	dir: Dir,
2351	sub_path: []const u8,
2352	permissions: Dir.Permissions,
2353	) Dir.CreateDirPathError!Dir.CreatePathStatus {
2354	const ev: *Evented = @ptrCast(@alignCast(userdata));
2355
2356	var it = Dir.path.componentIterator(sub_path);
2357	var status: Dir.CreatePathStatus = .existed;
2358	var component = it.last() orelse return error.BadPathName;
2359	while (true) {
2360	if (dirCreateDir(ev, dir, component.path, permissions)) \|_\| {
2361	status = .created;
2362	} else \|err\| switch (err) {
2363	error.PathAlreadyExists => {
2364	// It is important to return an error if it's not a directory
2365	// because otherwise a dangling symlink could cause an infinite
2366	// loop.
2367	const fstat = try dirStatFile(ev, dir, component.path, .{});
2368	if (fstat.kind != .directory) return error.NotDir;
2369	},
2370	error.FileNotFound => \|e\| {
2371	component = it.previous() orelse return e;
2372	continue;
2373	},
2374	else => \|e\| return e,
2375	}
2376	component = it.next() orelse return status;
2377	}
2378	}
2379
2380	fn dirCreateDirPathOpen(
2381	userdata: ?*anyopaque,
2382	dir: Dir,
2383	sub_path: []const u8,
2384	permissions: Dir.Permissions,
2385	options: Dir.OpenOptions,
2386	) Dir.CreateDirPathOpenError!Dir {
2387	const ev: *Evented = @ptrCast(@alignCast(userdata));
2388	return dirOpenDir(ev, dir, sub_path, options) catch \|err\| switch (err) {
2389	error.FileNotFound => {
2390	_ = try dirCreateDirPath(ev, dir, sub_path, permissions);
2391	return dirOpenDir(ev, dir, sub_path, options);
2392	},
2393	else => \|e\| return e,
2394	};
2395	}
2396
2397	fn dirOpenDir(
2398	userdata: ?*anyopaque,
2399	dir: Dir,
2400	sub_path: []const u8,
2401	options: Dir.OpenOptions,
2402	) Dir.OpenError!Dir {
2403	const ev: *Evented = @ptrCast(@alignCast(userdata));
2404	_ = ev;
2405
2406	var path_buffer: [c.PATH_MAX]u8 = undefined;
2407	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2408
2409	const flags: c.O = .{
2410	.ACCMODE = .RDONLY,
2411	.NOFOLLOW = !options.follow_symlinks,
2412	.DIRECTORY = true,
2413	.CLOEXEC = true,
2414	};
2415
2416	while (true) {
2417	const rc = c.openat(dir.handle, sub_path_posix, flags);
2418	switch (c.errno(rc)) {
2419	.SUCCESS => return .{ .handle = @intCast(rc) },
2420	.INTR => {},
2421	.INVAL => return error.BadPathName,
2422	.ACCES => return error.AccessDenied,
2423	.LOOP => return error.SymLinkLoop,
2424	.MFILE => return error.ProcessFdQuotaExceeded,
2425	.NAMETOOLONG => return error.NameTooLong,
2426	.NFILE => return error.SystemFdQuotaExceeded,
2427	.NODEV => return error.NoDevice,
2428	.NOENT => return error.FileNotFound,
2429	.NOMEM => return error.SystemResources,
2430	.NOTDIR => return error.NotDir,
2431	.PERM => return error.PermissionDenied,
2432	.NXIO => return error.NoDevice,
2433	.ILSEQ => return error.BadPathName,
2434	.FAULT => \|err\| return errnoBug(err),
2435	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
2436	.BUSY => \|err\| return errnoBug(err), // O_EXCL not passed
2437	else => \|err\| return unexpectedErrno(err),
2438	}
2439	}
2440	}
2441
2442	fn dirStat(userdata: ?*anyopaque, dir: Dir) Dir.StatError!Dir.Stat {
2443	const ev: *Evented = @ptrCast(@alignCast(userdata));
2444	return fileStat(ev, .{
2445	.handle = dir.handle,
2446	.flags = .{ .nonblocking = false },
2447	});
2448	}
2449
2450	fn dirStatFile(
2451	userdata: ?*anyopaque,
2452	dir: Dir,
2453	sub_path: []const u8,
2454	options: Dir.StatFileOptions,
2455	) Dir.StatFileError!File.Stat {
2456	const ev: *Evented = @ptrCast(@alignCast(userdata));
2457	_ = ev;
2458
2459	var path_buffer: [c.PATH_MAX]u8 = undefined;
2460	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2461
2462	const flags: u32 = if (options.follow_symlinks) 0 else c.AT.SYMLINK_NOFOLLOW;
2463
2464	while (true) {
2465	var stat = std.mem.zeroes(c.Stat);
2466	switch (c.errno(c.fstatat(dir.handle, sub_path_posix, &stat, flags))) {
2467	.SUCCESS => return statFromPosix(&stat),
2468	.INTR => {},
2469	.INVAL => \|err\| return errnoBug(err),
2470	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
2471	.NOMEM => return error.SystemResources,
2472	.ACCES => return error.AccessDenied,
2473	.PERM => return error.PermissionDenied,
2474	.FAULT => \|err\| return errnoBug(err),
2475	.NAMETOOLONG => return error.NameTooLong,
2476	.LOOP => return error.SymLinkLoop,
2477	.NOENT => return error.FileNotFound,
2478	.NOTDIR => return error.FileNotFound,
2479	.ILSEQ => return error.BadPathName,
2480	else => \|err\| return unexpectedErrno(err),
2481	}
2482	}
2483	}
2484
2485	fn dirAccess(
2486	userdata: ?*anyopaque,
2487	dir: Dir,
2488	sub_path: []const u8,
2489	options: Dir.AccessOptions,
2490	) Dir.AccessError!void {
2491	const ev: *Evented = @ptrCast(@alignCast(userdata));
2492	_ = ev;
2493
2494	var path_buffer: [c.PATH_MAX]u8 = undefined;
2495	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2496
2497	const flags: u32 = if (options.follow_symlinks) 0 else c.AT.SYMLINK_NOFOLLOW;
2498
2499	const mode: u32 =
2500	@as(u32, if (options.read) c.R_OK else 0) \|
2501	@as(u32, if (options.write) c.W_OK else 0) \|
2502	@as(u32, if (options.execute) c.X_OK else 0);
2503
2504	while (true) switch (c.errno(c.faccessat(dir.handle, sub_path_posix, mode, flags))) {
2505	.SUCCESS => return,
2506	.INTR => {},
2507	.ACCES => return error.AccessDenied,
2508	.PERM => return error.PermissionDenied,
2509	.ROFS => return error.ReadOnlyFileSystem,
2510	.LOOP => return error.SymLinkLoop,
2511	.TXTBSY => return error.FileBusy,
2512	.NOTDIR => return error.FileNotFound,
2513	.NOENT => return error.FileNotFound,
2514	.NAMETOOLONG => return error.NameTooLong,
2515	.INVAL => \|err\| return errnoBug(err),
2516	.FAULT => \|err\| return errnoBug(err),
2517	.IO => return error.InputOutput,
2518	.NOMEM => return error.SystemResources,
2519	.ILSEQ => return error.BadPathName,
2520	else => \|err\| return unexpectedErrno(err),
2521	};
2522	}
2523
2524	fn dirCreateFile(
2525	userdata: ?*anyopaque,
2526	dir: Dir,
2527	sub_path: []const u8,
2528	flags: File.CreateFlags,
2529	) File.OpenError!File {
2530	const ev: *Evented = @ptrCast(@alignCast(userdata));
2531	_ = ev;
2532
2533	var path_buffer: [c.PATH_MAX]u8 = undefined;
2534	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2535
2536	const os_flags: c.O = .{
2537	.ACCMODE = if (flags.read) .RDWR else .WRONLY,
2538	.NONBLOCK = flags.lock == .none or flags.lock_nonblocking,
2539	.SHLOCK = flags.lock == .shared,
2540	.EXLOCK = flags.lock == .exclusive,
2541	.CREAT = true,
2542	.TRUNC = flags.truncate,
2543	.EXCL = flags.exclusive,
2544	.CLOEXEC = true,
2545	};
2546
2547	const fd: c.fd_t = while (true) {
2548	const rc = c.openat(dir.handle, sub_path_posix, os_flags, flags.permissions.toMode());
2549	switch (c.errno(rc)) {
2550	.SUCCESS => break @intCast(rc),
2551	.INTR => {},
2552	.FAULT => \|err\| return errnoBug(err),
2553	.INVAL => return error.BadPathName,
2554	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
2555	.ACCES => return error.AccessDenied,
2556	.FBIG => return error.FileTooBig,
2557	.OVERFLOW => return error.FileTooBig,
2558	.ISDIR => return error.IsDir,
2559	.LOOP => return error.SymLinkLoop,
2560	.MFILE => return error.ProcessFdQuotaExceeded,
2561	.NAMETOOLONG => return error.NameTooLong,
2562	.NFILE => return error.SystemFdQuotaExceeded,
2563	.NODEV => return error.NoDevice,
2564	.NOENT => return error.FileNotFound,
2565	.NOMEM => return error.SystemResources,
2566	.NOSPC => return error.NoSpaceLeft,
2567	.NOTDIR => return error.NotDir,
2568	.PERM => return error.PermissionDenied,
2569	.EXIST => return error.PathAlreadyExists,
2570	.BUSY => return error.DeviceBusy,
2571	.OPNOTSUPP => return error.FileLocksUnsupported,
2572	.AGAIN => return error.WouldBlock,
2573	.TXTBSY => return error.FileBusy,
2574	.NXIO => return error.NoDevice,
2575	.ILSEQ => return error.BadPathName,
2576	else => \|err\| return unexpectedErrno(err),
2577	}
2578	};
2579	errdefer closeFd(fd);
2580
2581	return .{
2582	.handle = fd,
2583	.flags = .{ .nonblocking = os_flags.NONBLOCK },
2584	};
2585	}
2586
2587	fn dirCreateFileAtomic(
2588	userdata: ?*anyopaque,
2589	dir: Dir,
2590	dest_path: []const u8,
2591	options: Dir.CreateFileAtomicOptions,
2592	) Dir.CreateFileAtomicError!File.Atomic {
2593	const ev: *Evented = @ptrCast(@alignCast(userdata));
2594	if (Dir.path.dirname(dest_path)) \|dirname\| {
2595	const new_dir = if (options.make_path)
2596	dirCreateDirPathOpen(ev, dir, dirname, .default_dir, .{}) catch \|err\| switch (err) {
2597	// None of these make sense in this context.
2598	error.IsDir,
2599	error.Streaming,
2600	error.DiskQuota,
2601	error.PathAlreadyExists,
2602	error.LinkQuotaExceeded,
2603	error.PipeBusy,
2604	error.FileTooBig,
2605	error.FileLocksUnsupported,
2606	error.DeviceBusy,
2607	=> return error.Unexpected,
2608
2609	else => \|e\| return e,
2610	}
2611	else
2612	try dirOpenDir(ev, dir, dirname, .{});
2613	return ev.atomicFileInit(Dir.path.basename(dest_path), options.permissions, new_dir, true);
2614	}
2615	return ev.atomicFileInit(dest_path, options.permissions, dir, false);
2616	}
2617
2618	fn atomicFileInit(
2619	ev: *Evented,
2620	dest_basename: []const u8,
2621	permissions: File.Permissions,
2622	dir: Dir,
2623	close_dir_on_deinit: bool,
2624	) Dir.CreateFileAtomicError!File.Atomic {
2625	while (true) {
2626	var random_integer: u64 = undefined;
2627	random(ev, @ptrCast(&random_integer));
2628	const tmp_sub_path = std.fmt.hex(random_integer);
2629	const file = dirCreateFile(ev, dir, &tmp_sub_path, .{
2630	.permissions = permissions,
2631	.exclusive = true,
2632	}) catch \|err\| switch (err) {
2633	error.PathAlreadyExists => continue,
2634	error.DeviceBusy => continue,
2635	error.FileBusy => continue,
2636
2637	error.IsDir => return error.Unexpected, // No path components.
2638	error.FileTooBig => return error.Unexpected, // Creating, not opening.
2639	error.FileLocksUnsupported => return error.Unexpected, // Not asking for locks.
2640	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
2641
2642	else => \|e\| return e,
2643	};
2644	return .{
2645	.file = file,
2646	.file_basename_hex = random_integer,
2647	.dest_sub_path = dest_basename,
2648	.file_open = true,
2649	.file_exists = true,
2650	.close_dir_on_deinit = close_dir_on_deinit,
2651	.dir = dir,
2652	};
2653	}
2654	}
2655
2656	fn dirOpenFile(
2657	userdata: ?*anyopaque,
2658	dir: Dir,
2659	sub_path: []const u8,
2660	flags: File.OpenFlags,
2661	) File.OpenError!File {
2662	const ev: *Evented = @ptrCast(@alignCast(userdata));
2663
2664	var path_buffer: [c.PATH_MAX]u8 = undefined;
2665	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2666
2667	const os_flags: c.O = .{
2668	.ACCMODE = switch (flags.mode) {
2669	.read_only => .RDONLY,
2670	.write_only => .WRONLY,
2671	.read_write => .RDWR,
2672	},
2673	.NONBLOCK = flags.lock == .none or flags.lock_nonblocking,
2674	.SHLOCK = flags.lock == .shared,
2675	.EXLOCK = flags.lock == .exclusive,
2676	.NOFOLLOW = !flags.follow_symlinks,
2677	.NOCTTY = !flags.allow_ctty,
2678	.CLOEXEC = true,
2679	};
2680
2681	const fd: c.fd_t = while (true) {
2682	const rc = c.openat(dir.handle, sub_path_posix, os_flags);
2683	switch (c.errno(rc)) {
2684	.SUCCESS => break @intCast(rc),
2685	.INTR => {},
2686	.FAULT => \|err\| return errnoBug(err),
2687	.INVAL => return error.BadPathName,
2688	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
2689	.ACCES => return error.AccessDenied,
2690	.FBIG => return error.FileTooBig,
2691	.OVERFLOW => return error.FileTooBig,
2692	.ISDIR => return error.IsDir,
2693	.LOOP => return error.SymLinkLoop,
2694	.MFILE => return error.ProcessFdQuotaExceeded,
2695	.NAMETOOLONG => return error.NameTooLong,
2696	.NFILE => return error.SystemFdQuotaExceeded,
2697	.NODEV => return error.NoDevice,
2698	.NOENT => return error.FileNotFound,
2699	.NOMEM => return error.SystemResources,
2700	.NOSPC => return error.NoSpaceLeft,
2701	.NOTDIR => return error.NotDir,
2702	.PERM => return error.PermissionDenied,
2703	.EXIST => return error.PathAlreadyExists,
2704	.BUSY => return error.DeviceBusy,
2705	.OPNOTSUPP => return error.FileLocksUnsupported,
2706	.AGAIN => return error.WouldBlock,
2707	.TXTBSY => return error.FileBusy,
2708	.NXIO => return error.NoDevice,
2709	.ILSEQ => return error.BadPathName,
2710	else => \|err\| return unexpectedErrno(err),
2711	}
2712	};
2713	errdefer closeFd(fd);
2714
2715	if (!flags.allow_directory) {
2716	const is_dir = is_dir: {
2717	const stat = fileStat(ev, .{
2718	.handle = fd,
2719	.flags = .{ .nonblocking = false },
2720	}) catch \|err\| switch (err) {
2721	// The directory-ness is either unknown or unknowable
2722	error.Streaming => break :is_dir false,
2723	else => \|e\| return e,
2724	};
2725	break :is_dir stat.kind == .directory;
2726	};
2727	if (is_dir) return error.IsDir;
2728	}
2729
2730	return .{
2731	.handle = fd,
2732	.flags = .{ .nonblocking = os_flags.NONBLOCK },
2733	};
2734	}
2735
2736	fn dirClose(userdata: ?*anyopaque, dirs: []const Dir) void {
2737	const ev: *Evented = @ptrCast(@alignCast(userdata));
2738	_ = ev;
2739	for (dirs) \|dir\| closeFd(dir.handle);
2740	}
2741
2742	fn dirRead(userdata: ?anyopaque, dr: Dir.Reader, buffer: []Dir.Entry) Dir.Reader.Error!usize {
2743	const ev: *Evented = @ptrCast(@alignCast(userdata));
2744	const Header = extern struct {
2745	seek: i64,
2746	};
2747	const header: *Header = @ptrCast(dr.buffer.ptr);
2748	const header_end: usize = @sizeOf(Header);
2749	if (dr.index < header_end) {
2750	// Initialize header.
2751	dr.index = header_end;
2752	dr.end = header_end;
2753	header.* = .{ .seek = 0 };
2754	}
2755	var buffer_index: usize = 0;
2756	while (buffer.len - buffer_index != 0) {
2757	if (dr.end - dr.index == 0) {
2758	// Refill the buffer, unless we've already created references to
2759	// buffered data.
2760	if (buffer_index != 0) break;
2761	if (dr.state == .reset) {
2762	ev.lseek(dr.dir.handle, 0, c.SEEK.SET) catch \|err\| switch (err) {
2763	error.Unseekable => return error.Unexpected,
2764	else => \|e\| return e,
2765	};
2766	dr.state = .reading;
2767	}
2768	const dents_buffer = dr.buffer[header_end..];
2769	const n: usize = while (true) {
2770	const rc = c.getdirentries(dr.dir.handle, dents_buffer.ptr, dents_buffer.len, &header.seek);
2771	switch (c.errno(rc)) {
2772	.SUCCESS => break @intCast(rc),
2773	.INTR => {},
2774	.BADF => \|err\| return errnoBug(err), // Dir is invalid or was opened without iteration ability.
2775	.FAULT => \|err\| return errnoBug(err),
2776	.NOTDIR => \|err\| return errnoBug(err),
2777	.INVAL => \|err\| return errnoBug(err),
2778	else => \|err\| return unexpectedErrno(err),
2779	}
2780	};
2781	if (n == 0) {
2782	dr.state = .finished;
2783	return 0;
2784	}
2785	dr.index = header_end;
2786	dr.end = header_end + n;
2787	}
2788	const darwin_entry = @as(*align(1) c.dirent, @ptrCast(&dr.buffer[dr.index]));
2789	const next_index = dr.index + darwin_entry.reclen;
2790	dr.index = next_index;
2791
2792	const name = @as([*]u8, @ptrCast(&darwin_entry.name))[0..darwin_entry.namlen];
2793	if (std.mem.eql(u8, name, ".") or std.mem.eql(u8, name, "..") or (darwin_entry.ino == 0))
2794	continue;
2795
2796	const entry_kind: File.Kind = switch (darwin_entry.type) {
2797	c.DT.BLK => .block_device,
2798	c.DT.CHR => .character_device,
2799	c.DT.DIR => .directory,
2800	c.DT.FIFO => .named_pipe,
2801	c.DT.LNK => .sym_link,
2802	c.DT.REG => .file,
2803	c.DT.SOCK => .unix_domain_socket,
2804	c.DT.WHT => .whiteout,
2805	else => .unknown,
2806	};
2807	buffer[buffer_index] = .{
2808	.name = name,
2809	.kind = entry_kind,
2810	.inode = darwin_entry.ino,
2811	};
2812	buffer_index += 1;
2813	}
2814	return buffer_index;
2815	}
2816
2817	fn dirRealPath(userdata: ?*anyopaque, dir: Dir, out_buffer: []u8) Dir.RealPathError!usize {
2818	const ev: *Evented = @ptrCast(@alignCast(userdata));
2819	return ev.realPath(dir.handle, out_buffer);
2820	}
2821
2822	fn realPath(ev: *Evented, fd: c.fd_t, out_buffer: []u8) File.RealPathError!usize {
2823	_ = ev;
2824	var buffer: [c.PATH_MAX]u8 = undefined;
2825	@memset(&buffer, 0);
2826	while (true) {
2827	switch (c.errno(c.fcntl(fd, c.F.GETPATH, &buffer))) {
2828	.SUCCESS => break,
2829	.INTR => {},
2830	.ACCES => return error.AccessDenied,
2831	.BADF => return error.FileNotFound,
2832	.NOENT => return error.FileNotFound,
2833	.NOMEM => return error.SystemResources,
2834	.NOSPC => return error.NameTooLong,
2835	.RANGE => return error.NameTooLong,
2836	else => \|err\| return unexpectedErrno(err),
2837	}
2838	}
2839	const n = std.mem.indexOfScalar(u8, &buffer, 0) orelse buffer.len;
2840	if (n > out_buffer.len) return error.NameTooLong;
2841	@memcpy(out_buffer[0..n], buffer[0..n]);
2842	return n;
2843	}
2844
2845	fn dirRealPathFile(
2846	userdata: ?*anyopaque,
2847	dir: Dir,
2848	sub_path: []const u8,
2849	out_buffer: []u8,
2850	) Dir.RealPathFileError!usize {
2851	const ev: *Evented = @ptrCast(@alignCast(userdata));
2852
2853	var path_buffer: [c.PATH_MAX]u8 = undefined;
2854	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2855
2856	if (dir.handle == c.AT.FDCWD) {
2857	if (out_buffer.len < c.PATH_MAX) return error.NameTooLong;
2858	while (true) {
2859	if (c.realpath(sub_path_posix, out_buffer.ptr)) \|redundant_pointer\| {
2860	assert(redundant_pointer == out_buffer.ptr);
2861	return std.mem.indexOfScalar(u8, out_buffer, 0) orelse out_buffer.len;
2862	}
2863	const err: c.E = @enumFromInt(c._errno().*);
2864	switch (err) {
2865	.INTR => {},
2866	.INVAL => return errnoBug(err),
2867	.BADF => return errnoBug(err),
2868	.FAULT => return errnoBug(err),
2869	.ACCES => return error.AccessDenied,
2870	.NOENT => return error.FileNotFound,
2871	.OPNOTSUPP => return error.OperationUnsupported,
2872	.NOTDIR => return error.NotDir,
2873	.NAMETOOLONG => return error.NameTooLong,
2874	.LOOP => return error.SymLinkLoop,
2875	.IO => return error.InputOutput,
2876	else => return unexpectedErrno(err),
2877	}
2878	}
2879	}
2880
2881	const os_flags: c.O = .{
2882	.NONBLOCK = true,
2883	.CLOEXEC = true,
2884	};
2885
2886	const fd: c.fd_t = while (true) {
2887	const rc = c.openat(dir.handle, sub_path_posix, os_flags);
2888	switch (c.errno(rc)) {
2889	.SUCCESS => break @intCast(rc),
2890	.INTR => {},
2891	.FAULT => \|err\| return errnoBug(err),
2892	.INVAL => return error.BadPathName,
2893	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
2894	.ACCES => return error.AccessDenied,
2895	.FBIG => return error.FileTooBig,
2896	.OVERFLOW => return error.FileTooBig,
2897	.ISDIR => return error.IsDir,
2898	.LOOP => return error.SymLinkLoop,
2899	.MFILE => return error.ProcessFdQuotaExceeded,
2900	.NAMETOOLONG => return error.NameTooLong,
2901	.NFILE => return error.SystemFdQuotaExceeded,
2902	.NODEV => return error.NoDevice,
2903	.NOENT => return error.FileNotFound,
2904	.NOMEM => return error.SystemResources,
2905	.NOSPC => return error.NoSpaceLeft,
2906	.NOTDIR => return error.NotDir,
2907	.PERM => return error.PermissionDenied,
2908	.EXIST => return error.PathAlreadyExists,
2909	.BUSY => return error.DeviceBusy,
2910	.NXIO => return error.NoDevice,
2911	.ILSEQ => return error.BadPathName,
2912	else => \|err\| return unexpectedErrno(err),
2913	}
2914	};
2915	defer closeFd(fd);
2916	return ev.realPath(fd, out_buffer);
2917	}
2918
2919	fn dirDeleteFile(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteFileError!void {
2920	const ev: *Evented = @ptrCast(@alignCast(userdata));
2921	_ = ev;
2922
2923	var path_buffer: [c.PATH_MAX]u8 = undefined;
2924	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2925
2926	while (true) switch (c.errno(c.unlinkat(dir.handle, sub_path_posix, 0))) {
2927	.SUCCESS => return,
2928	.INTR => {},
2929	// Some systems return permission errors when trying to delete a
2930	// directory, so we need to handle that case specifically and
2931	// translate the error.
2932	.PERM => {
2933	// Don't follow symlinks to match unlinkat (which acts on symlinks rather than follows them).
2934	var st = std.mem.zeroes(c.Stat);
2935	while (true) switch (c.errno(c.fstatat(
2936	dir.handle,
2937	sub_path_posix,
2938	&st,
2939	c.AT.SYMLINK_NOFOLLOW,
2940	))) {
2941	.SUCCESS => break,
2942	.INTR => {},
2943	else => return error.PermissionDenied,
2944	};
2945	if (st.mode & c.S.IFMT == c.S.IFDIR) return error.IsDir else return error.PermissionDenied;
2946	},
2947	.ACCES => return error.AccessDenied,
2948	.BUSY => return error.FileBusy,
2949	.FAULT => \|err\| return errnoBug(err),
2950	.IO => return error.FileSystem,
2951	.ISDIR => return error.IsDir,
2952	.LOOP => return error.SymLinkLoop,
2953	.NAMETOOLONG => return error.NameTooLong,
2954	.NOENT => return error.FileNotFound,
2955	.NOTDIR => return error.NotDir,
2956	.NOMEM => return error.SystemResources,
2957	.ROFS => return error.ReadOnlyFileSystem,
2958	.EXIST => \|err\| return errnoBug(err),
2959	.NOTEMPTY => \|err\| return errnoBug(err), // Not passing AT.REMOVEDIR
2960	.ILSEQ => return error.BadPathName,
2961	.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
2962	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
2963	else => \|err\| return unexpectedErrno(err),
2964	};
2965	}
2966
2967	fn dirDeleteDir(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteDirError!void {
2968	const ev: *Evented = @ptrCast(@alignCast(userdata));
2969	_ = ev;
2970
2971	var path_buffer: [c.PATH_MAX]u8 = undefined;
2972	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2973
2974	while (true) switch (c.errno(c.unlinkat(dir.handle, sub_path_posix, c.AT.REMOVEDIR))) {
2975	.SUCCESS => return,
2976	.INTR => {},
2977	.ACCES => return error.AccessDenied,
2978	.PERM => return error.PermissionDenied,
2979	.BUSY => return error.FileBusy,
2980	.FAULT => \|err\| return errnoBug(err),
2981	.IO => return error.FileSystem,
2982	.ISDIR => \|err\| return errnoBug(err),
2983	.LOOP => return error.SymLinkLoop,
2984	.NAMETOOLONG => return error.NameTooLong,
2985	.NOENT => return error.FileNotFound,
2986	.NOTDIR => return error.NotDir,
2987	.NOMEM => return error.SystemResources,
2988	.ROFS => return error.ReadOnlyFileSystem,
2989	.EXIST => \|err\| return errnoBug(err),
2990	.NOTEMPTY => return error.DirNotEmpty,
2991	.ILSEQ => return error.BadPathName,
2992	.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
2993	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
2994	else => \|err\| return unexpectedErrno(err),
2995	};
2996	}
2997
2998	fn dirRename(
2999	userdata: ?*anyopaque,
3000	old_dir: Dir,
3001	old_sub_path: []const u8,
3002	new_dir: Dir,
3003	new_sub_path: []const u8,
3004	) Dir.RenameError!void {
3005	const ev: *Evented = @ptrCast(@alignCast(userdata));
3006	_ = ev;
3007
3008	var old_path_buffer: [c.PATH_MAX]u8 = undefined;
3009	var new_path_buffer: [c.PATH_MAX]u8 = undefined;
3010
3011	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
3012	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
3013
3014	while (true) switch (c.errno(c.renameat(old_dir.handle, old_sub_path_posix, new_dir.handle, new_sub_path_posix))) {
3015	.SUCCESS => return,
3016	.INTR => {},
3017	.ACCES => return error.AccessDenied,
3018	.PERM => return error.PermissionDenied,
3019	.BUSY => return error.FileBusy,
3020	.DQUOT => return error.DiskQuota,
3021	.ISDIR => return error.IsDir,
3022	.IO => return error.HardwareFailure,
3023	.LOOP => return error.SymLinkLoop,
3024	.MLINK => return error.LinkQuotaExceeded,
3025	.NAMETOOLONG => return error.NameTooLong,
3026	.NOENT => return error.FileNotFound,
3027	.NOTDIR => return error.NotDir,
3028	.NOMEM => return error.SystemResources,
3029	.NOSPC => return error.NoSpaceLeft,
3030	.EXIST => return error.DirNotEmpty,
3031	.NOTEMPTY => return error.DirNotEmpty,
3032	.ROFS => return error.ReadOnlyFileSystem,
3033	.XDEV => return error.CrossDevice,
3034	.ILSEQ => return error.BadPathName,
3035	.FAULT => \|err\| return errnoBug(err),
3036	.INVAL => \|err\| return errnoBug(err),
3037	else => \|err\| return unexpectedErrno(err),
3038	};
3039	}
3040
3041	fn dirRenamePreserve(
3042	userdata: ?*anyopaque,
3043	old_dir: Dir,
3044	old_sub_path: []const u8,
3045	new_dir: Dir,
3046	new_sub_path: []const u8,
3047	) Dir.RenamePreserveError!void {
3048	const ev: *Evented = @ptrCast(@alignCast(userdata));
3049	// Make a hard link then delete the original.
3050	try dirHardLink(ev, old_dir, old_sub_path, new_dir, new_sub_path, .{ .follow_symlinks = false });
3051	const prev = swapCancelProtection(ev, .blocked);
3052	defer _ = swapCancelProtection(ev, prev);
3053	dirDeleteFile(ev, old_dir, old_sub_path) catch {};
3054	}
3055
3056	fn dirSymLink(
3057	userdata: ?*anyopaque,
3058	dir: Dir,
3059	target_path: []const u8,
3060	sym_link_path: []const u8,
3061	flags: Dir.SymLinkFlags,
3062	) Dir.SymLinkError!void {
3063	const ev: *Evented = @ptrCast(@alignCast(userdata));
3064	_ = ev;
3065	_ = flags;
3066
3067	var target_path_buffer: [c.PATH_MAX]u8 = undefined;
3068	var sym_link_path_buffer: [c.PATH_MAX]u8 = undefined;
3069
3070	const target_path_posix = try pathToPosix(target_path, &target_path_buffer);
3071	const sym_link_path_posix = try pathToPosix(sym_link_path, &sym_link_path_buffer);
3072
3073	while (true) switch (c.errno(c.symlinkat(target_path_posix, dir.handle, sym_link_path_posix))) {
3074	.SUCCESS => return,
3075	.INTR => {},
3076	.FAULT => \|err\| return errnoBug(err),
3077	.INVAL => \|err\| return errnoBug(err),
3078	.ACCES => return error.AccessDenied,
3079	.PERM => return error.PermissionDenied,
3080	.DQUOT => return error.DiskQuota,
3081	.EXIST => return error.PathAlreadyExists,
3082	.IO => return error.FileSystem,
3083	.LOOP => return error.SymLinkLoop,
3084	.NAMETOOLONG => return error.NameTooLong,
3085	.NOENT => return error.FileNotFound,
3086	.NOTDIR => return error.NotDir,
3087	.NOMEM => return error.SystemResources,
3088	.NOSPC => return error.NoSpaceLeft,
3089	.ROFS => return error.ReadOnlyFileSystem,
3090	.ILSEQ => return error.BadPathName,
3091	else => \|err\| return unexpectedErrno(err),
3092	};
3093	}
3094
3095	fn dirReadLink(
3096	userdata: ?*anyopaque,
3097	dir: Dir,
3098	sub_path: []const u8,
3099	buffer: []u8,
3100	) Dir.ReadLinkError!usize {
3101	const ev: *Evented = @ptrCast(@alignCast(userdata));
3102	_ = ev;
3103	var sub_path_buffer: [c.PATH_MAX]u8 = undefined;
3104	const sub_path_posix = try pathToPosix(sub_path, &sub_path_buffer);
3105	while (true) {
3106	const rc = c.readlinkat(dir.handle, sub_path_posix, buffer.ptr, buffer.len);
3107	switch (c.errno(rc)) {
3108	.SUCCESS => return @intCast(rc),
3109	.INTR => {},
3110	.ACCES => return error.AccessDenied,
3111	.FAULT => \|err\| return errnoBug(err),
3112	.INVAL => return error.NotLink,
3113	.IO => return error.FileSystem,
3114	.LOOP => return error.SymLinkLoop,
3115	.NAMETOOLONG => return error.NameTooLong,
3116	.NOENT => return error.FileNotFound,
3117	.NOMEM => return error.SystemResources,
3118	.NOTDIR => return error.NotDir,
3119	.ILSEQ => return error.BadPathName,
3120	else => \|err\| return unexpectedErrno(err),
3121	}
3122	}
3123	}
3124
3125	fn dirSetOwner(
3126	userdata: ?*anyopaque,
3127	dir: Dir,
3128	owner: ?File.Uid,
3129	group: ?File.Gid,
3130	) Dir.SetOwnerError!void {
3131	const ev: *Evented = @ptrCast(@alignCast(userdata));
3132	_ = ev;
3133	return fchown(dir.handle, owner, group);
3134	}
3135
3136	fn fchown(fd: c.fd_t, owner: ?File.Uid, group: ?File.Gid) File.SetOwnerError!void {
3137	const uid = owner orelse std.math.maxInt(c.uid_t);
3138	const gid = group orelse std.math.maxInt(c.gid_t);
3139	while (true) switch (c.errno(c.fchown(fd, uid, gid))) {
3140	.SUCCESS => return,
3141	.INTR => {},
3142	.BADF => \|err\| return errnoBug(err), // likely fd refers to directory opened without `Dir.OpenOptions.iterate`
3143	.FAULT => \|err\| return errnoBug(err),
3144	.INVAL => \|err\| return errnoBug(err),
3145	.ACCES => return error.AccessDenied,
3146	.IO => return error.InputOutput,
3147	.LOOP => return error.SymLinkLoop,
3148	.NOENT => return error.FileNotFound,
3149	.NOMEM => return error.SystemResources,
3150	.NOTDIR => return error.FileNotFound,
3151	.PERM => return error.PermissionDenied,
3152	.ROFS => return error.ReadOnlyFileSystem,
3153	else => \|err\| return unexpectedErrno(err),
3154	};
3155	}
3156
3157	fn dirSetFileOwner(
3158	userdata: ?*anyopaque,
3159	dir: Dir,
3160	sub_path: []const u8,
3161	owner: ?File.Uid,
3162	group: ?File.Gid,
3163	options: Dir.SetFileOwnerOptions,
3164	) Dir.SetFileOwnerError!void {
3165	const ev: *Evented = @ptrCast(@alignCast(userdata));
3166	var path_buffer: [c.PATH_MAX]u8 = undefined;
3167	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3168	_ = ev;
3169	while (true) switch (c.errno(c.fchownat(
3170	dir.handle,
3171	sub_path_posix,
3172	owner orelse std.math.maxInt(c.uid_t),
3173	group orelse std.math.maxInt(c.gid_t),
3174	if (options.follow_symlinks) 0 else c.AT.SYMLINK_NOFOLLOW,
3175	))) {
3176	.SUCCESS => return,
3177	.INTR => continue,
3178	.BADF => \|err\| return errnoBug(err), // likely fd refers to directory opened without `Dir.OpenOptions.iterate`
3179	.FAULT => \|err\| return errnoBug(err),
3180	.INVAL => \|err\| return errnoBug(err),
3181	.ACCES => return error.AccessDenied,
3182	.IO => return error.InputOutput,
3183	.LOOP => return error.SymLinkLoop,
3184	.NOENT => return error.FileNotFound,
3185	.NOMEM => return error.SystemResources,
3186	.NOTDIR => return error.FileNotFound,
3187	.PERM => return error.PermissionDenied,
3188	.ROFS => return error.ReadOnlyFileSystem,
3189	else => \|err\| return unexpectedErrno(err),
3190	};
3191	}
3192
3193	fn dirSetPermissions(
3194	userdata: ?*anyopaque,
3195	dir: Dir,
3196	permissions: Dir.Permissions,
3197	) Dir.SetPermissionsError!void {
3198	const ev: *Evented = @ptrCast(@alignCast(userdata));
3199	return ev.fchmod(dir.handle, permissions.toMode());
3200	}
3201
3202	fn dirSetFilePermissions(
3203	userdata: ?*anyopaque,
3204	dir: Dir,
3205	sub_path: []const u8,
3206	permissions: Dir.Permissions,
3207	options: Dir.SetFilePermissionsOptions,
3208	) Dir.SetFilePermissionsError!void {
3209	const ev: *Evented = @ptrCast(@alignCast(userdata));
3210	_ = ev;
3211
3212	var path_buffer: [c.PATH_MAX]u8 = undefined;
3213	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3214
3215	const mode = permissions.toMode();
3216	const flags: u32 = if (options.follow_symlinks) 0 else c.AT.SYMLINK_NOFOLLOW;
3217
3218	while (true) switch (c.errno(c.fchmodat(dir.handle, sub_path_posix, mode, flags))) {
3219	.SUCCESS => return,
3220	.INTR => {},
3221	.BADF => \|err\| return errnoBug(err),
3222	.FAULT => \|err\| return errnoBug(err),
3223	.INVAL => \|err\| return errnoBug(err),
3224	.ACCES => return error.AccessDenied,
3225	.IO => return error.InputOutput,
3226	.LOOP => return error.SymLinkLoop,
3227	.MFILE => return error.ProcessFdQuotaExceeded,
3228	.NAMETOOLONG => return error.NameTooLong,
3229	.NFILE => return error.SystemFdQuotaExceeded,
3230	.NOENT => return error.FileNotFound,
3231	.NOTDIR => return error.FileNotFound,
3232	.NOMEM => return error.SystemResources,
3233	.OPNOTSUPP => return error.OperationUnsupported,
3234	.PERM => return error.PermissionDenied,
3235	.ROFS => return error.ReadOnlyFileSystem,
3236	else => \|err\| return unexpectedErrno(err),
3237	};
3238	}
3239
3240	fn dirSetTimestamps(
3241	userdata: ?*anyopaque,
3242	dir: Dir,
3243	sub_path: []const u8,
3244	options: Dir.SetTimestampsOptions,
3245	) Dir.SetTimestampsError!void {
3246	const ev: *Evented = @ptrCast(@alignCast(userdata));
3247	_ = ev;
3248
3249	var times_buffer: [2]c.timespec = undefined;
3250	const times = if (options.modify_timestamp == .now and options.access_timestamp == .now) null else p: {
3251	times_buffer = .{
3252	setTimestampToPosix(options.access_timestamp),
3253	setTimestampToPosix(options.modify_timestamp),
3254	};
3255	break :p &times_buffer;
3256	};
3257
3258	const flags: u32 = if (options.follow_symlinks) 0 else c.AT.SYMLINK_NOFOLLOW;
3259
3260	var path_buffer: [c.PATH_MAX]u8 = undefined;
3261	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3262
3263	while (true) switch (c.errno(c.utimensat(dir.handle, sub_path_posix, times, flags))) {
3264	.SUCCESS => return,
3265	.INTR => {},
3266	.BADF => \|err\| return errnoBug(err), // always a race condition
3267	.FAULT => \|err\| return errnoBug(err),
3268	.INVAL => \|err\| return errnoBug(err),
3269	.ACCES => return error.AccessDenied,
3270	.PERM => return error.PermissionDenied,
3271	.ROFS => return error.ReadOnlyFileSystem,
3272	else => \|err\| return unexpectedErrno(err),
3273	};
3274	}
3275
3276	fn dirHardLink(
3277	userdata: ?*anyopaque,
3278	old_dir: Dir,
3279	old_sub_path: []const u8,
3280	new_dir: Dir,
3281	new_sub_path: []const u8,
3282	options: Dir.HardLinkOptions,
3283	) Dir.HardLinkError!void {
3284	const ev: *Evented = @ptrCast(@alignCast(userdata));
3285	_ = ev;
3286
3287	var old_path_buffer: [c.PATH_MAX]u8 = undefined;
3288	var new_path_buffer: [c.PATH_MAX]u8 = undefined;
3289
3290	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
3291	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
3292
3293	const flags: u32 = if (options.follow_symlinks) c.AT.SYMLINK_FOLLOW else 0;
3294	return linkat(old_dir.handle, old_sub_path_posix, new_dir.handle, new_sub_path_posix, flags);
3295	}
3296
3297	fn fileStat(userdata: ?*anyopaque, file: File) File.StatError!File.Stat {
3298	const ev: *Evented = @ptrCast(@alignCast(userdata));
3299	_ = ev;
3300	while (true) {
3301	var stat = std.mem.zeroes(c.Stat);
3302	switch (c.errno(c.fstat(file.handle, &stat))) {
3303	.SUCCESS => return statFromPosix(&stat),
3304	.INTR => {},
3305	.INVAL => \|err\| return errnoBug(err),
3306	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
3307	.NOMEM => return error.SystemResources,
3308	.ACCES => return error.AccessDenied,
3309	else => \|err\| return unexpectedErrno(err),
3310	}
3311	}
3312	}
3313
3314	fn fileLength(userdata: ?*anyopaque, file: File) File.LengthError!u64 {
3315	const ev: *Evented = @ptrCast(@alignCast(userdata));
3316	const stat = try fileStat(ev, file);
3317	return stat.size;
3318	}
3319
3320	fn fileClose(userdata: ?*anyopaque, files: []const File) void {
3321	const ev: *Evented = @ptrCast(@alignCast(userdata));
3322	_ = ev;
3323	for (files) \|file\| closeFd(file.handle);
3324	}
3325
3326	fn fileWritePositional(
3327	userdata: ?*anyopaque,
3328	file: File,
3329	header: []const u8,
3330	data: []const []const u8,
3331	splat: usize,
3332	offset: u64,
3333	) File.WritePositionalError!usize {
3334	const ev: *Evented = @ptrCast(@alignCast(userdata));
3335	_ = ev;
3336	var iovecs: [max_iovecs_len]iovec_const = undefined;
3337	var iovlen: iovlen_t = 0;
3338	var remaining: Io.Limit = .unlimited;
3339	addBuf(true, &iovecs, &iovlen, &remaining, header);
3340	for (data[0 .. data.len - 1]) \|bytes\| addBuf(true, &iovecs, &iovlen, &remaining, bytes);
3341	const pattern = data[data.len - 1];
3342	var backup_buffer: [splat_buffer_size]u8 = undefined;
3343	if (iovecs.len - iovlen != 0 and remaining != .nothing) switch (splat) {
3344	0 => {},
3345	1 => addBuf(true, &iovecs, &iovlen, &remaining, pattern),
3346	else => switch (pattern.len) {
3347	0 => {},
3348	1 => {
3349	const splat_buffer = &backup_buffer;
3350	const memset_len = @min(splat_buffer.len, splat);
3351	const buf = splat_buffer[0..memset_len];
3352	@memset(buf, pattern[0]);
3353	addBuf(true, &iovecs, &iovlen, &remaining, buf);
3354	var remaining_splat = splat - buf.len;
3355	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0 and remaining != .nothing) {
3356	assert(buf.len == splat_buffer.len);
3357	addBuf(true, &iovecs, &iovlen, &remaining, splat_buffer);
3358	remaining_splat -= splat_buffer.len;
3359	}
3360	addBuf(true, &iovecs, &iovlen, &remaining, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
3361	},
3362	else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
3363	if (remaining == .nothing) break;
3364	addBuf(true, &iovecs, &iovlen, &remaining, pattern);
3365	},
3366	},
3367	};
3368	if (iovlen == 0) return 0;
3369	while (true) {
3370	const rc = c.pwritev(file.handle, &iovecs, iovlen, @bitCast(offset));
3371	switch (c.errno(rc)) {
3372	.SUCCESS => return @intCast(rc),
3373	.INTR => {},
3374	.INVAL => \|err\| return errnoBug(err),
3375	.FAULT => \|err\| return errnoBug(err),
3376	.DESTADDRREQ => \|err\| return errnoBug(err), // `connect` was never called.
3377	.CONNRESET => \|err\| return errnoBug(err), // Not a socket handle.
3378	.BADF => return error.NotOpenForWriting,
3379	.AGAIN => return error.WouldBlock,
3380	.DQUOT => return error.DiskQuota,
3381	.FBIG => return error.FileTooBig,
3382	.IO => return error.InputOutput,
3383	.NOSPC => return error.NoSpaceLeft,
3384	.PERM => return error.PermissionDenied,
3385	.PIPE => return error.BrokenPipe,
3386	.BUSY => return error.DeviceBusy,
3387	.TXTBSY => return error.FileBusy,
3388	.NXIO => return error.Unseekable,
3389	.SPIPE => return error.Unseekable,
3390	.OVERFLOW => return error.Unseekable,
3391	else => \|err\| return unexpectedErrno(err),
3392	}
3393	}
3394	}
3395
3396	fn fileWriteFileStreaming(
3397	userdata: ?*anyopaque,
3398	file: File,
3399	header: []const u8,
3400	file_reader: *File.Reader,
3401	limit: Io.Limit,
3402	) File.Writer.WriteFileError!usize {
3403	const ev: *Evented = @ptrCast(@alignCast(userdata));
3404	const reader_buffered = file_reader.interface.buffered();
3405	if (reader_buffered.len >= @intFromEnum(limit)) {
3406	const n = try fileWriteStreaming(ev, file, header, &.{limit.slice(reader_buffered)}, 1);
3407	file_reader.interface.toss(n -\| header.len);
3408	return n;
3409	}
3410	const file_limit = @intFromEnum(limit) - reader_buffered.len;
3411	const out_fd = file.handle;
3412	const in_fd = file_reader.file.handle;
3413
3414	if (file_reader.size) \|size\| {
3415	if (size - file_reader.pos == 0) {
3416	if (reader_buffered.len != 0) {
3417	const n = try fileWriteStreaming(ev, file, header, &.{limit.slice(reader_buffered)}, 1);
3418	file_reader.interface.toss(n -\| header.len);
3419	return n;
3420	} else {
3421	return error.EndOfStream;
3422	}
3423	}
3424	}
3425
3426	if (@atomicLoad(UseSendfile, &ev.use_sendfile, .monotonic) == .disabled) return error.Unimplemented;
3427	const offset = std.math.cast(c.off_t, file_reader.pos) orelse return error.Unimplemented;
3428	var hdtr_data: c.sf_hdtr = undefined;
3429	var headers: [2]iovec_const = undefined;
3430	var headers_i: u8 = 0;
3431	if (header.len != 0) {
3432	headers[headers_i] = .{ .base = header.ptr, .len = header.len };
3433	headers_i += 1;
3434	}
3435	if (reader_buffered.len != 0) {
3436	headers[headers_i] = .{ .base = reader_buffered.ptr, .len = reader_buffered.len };
3437	headers_i += 1;
3438	}
3439	const hdtr: ?*c.sf_hdtr = if (headers_i == 0) null else b: {
3440	hdtr_data = .{
3441	.headers = &headers,
3442	.hdr_cnt = headers_i,
3443	.trailers = null,
3444	.trl_cnt = 0,
3445	};
3446	break :b &hdtr_data;
3447	};
3448	const max_count = std.math.maxInt(i32); // Avoid EINVAL.
3449	var len: c.off_t = @min(file_limit, max_count);
3450	const flags = 0;
3451	while (true) switch (c.errno(c.sendfile(in_fd, out_fd, offset, &len, hdtr, flags))) {
3452	.SUCCESS => break,
3453	.OPNOTSUPP, .NOTSOCK, .NOSYS => {
3454	// Give calling code chance to observe before trying
3455	// something else.
3456	@atomicStore(UseSendfile, &ev.use_sendfile, .disabled, .monotonic);
3457	return 0;
3458	},
3459	.INTR => if (len > 0) break,
3460	.AGAIN => {
3461	if (len == 0) return error.WouldBlock;
3462	break;
3463	},
3464	else => \|e\| {
3465	assert(error.Unexpected == switch (e) {
3466	.NOTCONN => return error.BrokenPipe,
3467	.IO => return error.InputOutput,
3468	.PIPE => return error.BrokenPipe,
3469	.BADF => \|err\| errnoBug(err),
3470	.FAULT => \|err\| errnoBug(err),
3471	.INVAL => \|err\| errnoBug(err),
3472	else => \|err\| unexpectedErrno(err),
3473	});
3474	// Give calling code chance to observe the error before trying
3475	// something else.
3476	@atomicStore(UseSendfile, &ev.use_sendfile, .disabled, .monotonic);
3477	return 0;
3478	},
3479	};
3480	if (len == 0) {
3481	file_reader.size = file_reader.pos;
3482	return error.EndOfStream;
3483	}
3484	const u_len: usize = @bitCast(len);
3485	file_reader.interface.toss(u_len -\| header.len);
3486	return u_len;
3487	}
3488
3489	fn fileWriteFilePositional(
3490	userdata: ?*anyopaque,
3491	file: File,
3492	header: []const u8,
3493	file_reader: *File.Reader,
3494	limit: Io.Limit,
3495	offset: u64,
3496	) File.WriteFilePositionalError!usize {
3497	const ev: *Evented = @ptrCast(@alignCast(userdata));
3498	const reader_buffered = file_reader.interface.buffered();
3499	if (reader_buffered.len >= @intFromEnum(limit)) {
3500	const n = try fileWritePositional(
3501	ev,
3502	file,
3503	header,
3504	&.{limit.slice(reader_buffered)},
3505	1,
3506	offset,
3507	);
3508	file_reader.interface.toss(n -\| header.len);
3509	return n;
3510	}
3511	const out_fd = file.handle;
3512	const in_fd = file_reader.file.handle;
3513
3514	if (file_reader.size) \|size\| {
3515	if (size - file_reader.pos == 0) {
3516	if (reader_buffered.len != 0) {
3517	const n = try fileWritePositional(
3518	ev,
3519	file,
3520	header,
3521	&.{limit.slice(reader_buffered)},
3522	1,
3523	offset,
3524	);
3525	file_reader.interface.toss(n -\| header.len);
3526	return n;
3527	} else {
3528	return error.EndOfStream;
3529	}
3530	}
3531	}
3532
3533	if (@atomicLoad(UseFcopyfile, &ev.use_fcopyfile, .monotonic) == .disabled)
3534	return error.Unimplemented;
3535	if (file_reader.pos != 0) return error.Unimplemented;
3536	if (offset != 0) return error.Unimplemented;
3537	if (limit != .unlimited) return error.Unimplemented;
3538	const size = file_reader.getSize() catch return error.Unimplemented;
3539	if (header.len != 0 or reader_buffered.len != 0) {
3540	const n = try fileWritePositional(
3541	ev,
3542	file,
3543	header,
3544	&.{limit.slice(reader_buffered)},
3545	1,
3546	offset,
3547	);
3548	file_reader.interface.toss(n -\| header.len);
3549	return n;
3550	}
3551	while (true) {
3552	const rc = c.fcopyfile(in_fd, out_fd, null, .{ .DATA = true });
3553	switch (c.errno(rc)) {
3554	.SUCCESS => break,
3555	.INTR => {},
3556	.OPNOTSUPP => {
3557	// Give calling code chance to observe before trying
3558	// something else.
3559	@atomicStore(UseFcopyfile, &ev.use_fcopyfile, .disabled, .monotonic);
3560	return 0;
3561	},
3562	else => \|e\| {
3563	assert(error.Unexpected == switch (e) {
3564	.NOMEM => return error.SystemResources,
3565	.INVAL => \|err\| errnoBug(err),
3566	else => \|err\| unexpectedErrno(err),
3567	});
3568	return 0;
3569	},
3570	}
3571	}
3572	file_reader.pos = size;
3573	return size;
3574	}
3575
3576	fn fileReadPositional(
3577	userdata: ?*anyopaque,
3578	file: File,
3579	data: []const []u8,
3580	offset: u64,
3581	) File.ReadPositionalError!usize {
3582	const ev: *Evented = @ptrCast(@alignCast(userdata));
3583	_ = ev;
3584	var iovecs: [max_iovecs_len]iovec = undefined;
3585	var iovlen: iovlen_t = 0;
3586	var remaining: Io.Limit = .unlimited;
3587	for (data) \|buf\| addBuf(false, &iovecs, &iovlen, &remaining, buf);
3588	if (iovlen == 0) return 0;
3589	while (true) {
3590	const rc = c.preadv(file.handle, &iovecs, iovlen, @bitCast(offset));
3591	switch (c.errno(rc)) {
3592	.SUCCESS => return @intCast(rc),
3593	.INTR => {},
3594	.NXIO => return error.Unseekable,
3595	.SPIPE => return error.Unseekable,
3596	.OVERFLOW => return error.Unseekable,
3597	.NOBUFS => return error.SystemResources,
3598	.NOMEM => return error.SystemResources,
3599	.AGAIN => return error.WouldBlock,
3600	.IO => return error.InputOutput,
3601	.ISDIR => return error.IsDir,
3602	.NOTCONN => \|err\| return errnoBug(err), // not a socket
3603	.CONNRESET => \|err\| return errnoBug(err), // not a socket
3604	.INVAL => \|err\| return errnoBug(err),
3605	.FAULT => \|err\| return errnoBug(err),
3606	.BADF => return error.NotOpenForReading,
3607	else => \|err\| return unexpectedErrno(err),
3608	}
3609	}
3610	}
3611
3612	fn fileSeekBy(userdata: ?*anyopaque, file: File, offset: i64) File.SeekError!void {
3613	const ev: *Evented = @ptrCast(@alignCast(userdata));
3614	return ev.lseek(file.handle, @bitCast(offset), c.SEEK.CUR);
3615	}
3616
3617	fn fileSeekTo(userdata: ?*anyopaque, file: File, offset: u64) File.SeekError!void {
3618	const ev: *Evented = @ptrCast(@alignCast(userdata));
3619	return ev.lseek(file.handle, offset, c.SEEK.SET);
3620	}
3621
3622	fn lseek(ev: *Evented, fd: c.fd_t, offset: u64, whence: i32) File.SeekError!void {
3623	_ = ev;
3624	while (true) switch (c.errno(c.lseek(fd, @bitCast(offset), whence))) {
3625	.SUCCESS => return,
3626	.INTR => {},
3627	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
3628	.INVAL => return error.Unseekable,
3629	.OVERFLOW => return error.Unseekable,
3630	.SPIPE => return error.Unseekable,
3631	.NXIO => return error.Unseekable,
3632	else => \|err\| return unexpectedErrno(err),
3633	};
3634	}
3635
3636	fn fileSync(userdata: ?*anyopaque, file: File) File.SyncError!void {
3637	const ev: *Evented = @ptrCast(@alignCast(userdata));
3638	_ = ev;
3639	while (true) switch (c.errno(c.fsync(file.handle))) {
3640	.SUCCESS => return,
3641	.INTR => {},
3642	.BADF => \|err\| return errnoBug(err),
3643	.INVAL => \|err\| return errnoBug(err),
3644	.ROFS => \|err\| return errnoBug(err),
3645	.IO => return error.InputOutput,
3646	.NOSPC => return error.NoSpaceLeft,
3647	.DQUOT => return error.DiskQuota,
3648	else => \|err\| return unexpectedErrno(err),
3649	};
3650	}
3651
3652	fn fileIsTty(userdata: ?*anyopaque, file: File) Io.Cancelable!bool {
3653	const ev: *Evented = @ptrCast(@alignCast(userdata));
3654	_ = ev;
3655	while (true) {
3656	const rc = c.isatty(file.handle);
3657	switch (c.errno(rc - 1)) {
3658	.SUCCESS => return true,
3659	.INTR => {},
3660	else => return false,
3661	}
3662	}
3663	}
3664
3665	fn fileEnableAnsiEscapeCodes(userdata: ?*anyopaque, file: File) File.EnableAnsiEscapeCodesError!void {
3666	const ev: *Evented = @ptrCast(@alignCast(userdata));
3667	if (!try fileIsTty(ev, file)) return error.NotTerminalDevice;
3668	}
3669
3670	fn fileSetLength(userdata: ?*anyopaque, file: File, length: u64) File.SetLengthError!void {
3671	const ev: *Evented = @ptrCast(@alignCast(userdata));
3672	_ = ev;
3673
3674	const signed_len: i64 = @bitCast(length);
3675	if (signed_len < 0) return error.FileTooBig; // Avoid ambiguous EINVAL errors.
3676
3677	while (true) switch (c.errno(c.ftruncate(file.handle, signed_len))) {
3678	.SUCCESS => return,
3679	.INTR => {},
3680	.FBIG => return error.FileTooBig,
3681	.IO => return error.InputOutput,
3682	.PERM => return error.PermissionDenied,
3683	.TXTBSY => return error.FileBusy,
3684	.BADF => \|err\| return errnoBug(err), // Handle not open for writing.
3685	.INVAL => return error.NonResizable, // This is returned for /dev/null for example.
3686	else => \|err\| return unexpectedErrno(err),
3687	};
3688	}
3689
3690	fn fileSetOwner(
3691	userdata: ?*anyopaque,
3692	file: File,
3693	owner: ?File.Uid,
3694	group: ?File.Gid,
3695	) File.SetOwnerError!void {
3696	const ev: *Evented = @ptrCast(@alignCast(userdata));
3697	_ = ev;
3698	return fchown(file.handle, owner, group);
3699	}
3700
3701	fn fileSetPermissions(
3702	userdata: ?*anyopaque,
3703	file: File,
3704	permissions: File.Permissions,
3705	) File.SetPermissionsError!void {
3706	const ev: *Evented = @ptrCast(@alignCast(userdata));
3707	return ev.fchmod(file.handle, permissions.toMode());
3708	}
3709
3710	fn fchmod(ev: *Evented, fd: c.fd_t, mode: c.mode_t) File.SetPermissionsError!void {
3711	_ = ev;
3712	while (true) switch (c.errno(c.fchmod(fd, mode))) {
3713	.SUCCESS => return,
3714	.INTR => {},
3715	.BADF => \|err\| return errnoBug(err),
3716	.FAULT => \|err\| return errnoBug(err),
3717	.INVAL => \|err\| return errnoBug(err),
3718	.ACCES => return error.AccessDenied,
3719	.IO => return error.InputOutput,
3720	.LOOP => return error.SymLinkLoop,
3721	.NOENT => return error.FileNotFound,
3722	.NOMEM => return error.SystemResources,
3723	.NOTDIR => return error.FileNotFound,
3724	.PERM => return error.PermissionDenied,
3725	.ROFS => return error.ReadOnlyFileSystem,
3726	else => \|err\| return unexpectedErrno(err),
3727	};
3728	}
3729
3730	fn fileSetTimestamps(
3731	userdata: ?*anyopaque,
3732	file: File,
3733	options: File.SetTimestampsOptions,
3734	) File.SetTimestampsError!void {
3735	const ev: *Evented = @ptrCast(@alignCast(userdata));
3736	_ = ev;
3737
3738	var times_buffer: [2]c.timespec = undefined;
3739	const times = if (options.modify_timestamp == .now and options.access_timestamp == .now) null else p: {
3740	times_buffer = .{
3741	setTimestampToPosix(options.access_timestamp),
3742	setTimestampToPosix(options.modify_timestamp),
3743	};
3744	break :p &times_buffer;
3745	};
3746
3747	while (true) switch (c.errno(c.futimens(file.handle, times))) {
3748	.SUCCESS => return,
3749	.INTR => {},
3750	.BADF => \|err\| return errnoBug(err), // always a race condition
3751	.FAULT => \|err\| return errnoBug(err),
3752	.INVAL => \|err\| return errnoBug(err),
3753	.ACCES => return error.AccessDenied,
3754	.PERM => return error.PermissionDenied,
3755	.ROFS => return error.ReadOnlyFileSystem,
3756	else => \|err\| return unexpectedErrno(err),
3757	};
3758	}
3759
3760	fn fileLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!void {
3761	const ev: *Evented = @ptrCast(@alignCast(userdata));
3762	_ = ev;
3763	const operation: i32 = switch (lock) {
3764	.none => c.LOCK.UN,
3765	.shared => c.LOCK.SH,
3766	.exclusive => c.LOCK.EX,
3767	};
3768	while (true) switch (c.errno(c.flock(file.handle, operation))) {
3769	.SUCCESS => return,
3770	.INTR => {},
3771	.BADF => \|err\| return errnoBug(err),
3772	.INVAL => \|err\| return errnoBug(err), // invalid parameters
3773	.NOLCK => return error.SystemResources,
3774	.AGAIN => \|err\| return errnoBug(err),
3775	.OPNOTSUPP => return error.FileLocksUnsupported,
3776	else => \|err\| return unexpectedErrno(err),
3777	};
3778	}
3779
3780	fn fileTryLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!bool {
3781	const ev: *Evented = @ptrCast(@alignCast(userdata));
3782	_ = ev;
3783	const operation: i32 = switch (lock) {
3784	.none => c.LOCK.UN,
3785	.shared => c.LOCK.SH \| c.LOCK.NB,
3786	.exclusive => c.LOCK.EX \| c.LOCK.NB,
3787	};
3788	while (true) switch (c.errno(c.flock(file.handle, operation))) {
3789	.SUCCESS => return true,
3790	.INTR => {},
3791	.AGAIN => return false,
3792	.BADF => \|err\| return errnoBug(err),
3793	.INVAL => \|err\| return errnoBug(err), // invalid parameters
3794	.NOLCK => return error.SystemResources,
3795	.OPNOTSUPP => return error.FileLocksUnsupported,
3796	else => \|err\| return unexpectedErrno(err),
3797	};
3798	}
3799
3800	fn fileUnlock(userdata: ?*anyopaque, file: File) void {
3801	const ev: *Evented = @ptrCast(@alignCast(userdata));
3802	_ = ev;
3803	while (true) switch (c.errno(c.flock(file.handle, c.LOCK.UN))) {
3804	.SUCCESS => return,
3805	.INTR => {},
3806	.AGAIN => return recoverableOsBugDetected(), // unlocking can't block
3807	.BADF => return recoverableOsBugDetected(), // File descriptor used after closed.
3808	.INVAL => return recoverableOsBugDetected(), // invalid parameters
3809	.NOLCK => return recoverableOsBugDetected(), // Resource deallocation.
3810	.OPNOTSUPP => return recoverableOsBugDetected(), // We already got the lock.
3811	else => return recoverableOsBugDetected(), // Resource deallocation must succeed.
3812	};
3813	}
3814
3815	fn fileDowngradeLock(userdata: ?*anyopaque, file: File) File.DowngradeLockError!void {
3816	const ev: *Evented = @ptrCast(@alignCast(userdata));
3817	_ = ev;
3818	const operation = c.LOCK.SH \| c.LOCK.NB;
3819	while (true) switch (c.errno(c.flock(file.handle, operation))) {
3820	.SUCCESS => return,
3821	.INTR => {},
3822	.AGAIN => \|err\| return errnoBug(err), // File was not locked in exclusive mode.
3823	.BADF => \|err\| return errnoBug(err),
3824	.INVAL => \|err\| return errnoBug(err), // invalid parameters
3825	.NOLCK => \|err\| return errnoBug(err), // Lock already obtained.
3826	.OPNOTSUPP => \|err\| return errnoBug(err), // Lock already obtained.
3827	else => \|err\| return unexpectedErrno(err),
3828	};
3829	}
3830
3831	fn fileRealPath(userdata: ?*anyopaque, file: File, out_buffer: []u8) File.RealPathError!usize {
3832	const ev: *Evented = @ptrCast(@alignCast(userdata));
3833	_ = ev;
3834	var buffer: [c.PATH_MAX]u8 = undefined;
3835	@memset(&buffer, 0);
3836	while (true) {
3837	switch (c.errno(c.fcntl(file.handle, c.F.GETPATH, &buffer))) {
3838	.SUCCESS => break,
3839	.INTR => {},
3840	.ACCES => return error.AccessDenied,
3841	.BADF => return error.FileNotFound,
3842	.NOENT => return error.FileNotFound,
3843	.NOMEM => return error.SystemResources,
3844	.NOSPC => return error.NameTooLong,
3845	.RANGE => return error.NameTooLong,
3846	else => \|err\| return unexpectedErrno(err),
3847	}
3848	}
3849	const n = std.mem.indexOfScalar(u8, &buffer, 0) orelse buffer.len;
3850	if (n > out_buffer.len) return error.NameTooLong;
3851	@memcpy(out_buffer[0..n], buffer[0..n]);
3852	return n;
3853	}
3854
3855	fn fileHardLink(
3856	userdata: ?*anyopaque,
3857	file: File,
3858	new_dir: Dir,
3859	new_sub_path: []const u8,
3860	options: File.HardLinkOptions,
3861	) File.HardLinkError!void {
3862	const ev: *Evented = @ptrCast(@alignCast(userdata));
3863	_ = ev;
3864	_ = file;
3865	_ = new_dir;
3866	_ = new_sub_path;
3867	_ = options;
3868	return error.OperationUnsupported;
3869	}
3870
3871	fn linkat(
3872	old_dir: c.fd_t,
3873	old_path: [*:0]const u8,
3874	new_dir: c.fd_t,
3875	new_path: [*:0]const u8,
3876	flags: u32,
3877	) File.HardLinkError!void {
3878	while (true) switch (c.errno(c.linkat(old_dir, old_path, new_dir, new_path, flags))) {
3879	.SUCCESS => return,
3880	.INTR => {},
3881	.ACCES => return error.AccessDenied,
3882	.DQUOT => return error.DiskQuota,
3883	.EXIST => return error.PathAlreadyExists,
3884	.IO => return error.HardwareFailure,
3885	.LOOP => return error.SymLinkLoop,
3886	.MLINK => return error.LinkQuotaExceeded,
3887	.NAMETOOLONG => return error.NameTooLong,
3888	.NOENT => return error.FileNotFound,
3889	.NOMEM => return error.SystemResources,
3890	.NOSPC => return error.NoSpaceLeft,
3891	.NOTDIR => return error.NotDir,
3892	.PERM => return error.PermissionDenied,
3893	.ROFS => return error.ReadOnlyFileSystem,
3894	.XDEV => return error.CrossDevice,
3895	.ILSEQ => return error.BadPathName,
3896	.FAULT => \|err\| return errnoBug(err),
3897	.INVAL => \|err\| return errnoBug(err),
3898	else => \|err\| return unexpectedErrno(err),
3899	};
3900	}
3901
3902	fn fileMemoryMapCreate(
3903	userdata: ?*anyopaque,
3904	file: File,
3905	options: File.MemoryMap.CreateOptions,
3906	) File.MemoryMap.CreateError!File.MemoryMap {
3907	const ev: *Evented = @ptrCast(@alignCast(userdata));
3908	_ = ev;
3909
3910	const prot: c.PROT = .{
3911	.READ = options.protection.read,
3912	.WRITE = options.protection.write,
3913	.EXEC = options.protection.execute,
3914	};
3915	const flags: c.MAP = .{
3916	.TYPE = .SHARED,
3917	};
3918
3919	const page_align = std.heap.page_size_min;
3920
3921	const contents = while (true) {
3922	const casted_offset = std.math.cast(i64, options.offset) orelse return error.Unseekable;
3923	const rc = c.mmap(null, options.len, prot, flags, file.handle, casted_offset);
3924	const err: c.E = if (rc != c.MAP_FAILED) .SUCCESS else @enumFromInt(c._errno().*);
3925	switch (err) {
3926	.SUCCESS => break @as([*]align(page_align) u8, @ptrCast(@alignCast(rc)))[0..options.len],
3927	.INTR => {},
3928	.ACCES => return error.AccessDenied,
3929	.AGAIN => return error.LockedMemoryLimitExceeded,
3930	.MFILE => return error.ProcessFdQuotaExceeded,
3931	.NFILE => return error.SystemFdQuotaExceeded,
3932	.NOMEM => return error.OutOfMemory,
3933	.PERM => return error.PermissionDenied,
3934	.OVERFLOW => return error.Unseekable,
3935	.BADF => return errnoBug(err), // Always a race condition.
3936	.INVAL => return errnoBug(err), // Invalid parameters to mmap()
3937	else => return unexpectedErrno(err),
3938	}
3939	};
3940	return .{
3941	.file = file,
3942	.offset = options.offset,
3943	.memory = contents,
3944	.section = {},
3945	};
3946	}
3947
3948	fn fileMemoryMapDestroy(userdata: ?anyopaque, mm: File.MemoryMap) void {
3949	const ev: *Evented = @ptrCast(@alignCast(userdata));
3950	_ = ev;
3951	const memory = mm.memory;
3952	if (memory.len == 0) return;
3953	switch (c.errno(c.munmap(memory.ptr, memory.len))) {
3954	.SUCCESS => {},
3955	else => \|err\| if (builtin.mode == .Debug)
3956	std.log.err("failed to unmap {d} bytes at {*}: {t}", .{ memory.len, memory.ptr, err }),
3957	}
3958	mm.* = undefined;
3959	}
3960
3961	fn fileMemoryMapSetLength(
3962	userdata: ?*anyopaque,
3963	mm: *File.MemoryMap,
3964	new_len: usize,
3965	) File.MemoryMap.SetLengthError!void {
3966	const ev: *Evented = @ptrCast(@alignCast(userdata));
3967	_ = ev;
3968
3969	const page_size = std.heap.pageSize();
3970	const alignment: Alignment = .fromByteUnits(page_size);
3971	const old_memory = mm.memory;
3972
3973	if (alignment.forward(new_len) == alignment.forward(old_memory.len)) {
3974	mm.memory.len = new_len;
3975	return;
3976	}
3977	return error.OperationUnsupported;
3978	}
3979
3980	fn fileMemoryMapRead(userdata: ?anyopaque, mm: File.MemoryMap) File.ReadPositionalError!void {
3981	const ev: *Evented = @ptrCast(@alignCast(userdata));
3982	_ = ev;
3983	_ = mm;
3984	}
3985
3986	fn fileMemoryMapWrite(userdata: ?anyopaque, mm: File.MemoryMap) File.WritePositionalError!void {
3987	const ev: *Evented = @ptrCast(@alignCast(userdata));
3988	_ = ev;
3989	_ = mm;
3990	}
3991
3992	fn processExecutableOpen(
3993	userdata: ?*anyopaque,
3994	flags: File.OpenFlags,
3995	) process.OpenExecutableError!File {
3996	const ev: *Evented = @ptrCast(@alignCast(userdata));
3997	// _NSGetExecutablePath() returns a path that might be a symlink to
3998	// the executable. Here it does not matter since we open it.
3999	var symlink_path_buf: [c.PATH_MAX + 1]u8 = undefined;
4000	var n: u32 = symlink_path_buf.len;
4001	const rc = c._NSGetExecutablePath(&symlink_path_buf, &n);
4002	if (rc != 0) return error.NameTooLong;
4003	const symlink_path = std.mem.sliceTo(&symlink_path_buf, 0);
4004	return dirOpenFile(ev, .cwd(), symlink_path, flags);
4005	}
4006
4007	fn processExecutablePath(userdata: ?*anyopaque, out_buffer: []u8) process.ExecutablePathError!usize {
4008	const ev: *Evented = @ptrCast(@alignCast(userdata));
4009	// _NSGetExecutablePath() returns a path that might be a symlink to
4010	// the executable.
4011	var symlink_path_buf: [c.PATH_MAX + 1]u8 = undefined;
4012	var n: u32 = symlink_path_buf.len;
4013	const rc = c._NSGetExecutablePath(&symlink_path_buf, &n);
4014	if (rc != 0) return error.NameTooLong;
4015	const symlink_path = std.mem.sliceTo(&symlink_path_buf, 0);
4016	assert(Dir.path.isAbsolute(symlink_path));
4017	return dirRealPathFile(ev, .cwd(), symlink_path, out_buffer) catch \|err\| switch (err) {
4018	error.NetworkNotFound => unreachable, // Windows-only
4019	error.FileBusy => unreachable, // Windows-only
4020	else => \|e\| return e,
4021	};
4022	}
4023
4024	fn lockStderr(userdata: ?*anyopaque, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
4025	const ev: *Evented = @ptrCast(@alignCast(userdata));
4026	try ev.stderr_mutex.lock(ev);
4027	errdefer ev.stderr_mutex.unlock();
4028	return ev.initLockedStderr(terminal_mode);
4029	}
4030
4031	fn tryLockStderr(
4032	userdata: ?*anyopaque,
4033	terminal_mode: ?Io.Terminal.Mode,
4034	) Io.Cancelable!?Io.LockedStderr {
4035	const ev: *Evented = @ptrCast(@alignCast(userdata));
4036	if (!ev.stderr_mutex.tryLock()) return null;
4037	errdefer ev.stderr_mutex.unlock();
4038	return try ev.initLockedStderr(terminal_mode);
4039	}
4040
4041	fn initLockedStderr(ev: *Evented, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
4042	ev.init_stderr_writer.once(ev, &initStderrWriter);
4043	return .{
4044	.file_writer = &ev.stderr_writer,
4045	.terminal_mode = terminal_mode orelse ev.stderr_mode,
4046	};
4047	}
4048
4049	fn initStderrWriter(context: ?*anyopaque) callconv(.c) void {
4050	const ev: *Evented = @ptrCast(@alignCast(context));
4051	const cancel_protection = swapCancelProtection(ev, .blocked);
4052	defer assert(swapCancelProtection(ev, cancel_protection) == .blocked);
4053	ev.scan_environ.once(ev, &scanEnviron);
4054	const NO_COLOR = ev.environ.exist.NO_COLOR;
4055	const CLICOLOR_FORCE = ev.environ.exist.CLICOLOR_FORCE;
4056	ev.stderr_mode = Io.Terminal.Mode.detect(
4057	ev.io(),
4058	ev.stderr_writer.file,
4059	NO_COLOR,
4060	CLICOLOR_FORCE,
4061	) catch \|err\| switch (err) {
4062	error.Canceled => unreachable, // blocked
4063	};
4064	}
4065
4066	fn unlockStderr(userdata: ?*anyopaque) void {
4067	const ev: *Evented = @ptrCast(@alignCast(userdata));
4068	if (ev.stderr_writer.err == null) ev.stderr_writer.interface.flush() catch {};
4069	if (ev.stderr_writer.err) \|err\| {
4070	switch (err) {
4071	error.Canceled => Thread.current().currentFiber().cancel_protection.recancel(),
4072	else => {},
4073	}
4074	ev.stderr_writer.err = null;
4075	}
4076	ev.stderr_writer.interface.end = 0;
4077	ev.stderr_writer.interface.buffer.len = 0;
4078	ev.stderr_mutex.unlock();
4079	}
4080
4081	fn processCurrentPath(userdata: ?*anyopaque, buffer: []u8) process.CurrentPathError!usize {
4082	const ev: *Evented = @ptrCast(@alignCast(userdata));
4083	_ = ev;
4084	const err: c.E = if (c.getcwd(buffer.ptr, buffer.len)) \|_\| .SUCCESS else @enumFromInt(c._errno().*);
4085	switch (err) {
4086	.SUCCESS => return std.mem.findScalar(u8, buffer, 0).?,
4087	.NOENT => return error.CurrentDirUnlinked,
4088	.RANGE => return error.NameTooLong,
4089	.FAULT => \|e\| return errnoBug(e),
4090	.INVAL => \|e\| return errnoBug(e),
4091	else => return unexpectedErrno(err),
4092	}
4093	}
4094
4095	fn processSetCurrentDir(userdata: ?*anyopaque, dir: Dir) process.SetCurrentDirError!void {
4096	const ev: *Evented = @ptrCast(@alignCast(userdata));
4097	_ = ev;
4098	if (dir.handle == c.AT.FDCWD) return;
4099	while (true) switch (c.errno(c.fchdir(dir.handle))) {
4100	.SUCCESS => return,
4101	.INTR => {},
4102	.ACCES => return error.AccessDenied,
4103	.NOTDIR => return error.NotDir,
4104	.IO => return error.FileSystem,
4105	.BADF => \|err\| return errnoBug(err),
4106	else => \|err\| return unexpectedErrno(err),
4107	};
4108	}
4109
4110	fn processSetCurrentPath(userdata: ?*anyopaque, dir_path: []const u8) ChdirError!void {
4111	const ev: *Evented = @ptrCast(@alignCast(userdata));
4112	_ = ev;
4113	var path_buffer: [c.PATH_MAX]u8 = undefined;
4114	const dir_path_posix = try pathToPosix(dir_path, &path_buffer);
4115	while (true) switch (c.errno(c.chdir(dir_path_posix))) {
4116	.SUCCESS => return,
4117	.INTR => {},
4118	.ACCES => return error.AccessDenied,
4119	.IO => return error.FileSystem,
4120	.LOOP => return error.SymLinkLoop,
4121	.NAMETOOLONG => return error.NameTooLong,
4122	.NOENT => return error.FileNotFound,
4123	.NOMEM => return error.SystemResources,
4124	.NOTDIR => return error.NotDir,
4125	.ILSEQ => return error.BadPathName,
4126	.FAULT => \|err\| return errnoBug(err),
4127	else => \|err\| return unexpectedErrno(err),
4128	};
4129	}
4130
4131	fn processReplace(userdata: ?*anyopaque, options: process.ReplaceOptions) process.ReplaceError {
4132	const ev: *Evented = @ptrCast(@alignCast(userdata));
4133
4134	if (!process.can_replace) return error.OperationUnsupported;
4135
4136	ev.scan_environ.once(ev, &scanEnviron); // for PATH
4137	const PATH = ev.environ.string.PATH orelse default_PATH;
4138
4139	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
4140	defer arena_allocator.deinit();
4141	const arena = arena_allocator.allocator();
4142
4143	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
4144	for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
4145
4146	const env_block = env_block: {
4147	const prog_fd: i32 = -1;
4148	if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
4149	.zig_progress_fd = prog_fd,
4150	});
4151	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
4152	.zig_progress_fd = prog_fd,
4153	});
4154	};
4155
4156	return ev.execv(options.expand_arg0, argv_buf.ptr[0].?, argv_buf.ptr, env_block, PATH);
4157	}
4158
4159	fn processReplacePath(
4160	userdata: ?*anyopaque,
4161	dir: Dir,
4162	options: process.ReplaceOptions,
4163	) process.ReplaceError {
4164	const ev: *Evented = @ptrCast(@alignCast(userdata));
4165	_ = ev;
4166	_ = dir;
4167	_ = options;
4168	@panic("TODO processReplacePath");
4169	}
4170
4171	fn processSpawn(userdata: ?*anyopaque, options: process.SpawnOptions) process.SpawnError!process.Child {
4172	const ev: *Evented = @ptrCast(@alignCast(userdata));
4173	const spawned = try ev.spawn(options);
4174	defer fileClose(ev, &.{spawned.err_pipe});
4175
4176	// Wait for the child to report any errors in or before `execvpe`.
4177	var child_err: ForkBailError = undefined;
4178	ev.readAll(spawned.err_pipe, @ptrCast(&child_err)) catch \|read_err\| {
4179	switch (read_err) {
4180	error.Canceled => unreachable, // blocked
4181	error.EndOfStream => {
4182	// Write end closed by CLOEXEC at the time of the `execvpe` call,
4183	// indicating success.
4184	},
4185	else => {
4186	// Problem reading the error from the error reporting pipe. We
4187	// don't know if the child is alive or dead. Better to assume it is
4188	// alive so the resource does not risk being leaked.
4189	},
4190	}
4191	return .{
4192	.id = spawned.pid,
4193	.thread_handle = {},
4194	.stdin = spawned.stdin,
4195	.stdout = spawned.stdout,
4196	.stderr = spawned.stderr,
4197	.request_resource_usage_statistics = options.request_resource_usage_statistics,
4198	};
4199	};
4200	return child_err;
4201	}
4202
4203	fn processSpawnPath(
4204	userdata: ?*anyopaque,
4205	dir: Dir,
4206	options: process.SpawnOptions,
4207	) process.SpawnError!process.Child {
4208	const ev: *Evented = @ptrCast(@alignCast(userdata));
4209	_ = ev;
4210	_ = dir;
4211	_ = options;
4212	@panic("TODO processSpawnPath");
4213	}
4214
4215	const prog_fileno = @max(c.STDIN_FILENO, c.STDOUT_FILENO, c.STDERR_FILENO) + 1;
4216
4217	const Spawned = struct {
4218	pid: c.pid_t,
4219	err_pipe: File,
4220	stdin: ?File,
4221	stdout: ?File,
4222	stderr: ?File,
4223	};
4224	fn spawn(ev: *Evented, options: process.SpawnOptions) process.SpawnError!Spawned {
4225	// The child process does need to access (one end of) these pipes. However,
4226	// we must initially set CLOEXEC to avoid a race condition. If another thread
4227	// is racing to spawn a different child process, we don't want it to inherit
4228	// these FDs in any scenario; that would mean that, for instance, calls to
4229	// `poll` from the parent would not report the child's stdout as closing when
4230	// expected, since the other child may retain a reference to the write end of
4231	// the pipe. So, we create the pipes with CLOEXEC initially. After fork, we
4232	// need to do something in the new child to make sure we preserve the reference
4233	// we want. We could use `fcntl` to remove CLOEXEC from the FD, but as it
4234	// turns out, we `dup2` everything anyway, so there's no need!
4235	const pipe_flags: c.O = .{ .CLOEXEC = true };
4236
4237	const stdin_pipe = if (options.stdin == .pipe) try pipe2(pipe_flags) else undefined;
4238	errdefer if (options.stdin == .pipe) {
4239	destroyPipe(stdin_pipe);
4240	};
4241
4242	const stdout_pipe = if (options.stdout == .pipe) try pipe2(pipe_flags) else undefined;
4243	errdefer if (options.stdout == .pipe) {
4244	destroyPipe(stdout_pipe);
4245	};
4246
4247	const stderr_pipe = if (options.stderr == .pipe) try pipe2(pipe_flags) else undefined;
4248	errdefer if (options.stderr == .pipe) {
4249	destroyPipe(stderr_pipe);
4250	};
4251
4252	const any_ignore =
4253	options.stdin == .ignore or options.stdout == .ignore or options.stderr == .ignore;
4254	const dev_null_file = if (any_ignore) dev_null_file: {
4255	ev.open_dev_null.once(ev, &openDevNullFile);
4256	break :dev_null_file try ev.dev_null_file;
4257	} else undefined;
4258
4259	const prog_pipe: [2]c.fd_t = if (options.progress_node.index != .none)
4260	// We use CLOEXEC for the same reason as in `pipe_flags`.
4261	try pipe2(.{ .NONBLOCK = true, .CLOEXEC = true })
4262	else
4263	.{ -1, -1 };
4264	errdefer destroyPipe(prog_pipe);
4265
4266	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
4267	defer arena_allocator.deinit();
4268	const arena = arena_allocator.allocator();
4269
4270	// The POSIX standard does not allow malloc() between fork() and execve(),
4271	// and this allocator may be a libc allocator.
4272	// I have personally observed the child process deadlocking when it tries
4273	// to call malloc() due to a heap allocation between fork() and execve(),
4274	// in musl v1.1.24.
4275	// Additionally, we want to reduce the number of possible ways things
4276	// can fail between fork() and execve().
4277	// Therefore, we do all the allocation for the execve() before the fork().
4278	// This means we must do the null-termination of argv and env vars here.
4279	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
4280	for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
4281
4282	const env_block = env_block: {
4283	const prog_fd: i32 = if (prog_pipe[1] == -1) -1 else prog_fileno;
4284	if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
4285	.zig_progress_fd = prog_fd,
4286	});
4287	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
4288	.zig_progress_fd = prog_fd,
4289	});
4290	};
4291
4292	// This pipe communicates to the parent errors in the child between `fork` and `execvpe`.
4293	// It is closed by the child (via CLOEXEC) without writing if `execvpe` succeeds.
4294	const err_pipe: [2]File = err_pipe: {
4295	const err_pipe = try pipe2(.{ .CLOEXEC = true });
4296	break :err_pipe .{
4297	.{ .handle = err_pipe[0], .flags = .{ .nonblocking = false } },
4298	.{ .handle = err_pipe[1], .flags = .{ .nonblocking = false } },
4299	};
4300	};
4301	errdefer fileClose(ev, &err_pipe);
4302
4303	ev.scan_environ.once(ev, &scanEnviron); // for PATH
4304	const PATH = ev.environ.string.PATH orelse default_PATH;
4305
4306	const pid_result: c.pid_t = fork: {
4307	const rc = c.fork();
4308	switch (c.errno(rc)) {
4309	.SUCCESS => break :fork @intCast(rc),
4310	.AGAIN => return error.SystemResources,
4311	.NOMEM => return error.SystemResources,
4312	.NOSYS => return error.OperationUnsupported,
4313	else => \|err\| return unexpectedErrno(err),
4314	}
4315	};
4316
4317	if (pid_result == 0) {
4318	defer comptime unreachable; // We are the child.
4319	const err = ev.setUpChild(.{
4320	.stdin_pipe = stdin_pipe[0],
4321	.stdout_pipe = stdout_pipe[1],
4322	.stderr_pipe = stderr_pipe[1],
4323	.dev_null_fd = dev_null_file.handle,
4324	.prog_pipe = prog_pipe[1],
4325	.argv_buf = argv_buf,
4326	.env_block = env_block,
4327	.PATH = PATH,
4328	.spawn = options,
4329	});
4330	ev.writeAll(err_pipe[1], @ptrCast(&err)) catch {};
4331	c.exit(1);
4332	}
4333
4334	const pid: c.pid_t = @intCast(pid_result); // We are the parent.
4335	errdefer comptime unreachable; // The child is forked; we must not error from now on
4336
4337	fileClose(ev, err_pipe[1..2]); // make sure only the child holds the write end open
4338
4339	if (options.stdin == .pipe) closeFd(stdin_pipe[0]);
4340	if (options.stdout == .pipe) closeFd(stdout_pipe[1]);
4341	if (options.stderr == .pipe) closeFd(stderr_pipe[1]);
4342
4343	if (prog_pipe[1] != -1) closeFd(prog_pipe[1]);
4344
4345	options.progress_node.setIpcFile(ev, .{ .handle = prog_pipe[0], .flags = .{ .nonblocking = true } });
4346
4347	return .{
4348	.pid = pid,
4349	.err_pipe = err_pipe[0],
4350	.stdin = switch (options.stdin) {
4351	.pipe => .{ .handle = stdin_pipe[1], .flags = .{ .nonblocking = false } },
4352	else => null,
4353	},
4354	.stdout = switch (options.stdout) {
4355	.pipe => .{ .handle = stdout_pipe[0], .flags = .{ .nonblocking = false } },
4356	else => null,
4357	},
4358	.stderr = switch (options.stderr) {
4359	.pipe => .{ .handle = stderr_pipe[0], .flags = .{ .nonblocking = false } },
4360	else => null,
4361	},
4362	};
4363	}
4364
4365	fn openDevNullFile(context: ?*anyopaque) callconv(.c) void {
4366	const ev: *Evented = @ptrCast(@alignCast(context));
4367	ev.dev_null_file = dirOpenFile(ev, .cwd(), "/dev/null", .{ .mode = .read_write });
4368	}
4369
4370	/// Errors that can occur between fork() and execv()
4371	const ForkBailError = process.SetCurrentDirError \|\| ChdirError \|\|
4372	process.SpawnError \|\| process.ReplaceError;
4373	fn setUpChild(ev: *Evented, options: struct {
4374	stdin_pipe: c.fd_t,
4375	stdout_pipe: c.fd_t,
4376	stderr_pipe: c.fd_t,
4377	dev_null_fd: c.fd_t,
4378	prog_pipe: c.fd_t,
4379	argv_buf: [:null]?[*:0]const u8,
4380	env_block: process.Environ.Block,
4381	PATH: []const u8,
4382	spawn: process.SpawnOptions,
4383	}) ForkBailError {
4384	try ev.setUpChildIo(
4385	options.spawn.stdin,
4386	options.stdin_pipe,
4387	c.STDIN_FILENO,
4388	options.dev_null_fd,
4389	);
4390	try ev.setUpChildIo(
4391	options.spawn.stdout,
4392	options.stdout_pipe,
4393	c.STDOUT_FILENO,
4394	options.dev_null_fd,
4395	);
4396	try ev.setUpChildIo(
4397	options.spawn.stderr,
4398	options.stderr_pipe,
4399	c.STDERR_FILENO,
4400	options.dev_null_fd,
4401	);
4402
4403	switch (options.spawn.cwd) {
4404	.inherit => {},
4405	.dir => \|cwd_dir\| try processSetCurrentDir(ev, cwd_dir),
4406	.path => \|cwd_path\| try processSetCurrentPath(ev, cwd_path),
4407	}
4408
4409	// Must happen after fchdir above, the cwd file descriptor might be
4410	// equal to prog_fileno and be clobbered by this dup2 call.
4411	if (options.prog_pipe != -1) try ev.dup2(options.prog_pipe, prog_fileno);
4412
4413	if (options.spawn.gid) \|gid\| while (true) switch (c.errno(c.setregid(gid, gid))) {
4414	.SUCCESS => break,
4415	.INTR => {},
4416	.AGAIN => return error.ResourceLimitReached,
4417	.INVAL => return error.InvalidUserId,
4418	.PERM => return error.PermissionDenied,
4419	else => return error.Unexpected,
4420	};
4421
4422	if (options.spawn.uid) \|uid\| while (true) switch (c.errno(c.setreuid(uid, uid))) {
4423	.SUCCESS => break,
4424	.INTR => {},
4425	.AGAIN => return error.ResourceLimitReached,
4426	.INVAL => return error.InvalidUserId,
4427	.PERM => return error.PermissionDenied,
4428	else => return error.Unexpected,
4429	};
4430
4431	if (options.spawn.pgid) \|pid\| while (true) switch (c.errno(c.setpgid(0, pid))) {
4432	.SUCCESS => break,
4433	.INTR => {},
4434	.ACCES => return error.ProcessAlreadyExec,
4435	.INVAL => return error.InvalidProcessGroupId,
4436	.PERM => return error.PermissionDenied,
4437	else => return error.Unexpected,
4438	};
4439
4440	if (options.spawn.start_suspended) while (true) switch (c.errno(c.kill(0, .STOP))) {
4441	.SUCCESS => break,
4442	.INTR => {},
4443	.PERM => return error.PermissionDenied,
4444	else => return error.Unexpected,
4445	};
4446
4447	return ev.execv(
4448	options.spawn.expand_arg0,
4449	options.argv_buf.ptr[0].?,
4450	options.argv_buf.ptr,
4451	options.env_block,
4452	options.PATH,
4453	);
4454	}
4455
4456	fn setUpChildIo(
4457	ev: *Evented,
4458	stdio: process.SpawnOptions.StdIo,
4459	pipe_fd: c.fd_t,
4460	std_fileno: i32,
4461	dev_null_fd: c.fd_t,
4462	) !void {
4463	switch (stdio) {
4464	.pipe => try ev.dup2(pipe_fd, std_fileno),
4465	.close => closeFd(std_fileno),
4466	.inherit => {},
4467	.ignore => try ev.dup2(dev_null_fd, std_fileno),
4468	.file => \|file\| {
4469	if (file.flags.nonblocking) @panic("TODO implement setUpChildIo when nonblocking file is used");
4470	try ev.dup2(file.handle, std_fileno);
4471	},
4472	}
4473	}
4474
4475	const PipeError = error{
4476	SystemFdQuotaExceeded,
4477	ProcessFdQuotaExceeded,
4478	} \|\| Io.UnexpectedError;
4479
4480	fn pipe2(flags: c.O) PipeError![2]c.fd_t {
4481	var fds: [2]c.fd_t = undefined;
4482
4483	while (true) switch (c.errno(c.pipe(&fds))) {
4484	.SUCCESS => break,
4485	.INTR => {},
4486	.NFILE => return error.SystemFdQuotaExceeded,
4487	.MFILE => return error.ProcessFdQuotaExceeded,
4488	else => \|err\| return unexpectedErrno(err),
4489	};
4490	errdefer {
4491	closeFd(fds[0]);
4492	closeFd(fds[1]);
4493	}
4494
4495	// https://github.com/ziglang/zig/issues/18882
4496	if (@as(u32, @bitCast(flags)) == 0) return fds;
4497
4498	// CLOEXEC is special, it's a file descriptor flag and must be set using
4499	// F.SETFD.
4500	if (flags.CLOEXEC) for (fds) \|fd\| while (true) switch (c.errno(c.fcntl(fd, c.F.SETFD, @as(u32, c.FD_CLOEXEC)))) {
4501	.SUCCESS => break,
4502	.INTR => {},
4503	else => \|err\| return unexpectedErrno(err),
4504	};
4505
4506	const new_flags: u32 = f: {
4507	var new_flags = flags;
4508	new_flags.CLOEXEC = false;
4509	break :f @bitCast(new_flags);
4510	};
4511
4512	// Set every other flag affecting the file status using F.SETFL.
4513	if (new_flags != 0) for (fds) \|fd\| while (true) switch (c.errno(c.fcntl(fd, c.F.SETFL, new_flags))) {
4514	.SUCCESS => break,
4515	.INTR => {},
4516	.INVAL => \|err\| return errnoBug(err),
4517	else => \|err\| return unexpectedErrno(err),
4518	};
4519
4520	return fds;
4521	}
4522
4523	fn destroyPipe(pipe: [2]c.fd_t) void {
4524	if (pipe[0] != -1) closeFd(pipe[0]);
4525	if (pipe[0] != pipe[1]) closeFd(pipe[1]);
4526	}
4527
4528	const DupError = error{
4529	ProcessFdQuotaExceeded,
4530	SystemResources,
4531	} \|\| Io.UnexpectedError \|\| Io.Cancelable;
4532	fn dup2(ev: *Evented, old_fd: c.fd_t, new_fd: c.fd_t) DupError!void {
4533	_ = ev;
4534	while (true) switch (c.errno(c.dup2(old_fd, new_fd))) {
4535	.SUCCESS => return,
4536	.BUSY, .INTR => {},
4537	.INVAL => \|err\| return errnoBug(err), // invalid parameters
4538	.BADF => \|err\| return errnoBug(err), // use after free
4539	.MFILE => return error.ProcessFdQuotaExceeded,
4540	.NOMEM => return error.SystemResources,
4541	else => \|err\| return unexpectedErrno(err),
4542	};
4543	}
4544
4545	fn execv(
4546	ev: *Evented,
4547	arg0_expand: process.ArgExpansion,
4548	file: [*:0]const u8,
4549	child_argv: [:null]?[:0]const u8,
4550	env_block: process.Environ.PosixBlock,
4551	PATH: []const u8,
4552	) process.ReplaceError {
4553	const file_slice = std.mem.sliceTo(file, 0);
4554	if (std.mem.findScalar(u8, file_slice, '/') != null) return ev.execvPath(file, child_argv, env_block);
4555
4556	// Use of PATH_MAX here is valid as the path_buf will be passed
4557	// directly to the operating system in posixExecvPath.
4558	var path_buf: [c.PATH_MAX]u8 = undefined;
4559	var it = std.mem.tokenizeScalar(u8, PATH, ':');
4560	var seen_eacces = false;
4561	var err: process.ReplaceError = error.FileNotFound;
4562
4563	// In case of expanding arg0 we must put it back if we return with an error.
4564	const prev_arg0 = child_argv[0];
4565	defer switch (arg0_expand) {
4566	.expand => child_argv[0] = prev_arg0,
4567	.no_expand => {},
4568	};
4569
4570	while (it.next()) \|search_path\| {
4571	const path_len = search_path.len + file_slice.len + 1;
4572	if (path_buf.len < path_len + 1) return error.NameTooLong;
4573	@memcpy(path_buf[0..search_path.len], search_path);
4574	path_buf[search_path.len] = '/';
4575	@memcpy(path_buf[search_path.len + 1 ..][0..file_slice.len], file_slice);
4576	path_buf[path_len] = 0;
4577	const full_path = path_buf[0..path_len :0].ptr;
4578	switch (arg0_expand) {
4579	.expand => child_argv[0] = full_path,
4580	.no_expand => {},
4581	}
4582	err = ev.execvPath(full_path, child_argv, env_block);
4583	switch (err) {
4584	error.AccessDenied => seen_eacces = true,
4585	error.FileNotFound, error.NotDir => {},
4586	else => \|e\| return e,
4587	}
4588	}
4589	if (seen_eacces) return error.AccessDenied;
4590	return err;
4591	}
4592	/// This function ignores PATH environment variable.
4593	fn execvPath(
4594	ev: *Evented,
4595	path: [*:0]const u8,
4596	child_argv: [:null]const ?[:0]const u8,
4597	env_block: process.Environ.PosixBlock,
4598	) process.ReplaceError {
4599	_ = ev;
4600	switch (c.errno(c.execve(path, child_argv, env_block.slice.ptr))) {
4601	.FAULT => \|err\| return errnoBug(err), // Bad pointer parameter.
4602	.@"2BIG" => return error.SystemResources,
4603	.MFILE => return error.ProcessFdQuotaExceeded,
4604	.NAMETOOLONG => return error.NameTooLong,
4605	.NFILE => return error.SystemFdQuotaExceeded,
4606	.NOMEM => return error.SystemResources,
4607	.ACCES => return error.AccessDenied,
4608	.PERM => return error.PermissionDenied,
4609	.INVAL => return error.InvalidExe,
4610	.NOEXEC => return error.InvalidExe,
4611	.IO => return error.FileSystem,
4612	.LOOP => return error.FileSystem,
4613	.ISDIR => return error.IsDir,
4614	.NOENT => return error.FileNotFound,
4615	.NOTDIR => return error.NotDir,
4616	.TXTBSY => return error.FileBusy,
4617	.BADEXEC => return error.InvalidExe,
4618	.BADARCH => return error.InvalidExe,
4619	else => \|err\| return unexpectedErrno(err),
4620	}
4621	}
4622
4623	fn childWait(userdata: ?anyopaque, child: process.Child) process.Child.WaitError!process.Child.Term {
4624	const ev: *Evented = @ptrCast(@alignCast(userdata));
4625	defer ev.childCleanup(child);
4626	const pid = child.id.?;
4627	const source = c.dispatch.source_create(
4628	.PROC,
4629	@bitCast(@as(isize, pid)),
4630	.{ .PROC = .{ .EXIT = true } },
4631	ev.queue,
4632	) orelse return error.Unexpected;
4633	source.as_object().set_context(Thread.current().currentFiber());
4634	source.set_event_handler(&Fiber.@"resume");
4635	ev.yield(.{ .activate = source.as_object() });
4636	source.as_object().release();
4637	var status: c_int = undefined;
4638	var ru: c.rusage = undefined;
4639	const ru_ptr = if (child.request_resource_usage_statistics) &ru else null;
4640	while (true) switch (c.errno(c.wait4(pid, &status, 0, ru_ptr))) {
4641	.SUCCESS => {
4642	if (ru_ptr) \|p\| child.resource_usage_statistics.rusage = p.*;
4643	return statusToTerm(@bitCast(status));
4644	},
4645	.INTR => {},
4646	.CHILD => \|err\| return errnoBug(err), // Double-free.
4647	else => \|err\| return unexpectedErrno(err),
4648	};
4649	}
4650
4651	fn childKill(userdata: ?anyopaque, child: process.Child) void {
4652	const ev: *Evented = @ptrCast(@alignCast(userdata));
4653	defer ev.childCleanup(child);
4654	const pid = child.id.?;
4655	while (true) switch (c.errno(c.kill(pid, .TERM))) {
4656	.SUCCESS => break,
4657	.INTR => {},
4658	.PERM => return,
4659	.INVAL => \|err\| errnoBug(err) catch return,
4660	.SRCH => \|err\| errnoBug(err) catch return,
4661	else => \|err\| unexpectedErrno(err) catch return,
4662	};
4663	var status: c_int = undefined;
4664	while (true) switch (c.errno(c.wait4(pid, &status, 0, null))) {
4665	.SUCCESS => return,
4666	.INTR => {},
4667	.CHILD => \|err\| errnoBug(err) catch return, // Double-free.
4668	else => \|err\| unexpectedErrno(err) catch return,
4669	};
4670	}
4671
4672	fn childCleanup(ev: Evented, child: process.Child) void {
4673	if (child.stdin) \|stdin\| {
4674	fileClose(ev, &.{stdin});
4675	child.stdin = null;
4676	}
4677	if (child.stdout) \|stdout\| {
4678	fileClose(ev, &.{stdout});
4679	child.stdout = null;
4680	}
4681	if (child.stderr) \|stderr\| {
4682	fileClose(ev, &.{stderr});
4683	child.stderr = null;
4684	}
4685	child.id = null;
4686	}
4687
4688	fn progressParentFile(userdata: ?*anyopaque) std.Progress.ParentFileError!File {
4689	const ev: *Evented = @ptrCast(@alignCast(userdata));
4690	ev.scan_environ.once(ev, &scanEnviron);
4691	return ev.environ.zig_progress_file;
4692	}
4693
4694	fn scanEnviron(context: ?*anyopaque) callconv(.c) void {
4695	const ev: *Evented = @ptrCast(@alignCast(context));
4696	ev.environ.scan(ev.allocator());
4697	}
4698
4699	fn now(userdata: ?*anyopaque, clock: Io.Clock) Io.Timestamp {
4700	const ev: *Evented = @ptrCast(@alignCast(userdata));
4701	_ = ev;
4702	const clock_id: c.clockid_t = clockToPosix(clock);
4703	var timespec: c.timespec = undefined;
4704	switch (c.errno(c.clock_gettime(clock_id, &timespec))) {
4705	.SUCCESS => return timestampFromPosix(&timespec),
4706	else => return .zero,
4707	}
4708	}
4709
4710	fn clockResolution(userdata: ?*anyopaque, clock: Io.Clock) Io.Clock.ResolutionError!Io.Duration {
4711	const ev: *Evented = @ptrCast(@alignCast(userdata));
4712	_ = ev;
4713	const clock_id: c.clockid_t = clockToPosix(clock);
4714	var timespec: c.timespec = undefined;
4715	return switch (c.errno(c.clock_getres(clock_id, &timespec))) {
4716	.SUCCESS => .fromNanoseconds(nanosecondsFromPosix(&timespec)),
4717	.INVAL => return error.ClockUnavailable,
4718	else => \|err\| return unexpectedErrno(err),
4719	};
4720	}
4721
4722	fn sleep(userdata: ?*anyopaque, timeout: Io.Timeout) Io.Cancelable!void {
4723	const ev: *Evented = @ptrCast(@alignCast(userdata));
4724	ev.yield(.{ .sleep = ev.timeFromTimeout(timeout) });
4725	}
4726
4727	fn timeFromTimeout(ev: *Evented, timeout: Io.Timeout) c.dispatch.time_t {
4728	return timeout: switch (timeout) {
4729	.none => .FOREVER,
4730	.duration => \|duration\| .time(switch (duration.clock) {
4731	.real => .WALL_NOW,
4732	else => .NOW,
4733	}, std.math.lossyCast(i64, duration.raw.toNanoseconds())),
4734	.deadline => \|deadline\| switch (deadline.clock) {
4735	.real => .walltime(&.{
4736	.sec = @intCast(@divFloor(deadline.raw.toNanoseconds(), std.time.ns_per_s)),
4737	.nsec = @intCast(@mod(deadline.raw.toNanoseconds(), std.time.ns_per_s)),
4738	}, 0),
4739	else => continue :timeout .{ .duration = deadline.durationFromNow(ev.io()) },
4740	},
4741	};
4742	}
4743
4744	const Random = struct {
4745	evented: *Evented,
4746	thread: *Thread,
4747	buffer: []u8,
4748
4749	fn seed(context: ?*anyopaque) callconv(.c) void {
4750	const rand: *Random = @ptrCast(@alignCast(context));
4751	const ev = rand.evented;
4752	ev.csprng_mutex.lockUncancelable(ev);
4753	defer ev.csprng_mutex.unlock();
4754	var buffer: [Csprng.seed_len]u8 = undefined;
4755	if (!ev.csprng.isInitialized()) {
4756	@branchHint(.unlikely);
4757	const cancel_protection = swapCancelProtection(ev, .blocked);
4758	defer assert(swapCancelProtection(ev, cancel_protection) == .blocked);
4759	randomSecure(ev, &buffer) catch \|err\| switch (err) {
4760	error.Canceled => unreachable, // blocked
4761	error.EntropyUnavailable => fallbackSeed(ev, &buffer),
4762	};
4763	ev.csprng.rng = .init(buffer);
4764	}
4765	ev.csprng.rng.fill(&buffer);
4766	rand.thread.csprng.rng = .init(buffer);
4767	rand.thread.csprng.rng.fill(rand.buffer);
4768	rand.buffer.len = 0;
4769	}
4770	};
4771
4772	fn random(userdata: ?*anyopaque, buffer: []u8) void {
4773	const ev: *Evented = @ptrCast(@alignCast(userdata));
4774	if (buffer.len == 0) return;
4775	const thread: *Thread = .current();
4776	var rand: Random = .{ .evented = ev, .thread = thread, .buffer = buffer };
4777	thread.seed_csprng.once(&rand, &Random.seed);
4778	if (rand.buffer.len > 0) thread.csprng.rng.fill(buffer);
4779	}
4780
4781	fn randomSecure(userdata: ?*anyopaque, buffer: []u8) Io.RandomSecureError!void {
4782	const ev: *Evented = @ptrCast(@alignCast(userdata));
4783	_ = ev;
4784	if (buffer.len > 0) c.arc4random_buf(buffer.ptr, buffer.len);
4785	}
4786
4787	fn netListenIpUnavailable(
4788	userdata: ?*anyopaque,
4789	address: net.IpAddress,
4790	options: net.IpAddress.ListenOptions,
4791	) net.IpAddress.ListenError!net.Server {
4792	const ev: *Evented = @ptrCast(@alignCast(userdata));
4793	_ = ev;
4794	_ = address;
4795	_ = options;
4796	return error.NetworkDown;
4797	}
4798
4799	fn netAcceptUnavailable(
4800	userdata: ?*anyopaque,
4801	listen_handle: net.Socket.Handle,
4802	) net.Server.AcceptError!net.Stream {
4803	const ev: *Evented = @ptrCast(@alignCast(userdata));
4804	_ = ev;
4805	_ = listen_handle;
4806	return error.NetworkDown;
4807	}
4808
4809	fn netBindIpUnavailable(
4810	userdata: ?*anyopaque,
4811	address: *const net.IpAddress,
4812	options: net.IpAddress.BindOptions,
4813	) net.IpAddress.BindError!net.Socket {
4814	const ev: *Evented = @ptrCast(@alignCast(userdata));
4815	_ = ev;
4816	_ = address;
4817	_ = options;
4818	return error.NetworkDown;
4819	}
4820
4821	fn netConnectIpUnavailable(
4822	userdata: ?*anyopaque,
4823	address: *const net.IpAddress,
4824	options: net.IpAddress.ConnectOptions,
4825	) net.IpAddress.ConnectError!net.Stream {
4826	const ev: *Evented = @ptrCast(@alignCast(userdata));
4827	_ = ev;
4828	_ = address;
4829	_ = options;
4830	return error.NetworkDown;
4831	}
4832
4833	fn netListenUnixUnavailable(
4834	userdata: ?*anyopaque,
4835	address: *const net.UnixAddress,
4836	options: net.UnixAddress.ListenOptions,
4837	) net.UnixAddress.ListenError!net.Socket.Handle {
4838	const ev: *Evented = @ptrCast(@alignCast(userdata));
4839	_ = ev;
4840	_ = address;
4841	_ = options;
4842	return error.AddressFamilyUnsupported;
4843	}
4844
4845	fn netConnectUnixUnavailable(
4846	userdata: ?*anyopaque,
4847	address: *const net.UnixAddress,
4848	) net.UnixAddress.ConnectError!net.Socket.Handle {
4849	const ev: *Evented = @ptrCast(@alignCast(userdata));
4850	_ = ev;
4851	_ = address;
4852	return error.AddressFamilyUnsupported;
4853	}
4854
4855	fn netSocketCreatePairUnavailable(
4856	userdata: ?*anyopaque,
4857	options: net.Socket.CreatePairOptions,
4858	) net.Socket.CreatePairError![2]net.Socket {
4859	_ = userdata;
4860	_ = options;
4861	return error.OperationUnsupported;
4862	}
4863
4864	fn netSendUnavailable(
4865	userdata: ?*anyopaque,
4866	handle: net.Socket.Handle,
4867	messages: []net.OutgoingMessage,
4868	flags: net.SendFlags,
4869	) struct { ?net.Socket.SendError, usize } {
4870	const ev: *Evented = @ptrCast(@alignCast(userdata));
4871	_ = ev;
4872	_ = handle;
4873	_ = messages;
4874	_ = flags;
4875	return .{ error.NetworkDown, 0 };
4876	}
4877
4878	fn netReceiveUnavailable(
4879	userdata: ?*anyopaque,
4880	handle: net.Socket.Handle,
4881	message_buffer: []net.IncomingMessage,
4882	data_buffer: []u8,
4883	flags: net.ReceiveFlags,
4884	timeout: Io.Timeout,
4885	) struct { ?net.Socket.ReceiveTimeoutError, usize } {
4886	const ev: *Evented = @ptrCast(@alignCast(userdata));
4887	_ = ev;
4888	_ = handle;
4889	_ = message_buffer;
4890	_ = data_buffer;
4891	_ = flags;
4892	_ = timeout;
4893	return .{ error.NetworkDown, 0 };
4894	}
4895
4896	fn netReadUnavailable(
4897	userdata: ?*anyopaque,
4898	fd: net.Socket.Handle,
4899	data: [][]u8,
4900	) net.Stream.Reader.Error!usize {
4901	const ev: *Evented = @ptrCast(@alignCast(userdata));
4902	_ = ev;
4903	_ = fd;
4904	_ = data;
4905	return error.NetworkDown;
4906	}
4907
4908	fn netWriteUnavailable(
4909	userdata: ?*anyopaque,
4910	handle: net.Socket.Handle,
4911	header: []const u8,
4912	data: []const []const u8,
4913	splat: usize,
4914	) net.Stream.Writer.Error!usize {
4915	const ev: *Evented = @ptrCast(@alignCast(userdata));
4916	_ = ev;
4917	_ = handle;
4918	_ = header;
4919	_ = data;
4920	_ = splat;
4921	return error.NetworkDown;
4922	}
4923
4924	fn netWriteFileUnavailable(
4925	userdata: ?*anyopaque,
4926	socket_handle: net.Socket.Handle,
4927	header: []const u8,
4928	file_reader: *File.Reader,
4929	limit: Io.Limit,
4930	) net.Stream.Writer.WriteFileError!usize {
4931	const ev: *Evented = @ptrCast(@alignCast(userdata));
4932	_ = ev;
4933	_ = socket_handle;
4934	_ = header;
4935	_ = file_reader;
4936	_ = limit;
4937	return error.NetworkDown;
4938	}
4939
4940	fn netClose(userdata: ?*anyopaque, handles: []const net.Socket.Handle) void {
4941	const ev: *Evented = @ptrCast(@alignCast(userdata));
4942	_ = ev;
4943	for (handles) \|handle\| closeFd(handle);
4944	}
4945
4946	fn netShutdownUnavailable(
4947	userdata: ?*anyopaque,
4948	handle: net.Socket.Handle,
4949	how: net.ShutdownHow,
4950	) net.ShutdownError!void {
4951	const ev: *Evented = @ptrCast(@alignCast(userdata));
4952	_ = ev;
4953	_ = handle;
4954	_ = how;
4955	unreachable; // How you gonna shutdown something that was impossible to open?
4956	}
4957
4958	fn netInterfaceNameResolveUnavailable(
4959	userdata: ?*anyopaque,
4960	name: *const net.Interface.Name,
4961	) net.Interface.Name.ResolveError!net.Interface {
4962	const ev: *Evented = @ptrCast(@alignCast(userdata));
4963	_ = ev;
4964	_ = name;
4965	return error.InterfaceNotFound;
4966	}
4967
4968	fn netInterfaceNameUnavailable(
4969	userdata: ?*anyopaque,
4970	interface: net.Interface,
4971	) net.Interface.NameError!net.Interface.Name {
4972	const ev: *Evented = @ptrCast(@alignCast(userdata));
4973	_ = ev;
4974	_ = interface;
4975	return error.Unexpected;
4976	}
4977
4978	fn netLookupUnavailable(
4979	userdata: ?*anyopaque,
4980	host_name: net.HostName,
4981	resolved: *Io.Queue(net.HostName.LookupResult),
4982	options: net.HostName.LookupOptions,
4983	) net.HostName.LookupError!void {
4984	const ev: *Evented = @ptrCast(@alignCast(userdata));
4985	_ = host_name;
4986	_ = options;
4987	resolved.close(ev.io());
4988	return error.NetworkDown;
4989	}
4990
4991	fn readAll(ev: *Evented, file: File, buffer: []u8) File.ReadStreamingError!void {
4992	var index: usize = 0;
4993	while (buffer.len - index != 0) {
4994	const len = try ev.fileReadStreaming(file, &.{buffer[index..]});
4995	if (len == 0) return error.EndOfStream;
4996	index += len;
4997	}
4998	}
4999
5000	fn writeAll(ev: *Evented, file: File, buffer: []const u8) (File.Writer.Error \|\| error{EndOfStream})!void {
5001	var index: usize = 0;
5002	while (buffer.len - index != 0) {
5003	const len = try ev.fileWriteStreaming(file, &.{}, &.{buffer[index..]}, 1);
5004	if (len == 0) return error.EndOfStream;
5005	index += len;
5006	}
5007	}
5008
5009	/// This is either usize or u32. Since, either is fine, let's use the same
5010	/// `addBuf` function for both writing to a file and sending network messages.
5011	const iovlen_t = @FieldType(c.msghdr_const, "iovlen");
5012
5013	fn addConstBuf(v: []iovec_const, i: iovlen_t, remaining: ?usize, bytes: []const u8) void {
5014	if (v.len - i.* == 0) return;
5015	const len = @min(remaining.*, bytes.len);
5016	if (len == 0) return;
5017	v[i.*] = .{ .base = bytes.ptr, .len = len };
5018	i.* += 1;
5019	remaining.* -= len;
5020	}
5021	fn addBuf(
5022	comptime is_const: bool,
5023	vec: []if (is_const) iovec_const else iovec,
5024	vec_len: *iovlen_t,
5025	remaining: *Io.Limit,
5026	bytes: if (is_const) []const u8 else []u8,
5027	) void {
5028	if (vec.len - vec_len.* == 0) return;
5029	const len = remaining.minInt(bytes.len);
5030	if (len == 0) return;
5031	vec[vec_len.*] = .{ .base = bytes.ptr, .len = len };
5032	vec_len.* += 1;
5033	remaining.* = remaining.subtract(len).?;
5034	}
5035
5036	test {
5037	_ = Fiber.CancelProtection;
5038	}

lib/std/Io/IoUring.zig deleted-6336

...	@@ -1,6336 +0,0 @@
1	const addressFromPosix = Io.Threaded.addressFromPosix;
2	const addressToPosix = Io.Threaded.addressToPosix;
3	const Alignment = std.mem.Alignment;
4	const Allocator = std.mem.Allocator;
5	const Argv0 = Io.Threaded.Argv0;
6	const assert = std.debug.assert;
7	const builtin = @import("builtin");
8	const ChdirError = Io.Threaded.ChdirError;
9	const clockToPosix = Io.Threaded.clockToPosix;
10	const Csprng = Io.Threaded.Csprng;
11	const default_PATH = Io.Threaded.default_PATH;
12	const Dir = Io.Dir;
13	const Environ = Io.Threaded.Environ;
14	const errnoBug = Io.Threaded.errnoBug;
15	const Evented = @This();
16	const fallbackSeed = Io.Threaded.fallbackSeed;
17	const fd_t = linux.fd_t;
18	const File = Io.File;
19	const Io = std.Io;
20	const IoUring = linux.IoUring;
21	const iovec = std.posix.iovec;
22	const iovec_const = std.posix.iovec_const;
23	const linux = std.os.linux;
24	const linux_statx_request = Io.Threaded.linux_statx_request;
25	const LOCK = std.posix.LOCK;
26	const log = std.log.scoped(.@"io-uring");
27	const max_iovecs_len = Io.Threaded.max_iovecs_len;
28	const nanosecondsFromPosix = Io.Threaded.nanosecondsFromPosix;
29	const net = Io.net;
30	const PATH_MAX = linux.PATH_MAX;
31	const pathToPosix = Io.Threaded.pathToPosix;
32	const pid_t = linux.pid_t;
33	const PosixAddress = Io.Threaded.PosixAddress;
34	const posixAddressFamily = Io.Threaded.posixAddressFamily;
35	const posixProtocol = Io.Threaded.posixProtocol;
36	const posixSocketMode = Io.Threaded.posixSocketMode;
37	const process = std.process;
38	const recoverableOsBugDetected = Io.Threaded.recoverableOsBugDetected;
39	const setTimestampToPosix = Io.Threaded.setTimestampToPosix;
40	const splat_buffer_size = Io.Threaded.splat_buffer_size;
41	const statFromLinux = Io.Threaded.statFromLinux;
42	const std = @import("../std.zig");
43	const timestampFromPosix = Io.Threaded.timestampFromPosix;
44	const unexpectedErrno = std.posix.unexpectedErrno;
45	const winsize = std.posix.winsize;
46
47	const tracy = if (@hasDecl(@import("root"), "tracy")) @import("root").tracy else struct {
48	const enable = false;
49	inline fn fiberEnter(fiber: [*:0]const u8) void {
50	_ = fiber;
51	}
52	inline fn fiberLeave() void {}
53	};
54
55	backing_allocator_needs_mutex: bool,
56	backing_allocator_mutex: Io.Mutex,
57	/// Does not need to be thread-safe if not used elsewhere.
58	backing_allocator: Allocator,
59	main_fiber_buffer: [
60	std.mem.alignForward(usize, @sizeOf(Fiber), @alignOf(Completion)) + @sizeOf(Completion)
61	]u8 align(@max(@alignOf(Fiber), @alignOf(Completion))),
62	log2_ring_entries: u4,
63	threads: Thread.List,
64	sync_limit: ?Io.Semaphore,
65
66	stderr_mutex: Io.Mutex,
67	stderr_writer: File.Writer = .{
68	.io = undefined,
69	.interface = Io.File.Writer.initInterface(&.{}),
70	.file = .stderr(),
71	.mode = .streaming,
72	},
73	stderr_mode: Io.Terminal.Mode = .no_color,
74	stderr_writer_initialized: bool = false,
75
76	environ_mutex: Io.Mutex,
77	environ: Environ,
78
79	null_fd: CachedFd,
80	random_fd: CachedFd,
81
82	csprng_mutex: Io.Mutex,
83	csprng: Csprng,
84
85	/// Empirically saw >128KB being used by the self-hosted backend to panic.
86	/// Empirically saw glibc complain about 256KB.
87	const idle_stack_size = 512 * 1024;
88
89	const max_idle_search = 1;
90	const max_steal_ready_search = 2;
91	const max_steal_free_search = 4;
92
93	const Thread = struct {
94	required_align: void align(4),
95	thread: std.Thread,
96	idle_context: Context,
97	current_context: *Context,
98	ready_queue: ?*Fiber,
99	free_queue: ?*Fiber,
100	io_uring: IoUring,
101	idle_search_index: u32,
102	steal_ready_search_index: u32,
103	steal_free_search_index: u32,
104	name_arena: if (tracy.enable) std.heap.ArenaAllocator.State else struct {},
105	csprng: Csprng,
106
107	threadlocal var self: ?*Thread = null;
108
109	noinline fn current() *Thread {
110	return self.?;
111	}
112
113	fn deinit(thread: *Thread, gpa: Allocator) void {
114	var next_fiber = thread.free_queue;
115	while (next_fiber) \|free_fiber\| {
116	next_fiber = free_fiber.status.free_next;
117	gpa.free(free_fiber.allocatedSlice());
118	}
119	thread.io_uring.deinit();
120	}
121
122	fn currentFiber(thread: Thread) Fiber {
123	assert(thread.current_context != &thread.idle_context);
124	return @fieldParentPtr("context", thread.current_context);
125	}
126
127	fn enqueue(thread: Thread) linux.io_uring_sqe {
128	while (true) return thread.io_uring.get_sqe() catch {
129	thread.submit();
130	continue;
131	};
132	}
133
134	fn submit(thread: *Thread) void {
135	_ = thread.io_uring.submit() catch \|err\| switch (err) {
136	error.SignalInterrupt => {},
137	else => \|e\| @panic(@errorName(e)),
138	};
139	}
140
141	const List = struct {
142	allocated: []Thread,
143	reserved: u32,
144	active: u32,
145	};
146	};
147
148	const Fiber = struct {
149	required_align: void align(4),
150	context: Context,
151	await_count: i32,
152	link: union {
153	awaiter: ?*Fiber,
154	group: struct { prev: ?Fiber, next: ?Fiber },
155	},
156	status: union(enum) {
157	queue_next: ?*Fiber,
158	awaiting_group: Group,
159	free_next: ?*Fiber,
160	},
161	cancel_status: CancelStatus,
162	cancel_protection: CancelProtection,
163	name: if (tracy.enable) [*:0]const u8 else void,
164
165	var next_name: u64 = 0;
166
167	const CancelStatus = packed struct(u32) {
168	requested: bool,
169	awaiting: Awaiting,
170
171	const unrequested: CancelStatus = .{ .requested = false, .awaiting = .nothing };
172
173	const Awaiting = enum(u31) {
174	nothing = std.math.maxInt(u31),
175	group = std.math.maxInt(u31) - 1,
176	select = std.math.maxInt(u31) - 2,
177	/// An io_uring fd.
178	_,
179
180	fn subWrap(lhs: Awaiting, rhs: Awaiting) Awaiting {
181	return @enumFromInt(@intFromEnum(lhs) -% @intFromEnum(rhs));
182	}
183
184	fn fromIoUringFd(fd: fd_t) Awaiting {
185	const awaiting: Awaiting = @enumFromInt(fd);
186	switch (awaiting) {
187	.nothing, .group, .select => unreachable,
188	_ => return awaiting,
189	}
190	}
191
192	fn toIoUringFd(awaiting: Awaiting) fd_t {
193	switch (awaiting) {
194	.nothing, .group => unreachable,
195	_ => return @intFromEnum(awaiting),
196	}
197	}
198	};
199
200	fn changeAwaiting(
201	cancel_status: *CancelStatus,
202	old_awaiting: Awaiting,
203	new_awaiting: Awaiting,
204	) bool {
205	const old_cancel_status = @atomicRmw(CancelStatus, cancel_status, .Add, .{
206	.requested = false,
207	.awaiting = new_awaiting.subWrap(old_awaiting),
208	}, .monotonic);
209	assert(old_cancel_status.awaiting == old_awaiting);
210	return old_cancel_status.requested;
211	}
212	};
213
214	const CancelProtection = packed struct {
215	user: Io.CancelProtection,
216	acknowledged: bool,
217
218	const unblocked: CancelProtection = .{ .user = .unblocked, .acknowledged = false };
219
220	fn check(cancel_protection: CancelProtection) Io.CancelProtection {
221	return @enumFromInt(@intFromBool(cancel_protection != unblocked));
222	}
223
224	fn acknowledge(cancel_protection: *CancelProtection) void {
225	assert(!cancel_protection.acknowledged);
226	cancel_protection.acknowledged = true;
227	}
228
229	fn recancel(cancel_protection: *CancelProtection) void {
230	assert(cancel_protection.acknowledged);
231	cancel_protection.acknowledged = false;
232	}
233
234	test check {
235	try std.testing.expectEqual(Io.CancelProtection.unblocked, check(.unblocked));
236	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
237	.user = .unblocked,
238	.acknowledged = true,
239	}));
240	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
241	.user = .blocked,
242	.acknowledged = false,
243	}));
244	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
245	.user = .blocked,
246	.acknowledged = true,
247	}));
248	}
249	};
250
251	const finished: ?*Fiber = @ptrFromInt(@alignOf(Fiber));
252
253	const max_result_align: Alignment = .@"16";
254	const max_result_size = max_result_align.forward(512);
255	/// This includes any stack realignments that need to happen, and also the
256	/// initial frame return address slot and argument frame, depending on target.
257	const min_stack_size = 60 * 1024 * 1024;
258	const max_context_align: Alignment = .@"16";
259	const max_context_size = max_context_align.forward(1024);
260	const max_closure_size: usize = @sizeOf(AsyncClosure);
261	const max_closure_align: Alignment = .of(AsyncClosure);
262	const allocation_size = std.mem.alignForward(
263	usize,
264	max_closure_align.max(max_context_align).forward(
265	max_result_align.forward(@sizeOf(Fiber)) + max_result_size + min_stack_size,
266	) + max_closure_size + max_context_size,
267	std.heap.page_size_max,
268	);
269	comptime {
270	assert(max_result_align.compare(.gte, .of(Completion)));
271	assert(max_result_size >= @sizeOf(Completion));
272	}
273
274	fn create(ev: Evented) error{OutOfMemory}!Fiber {
275	const thread: *Thread = .current();
276	if (@atomicRmw(?*Fiber, &thread.free_queue, .Xchg, finished, .acquire)) \|free_fiber\| {
277	assert(free_fiber != finished);
278	@atomicStore(?*Fiber, &thread.free_queue, free_fiber.status.free_next, .release);
279	return free_fiber;
280	}
281	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
282	for (0..@min(max_steal_free_search, active_threads)) \|_\| {
283	defer thread.steal_free_search_index += 1;
284	if (thread.steal_free_search_index == active_threads) thread.steal_free_search_index = 0;
285	const steal_free_search_thread =
286	&ev.threads.allocated[0..active_threads][thread.steal_free_search_index];
287	if (steal_free_search_thread == thread) continue;
288	const free_fiber =
289	@atomicLoad(?*Fiber, &steal_free_search_thread.free_queue, .monotonic) orelse continue;
290	if (free_fiber == finished) continue;
291	if (@cmpxchgWeak(
292	?*Fiber,
293	&steal_free_search_thread.free_queue,
294	free_fiber,
295	null,
296	.acquire,
297	.monotonic,
298	)) \|_\| continue;
299	@atomicStore(?*Fiber, &thread.free_queue, free_fiber.status.free_next, .release);
300	return free_fiber;
301	}
302	@atomicStore(?*Fiber, &thread.free_queue, null, .monotonic);
303	return @ptrCast(try ev.allocator().alignedAlloc(u8, .of(Fiber), allocation_size));
304	}
305
306	fn destroy(fiber: *Fiber) void {
307	const thread: *Thread = .current();
308	assert(fiber.status.queue_next == null);
309	fiber.status = .{ .free_next = @atomicLoad(?*Fiber, &thread.free_queue, .acquire) };
310	while (true) fiber.status.free_next = @cmpxchgWeak(
311	?*Fiber,
312	&thread.free_queue,
313	fiber.status.free_next,
314	fiber,
315	.acq_rel,
316	.acquire,
317	) orelse break;
318	}
319
320	fn allocatedSlice(f: *Fiber) []align(@alignOf(Fiber)) u8 {
321	return @as([*]align(@alignOf(Fiber)) u8, @ptrCast(f))[0..allocation_size];
322	}
323
324	fn allocatedEnd(f: Fiber) []u8 {
325	const allocated_slice = f.allocatedSlice();
326	return allocated_slice[allocated_slice.len..].ptr;
327	}
328
329	fn resultPointer(f: Fiber, comptime Result: type) Result {
330	return @ptrCast(@alignCast(f.resultBytes(.of(Result))));
331	}
332
333	fn resultBytes(f: Fiber, alignment: Alignment) []u8 {
334	return @ptrFromInt(alignment.forward(@intFromPtr(f) + @sizeOf(Fiber)));
335	}
336
337	const Queue = struct { head: Fiber, tail: Fiber };
338
339	/// Like a `*Fiber`, but 2 bits smaller than a pointer (because the LSBs are always 0 due to
340	/// alignment) so that those two bits can be used in a `packed struct`.
341	const PackedPtr = enum(@Int(.unsigned, @bitSizeOf(usize) - 2)) {
342	null = 0,
343	all_ones = std.math.maxInt(@Int(.unsigned, @bitSizeOf(usize) - 2)),
344	_,
345
346	const Split = packed struct(usize) { low: u2, high: PackedPtr };
347	fn pack(ptr: ?*Fiber) PackedPtr {
348	const split: Split = @bitCast(@intFromPtr(ptr));
349	assert(split.low == 0);
350	return split.high;
351	}
352	fn unpack(ptr: PackedPtr) ?*Fiber {
353	const split: Split = .{ .low = 0, .high = ptr };
354	return @ptrFromInt(@as(usize, @bitCast(split)));
355	}
356	};
357
358	fn requestCancel(fiber: Fiber, ev: Evented) void {
359	const cancel_status = @atomicRmw(
360	Fiber.CancelStatus,
361	&fiber.cancel_status,
362	.Or,
363	.{ .requested = true, .awaiting = @enumFromInt(0) },
364	.acq_rel,
365	);
366	assert(!cancel_status.requested);
367	switch (cancel_status.awaiting) {
368	.nothing => {},
369	.group => {
370	// The awaiter received a cancelation request while awaiting a group,
371	// so propagate the cancelation to the group.
372	if (fiber.status.awaiting_group.cancel(ev, null)) {
373	fiber.status = .{ .queue_next = null };
374	_ = ev.schedule(.current(), .{ .head = fiber, .tail = fiber });
375	}
376	},
377	.select => if (@atomicRmw(i32, &fiber.await_count, .Add, 1, .monotonic) == -1) {
378	_ = ev.schedule(.current(), .{ .head = fiber, .tail = fiber });
379	},
380	_ => \|cancel_io_uring_fd\| {
381	const thread: *Thread = .current();
382	thread.enqueue().* = if (thread.io_uring.fd == @intFromEnum(cancel_io_uring_fd)) .{
383	.opcode = .ASYNC_CANCEL,
384	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
385	.ioprio = 0,
386	.fd = 0,
387	.off = 0,
388	.addr = @intFromPtr(fiber),
389	.len = 0,
390	.rw_flags = 0,
391	.user_data = @intFromEnum(Completion.UserData.wakeup),
392	.buf_index = 0,
393	.personality = 0,
394	.splice_fd_in = 0,
395	.addr3 = 0,
396	.resv = 0,
397	} else .{
398	.opcode = .MSG_RING,
399	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
400	.ioprio = 0,
401	.fd = @intFromEnum(cancel_io_uring_fd),
402	.off = @intFromPtr(fiber) \| 0b01,
403	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
404	.len = 0,
405	.rw_flags = 0,
406	.user_data = @intFromEnum(Completion.UserData.cleanup),
407	.buf_index = 0,
408	.personality = 0,
409	.splice_fd_in = 0,
410	.addr3 = 0,
411	.resv = 0,
412	};
413	},
414	}
415	}
416	};
417
418	const CancelRegion = struct {
419	fiber: *Fiber,
420	status: Fiber.CancelStatus,
421	fn init() CancelRegion {
422	const fiber = Thread.current().currentFiber();
423	return .{
424	.fiber = fiber,
425	.status = .{
426	.requested = fiber.cancel_protection.check() == .unblocked,
427	.awaiting = .nothing,
428	},
429	};
430	}
431	fn initBlocked() CancelRegion {
432	return .{
433	.fiber = Thread.current().currentFiber(),
434	.status = .{ .requested = false, .awaiting = .nothing },
435	};
436	}
437	fn deinit(cancel_region: *CancelRegion) void {
438	if (cancel_region.status.requested) _ = cancel_region.fiber.cancel_status.changeAwaiting(
439	cancel_region.status.awaiting,
440	.nothing,
441	);
442	cancel_region.* = undefined;
443	}
444	fn await(cancel_region: *CancelRegion, awaiting: Fiber.CancelStatus.Awaiting) Io.Cancelable!void {
445	if (!cancel_region.status.requested) return;
446	const status: Fiber.CancelStatus = .{ .requested = true, .awaiting = awaiting };
447	if (cancel_region.fiber.cancel_status.changeAwaiting(
448	cancel_region.status.awaiting,
449	status.awaiting,
450	)) {
451	cancel_region.fiber.cancel_protection.acknowledge();
452	cancel_region.status = .unrequested;
453	return error.Canceled;
454	}
455	cancel_region.status = status;
456	}
457	fn awaitIoUring(cancel_region: CancelRegion) Io.Cancelable!Thread {
458	const thread: *Thread = .current();
459	try cancel_region.await(.fromIoUringFd(thread.io_uring.fd));
460	return thread;
461	}
462	fn completion(cancel_region: *const CancelRegion) Completion {
463	return cancel_region.fiber.resultPointer(Completion).*;
464	}
465	fn errno(cancel_region: *const CancelRegion) linux.E {
466	return cancel_region.completion().errno();
467	}
468
469	const Sync = struct {
470	cancel_region: CancelRegion,
471	fn init(ev: *Evented) Io.Cancelable!Sync {
472	if (ev.sync_limit) \|*sync_limit\| try sync_limit.wait(ev.io());
473	return .{ .cancel_region = .init() };
474	}
475	fn initBlocked(ev: *Evented) Sync {
476	if (ev.sync_limit) \|*sync_limit\| sync_limit.waitUncancelable(ev.io());
477	return .{ .cancel_region = .initBlocked() };
478	}
479	fn deinit(sync: Sync, ev: Evented) void {
480	sync.cancel_region.deinit();
481	if (ev.sync_limit) \|*sync_limit\| sync_limit.post(ev.io());
482	}
483
484	const Maybe = union(enum) {
485	cancel_region: CancelRegion,
486	sync: Sync,
487
488	fn deinit(maybe: Maybe, ev: Evented) void {
489	switch (maybe.*) {
490	.cancel_region => \|*cancel_region\| cancel_region.deinit(),
491	.sync => \|*sync\| sync.deinit(ev),
492	}
493	}
494
495	fn enterSync(maybe: Maybe, ev: Evented) Io.Cancelable!*Sync {
496	switch (maybe.*) {
497	.cancel_region => \|cancel_region\| {
498	if (ev.sync_limit) \|*sync_limit\| try sync_limit.wait(ev.io());
499	maybe.* = .{ .sync = .{ .cancel_region = cancel_region } };
500	},
501	.sync => {},
502	}
503	return &maybe.sync;
504	}
505
506	fn leaveSync(maybe: Maybe, ev: Evented) void {
507	switch (maybe.*) {
508	.cancel_region => {},
509	.sync => \|sync\| {
510	if (ev.sync_limit) \|*sync_limit\| sync_limit.post(ev.io());
511	maybe.* = .{ .cancel_region = sync.cancel_region };
512	},
513	}
514	}
515
516	fn cancelRegion(maybe: Maybe) CancelRegion {
517	return switch (maybe.*) {
518	.cancel_region => \|*cancel_region\| cancel_region,
519	.sync => \|*sync\| &sync.cancel_region,
520	};
521	}
522	};
523	};
524	};
525
526	const CachedFd = struct {
527	once: Once,
528
529	const Once = enum(fd_t) {
530	uninitialized = -1,
531	initializing = -2,
532	/// fd
533	_,
534
535	fn fromFd(fd: fd_t) Once {
536	return @enumFromInt(@as(u31, @intCast(fd)));
537	}
538
539	fn toFd(once: Once) fd_t {
540	return @as(u31, @intCast(@intFromEnum(once)));
541	}
542	};
543
544	const init: CachedFd = .{ .once = .uninitialized };
545
546	fn close(cached_fd: *CachedFd) void {
547	switch (cached_fd.once) {
548	.uninitialized => {},
549	.initializing => unreachable,
550	_ => \|fd\| {
551	assert(@intFromEnum(fd) >= 0);
552	_ = std.os.linux.close(@intFromEnum(fd));
553	cached_fd.* = .init;
554	},
555	}
556	}
557
558	fn open(
559	cached_fd: *CachedFd,
560	ev: *Evented,
561	cancel_region: *CancelRegion,
562	path: [*:0]const u8,
563	flags: linux.O,
564	) File.OpenError!fd_t {
565	var once = @atomicLoad(Once, &cached_fd.once, .monotonic);
566	while (true) {
567	switch (once) {
568	.uninitialized => {},
569	.initializing => try futexWait(
570	ev,
571	@ptrCast(&cached_fd.once),
572	@bitCast(@intFromEnum(once)),
573	.none,
574	),
575	_ => \|fd\| {
576	@branchHint(.likely);
577	return fd.toFd();
578	},
579	}
580	once = @cmpxchgWeak(
581	Once,
582	&cached_fd.once,
583	.uninitialized,
584	.initializing,
585	.monotonic,
586	.monotonic,
587	) orelse {
588	errdefer {
589	@atomicStore(Once, &cached_fd.once, .uninitialized, .monotonic);
590	futexWake(ev, @ptrCast(&cached_fd.once), 1);
591	}
592	const fd = try ev.openat(cancel_region, linux.AT.FDCWD, path, flags, 0);
593	@atomicStore(Once, &cached_fd.once, .fromFd(fd), .monotonic);
594	futexWake(ev, @ptrCast(&cached_fd.once), std.math.maxInt(u32));
595	return fd;
596	};
597	}
598	}
599	};
600
601	pub fn allocator(ev: *Evented) std.mem.Allocator {
602	return if (ev.backing_allocator_needs_mutex) .{
603	.ptr = ev,
604	.vtable = &.{
605	.alloc = alloc,
606	.resize = resize,
607	.remap = remap,
608	.free = free,
609	},
610	} else ev.backing_allocator;
611	}
612
613	fn alloc(userdata: anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[]u8 {
614	const ev: *Evented = @ptrCast(@alignCast(userdata));
615	const ev_io = ev.io();
616	ev.backing_allocator_mutex.lockUncancelable(ev_io);
617	defer ev.backing_allocator_mutex.unlock(ev_io);
618	return ev.backing_allocator.rawAlloc(len, alignment, ret_addr);
619	}
620
621	fn resize(
622	userdata: *anyopaque,
623	memory: []u8,
624	alignment: std.mem.Alignment,
625	new_len: usize,
626	ret_addr: usize,
627	) bool {
628	const ev: *Evented = @ptrCast(@alignCast(userdata));
629	const ev_io = ev.io();
630	ev.backing_allocator_mutex.lockUncancelable(ev_io);
631	defer ev.backing_allocator_mutex.unlock(ev_io);
632	return ev.backing_allocator.rawResize(memory, alignment, new_len, ret_addr);
633	}
634
635	fn remap(
636	userdata: *anyopaque,
637	memory: []u8,
638	alignment: Alignment,
639	new_len: usize,
640	ret_addr: usize,
641	) ?[*]u8 {
642	const ev: *Evented = @ptrCast(@alignCast(userdata));
643	const ev_io = ev.io();
644	ev.backing_allocator_mutex.lockUncancelable(ev_io);
645	defer ev.backing_allocator_mutex.unlock(ev_io);
646	return ev.backing_allocator.rawRemap(memory, alignment, new_len, ret_addr);
647	}
648
649	fn free(userdata: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void {
650	const ev: *Evented = @ptrCast(@alignCast(userdata));
651	const ev_io = ev.io();
652	ev.backing_allocator_mutex.lockUncancelable(ev_io);
653	defer ev.backing_allocator_mutex.unlock(ev_io);
654	return ev.backing_allocator.rawFree(memory, alignment, ret_addr);
655	}
656
657	pub fn io(ev: *Evented) Io {
658	return .{
659	.userdata = ev,
660	.vtable = &.{
661	.async = async,
662	.concurrent = concurrent,
663	.await = await,
664	.cancel = cancel,
665
666	.groupAsync = groupAsync,
667	.groupConcurrent = groupConcurrent,
668	.groupAwait = groupAwait,
669	.groupCancel = groupCancel,
670
671	.recancel = recancel,
672	.swapCancelProtection = swapCancelProtection,
673	.checkCancel = checkCancel,
674
675	.select = select,
676
677	.futexWait = futexWait,
678	.futexWaitUncancelable = futexWaitUncancelable,
679	.futexWake = futexWake,
680
681	.operate = operate,
682	.batchAwaitAsync = batchAwaitAsync,
683	.batchAwaitConcurrent = batchAwaitConcurrent,
684	.batchCancel = batchCancel,
685
686	.dirCreateDir = dirCreateDir,
687	.dirCreateDirPath = dirCreateDirPath,
688	.dirCreateDirPathOpen = dirCreateDirPathOpen,
689	.dirOpenDir = dirOpenDir,
690	.dirStat = dirStat,
691	.dirStatFile = dirStatFile,
692	.dirAccess = dirAccess,
693	.dirCreateFile = dirCreateFile,
694	.dirCreateFileAtomic = dirCreateFileAtomic,
695	.dirOpenFile = dirOpenFile,
696	.dirClose = dirClose,
697	.dirRead = dirRead,
698	.dirRealPath = dirRealPath,
699	.dirRealPathFile = dirRealPathFile,
700	.dirDeleteFile = dirDeleteFile,
701	.dirDeleteDir = dirDeleteDir,
702	.dirRename = dirRename,
703	.dirRenamePreserve = dirRenamePreserve,
704	.dirSymLink = dirSymLink,
705	.dirReadLink = dirReadLink,
706	.dirSetOwner = dirSetOwner,
707	.dirSetFileOwner = dirSetFileOwner,
708	.dirSetPermissions = dirSetPermissions,
709	.dirSetFilePermissions = dirSetFilePermissions,
710	.dirSetTimestamps = dirSetTimestamps,
711	.dirHardLink = dirHardLink,
712
713	.fileStat = fileStat,
714	.fileLength = fileLength,
715	.fileClose = fileClose,
716	.fileWritePositional = fileWritePositional,
717	.fileWriteFileStreaming = fileWriteFileStreaming,
718	.fileWriteFilePositional = fileWriteFilePositional,
719	.fileReadPositional = fileReadPositional,
720	.fileSeekBy = fileSeekBy,
721	.fileSeekTo = fileSeekTo,
722	.fileSync = fileSync,
723	.fileIsTty = fileIsTty,
724	.fileEnableAnsiEscapeCodes = fileEnableAnsiEscapeCodes,
725	.fileSupportsAnsiEscapeCodes = fileIsTty,
726	.fileSetLength = fileSetLength,
727	.fileSetOwner = fileSetOwner,
728	.fileSetPermissions = fileSetPermissions,
729	.fileSetTimestamps = fileSetTimestamps,
730	.fileLock = fileLock,
731	.fileTryLock = fileTryLock,
732	.fileUnlock = fileUnlock,
733	.fileDowngradeLock = fileDowngradeLock,
734	.fileRealPath = fileRealPath,
735	.fileHardLink = fileHardLink,
736
737	.fileMemoryMapCreate = fileMemoryMapCreate,
738	.fileMemoryMapDestroy = fileMemoryMapDestroy,
739	.fileMemoryMapSetLength = fileMemoryMapSetLength,
740	.fileMemoryMapRead = fileMemoryMapRead,
741	.fileMemoryMapWrite = fileMemoryMapWrite,
742
743	.processExecutableOpen = processExecutableOpen,
744	.processExecutablePath = processExecutablePath,
745	.lockStderr = lockStderr,
746	.tryLockStderr = tryLockStderr,
747	.unlockStderr = unlockStderr,
748	.processCurrentPath = processCurrentPath,
749	.processSetCurrentDir = processSetCurrentDir,
750	.processReplace = processReplace,
751	.processReplacePath = processReplacePath,
752	.processSpawn = processSpawn,
753	.processSpawnPath = processSpawnPath,
754	.childWait = childWait,
755	.childKill = childKill,
756
757	.progressParentFile = progressParentFile,
758
759	.now = now,
760	.clockResolution = clockResolution,
761	.sleep = sleep,
762
763	.random = random,
764	.randomSecure = randomSecure,
765
766	.netListenIp = netListenIpUnavailable,
767	.netAccept = netAcceptUnavailable,
768	.netBindIp = netBindIp,
769	.netConnectIp = netConnectIpUnavailable,
770	.netListenUnix = netListenUnixUnavailable,
771	.netConnectUnix = netConnectUnixUnavailable,
772	.netSocketCreatePair = netSocketCreatePairUnavailable,
773	.netSend = netSendUnavailable,
774	.netReceive = netReceive,
775	.netRead = netReadUnavailable,
776	.netWrite = netWriteUnavailable,
777	.netWriteFile = netWriteFileUnavailable,
778	.netClose = netClose,
779	.netShutdown = netShutdown,
780	.netInterfaceNameResolve = netInterfaceNameResolveUnavailable,
781	.netInterfaceName = netInterfaceNameUnavailable,
782	.netLookup = netLookupUnavailable,
783	},
784	};
785	}
786
787	fn fileMemoryMapSetLength(
788	userdata: ?*anyopaque,
789	mm: *File.MemoryMap,
790	new_len: usize,
791	) File.MemoryMap.SetLengthError!void {
792	const ev: *Evented = @ptrCast(@alignCast(userdata));
793
794	const page_size = std.heap.pageSize();
795	const alignment: Alignment = .fromByteUnits(page_size);
796	const page_align = std.heap.page_size_min;
797	const old_memory = mm.memory;
798
799	if (alignment.forward(new_len) == alignment.forward(old_memory.len)) {
800	mm.memory.len = new_len;
801	return;
802	}
803	const flags: linux.MREMAP = .{ .MAYMOVE = true };
804	const addr_hint: ?[*]const u8 = null;
805	var sync: CancelRegion.Sync = try .init(ev);
806	defer sync.deinit(ev);
807	const new_memory = while (true) {
808	try sync.cancel_region.await(.nothing);
809	const rc = linux.mremap(old_memory.ptr, old_memory.len, new_len, flags, addr_hint);
810	switch (linux.errno(rc)) {
811	.SUCCESS => break @as([*]align(page_align) u8, @ptrFromInt(rc))[0..new_len],
812	.INTR => continue,
813	.AGAIN => return error.LockedMemoryLimitExceeded,
814	.NOMEM => return error.OutOfMemory,
815	.INVAL => \|err\| return errnoBug(err),
816	.FAULT => \|err\| return errnoBug(err),
817	else => \|err\| return unexpectedErrno(err),
818	}
819	};
820	mm.memory = new_memory;
821	}
822
823	fn fileMemoryMapRead(userdata: ?anyopaque, mm: File.MemoryMap) File.ReadPositionalError!void {
824	const ev: *Evented = @ptrCast(@alignCast(userdata));
825	_ = ev;
826	_ = mm;
827	}
828
829	fn fileMemoryMapWrite(userdata: ?anyopaque, mm: File.MemoryMap) File.WritePositionalError!void {
830	const ev: *Evented = @ptrCast(@alignCast(userdata));
831	_ = ev;
832	_ = mm;
833	}
834
835	pub const InitOptions = struct {
836	backing_allocator_needs_mutex: bool = true,
837
838	/// Maximum thread pool size (excluding the main thread).
839	/// Defaults to one less than the number of logical CPU cores.
840	thread_limit: ?usize = null,
841	/// Maximum number of threads that may perform synchronous syscalls.
842	sync_limit: Io.Limit = .unlimited,
843
844	log2_ring_entries: u4 = 3,
845
846	/// Affects the following operations:
847	/// * `processExecutablePath` on OpenBSD and Haiku.
848	argv0: Argv0 = .empty,
849	/// Affects the following operations:
850	/// * `fileIsTty`
851	/// * `processSpawn`, `processSpawnPath`, `processReplace`, `processReplacePath`
852	environ: process.Environ = .empty,
853	};
854
855	pub fn init(ev: *Evented, backing_allocator: Allocator, options: InitOptions) !void {
856	const threads_size = @sizeOf(Thread) * if (options.thread_limit) \|thread_limit\|
857	1 + thread_limit
858	else
859	@max(std.Thread.getCpuCount() catch 1, 1);
860	const idle_stack_end_offset =
861	std.mem.alignForward(usize, threads_size + idle_stack_size, std.heap.page_size_max);
862	const allocated_slice = try backing_allocator.alignedAlloc(u8, .of(Thread), idle_stack_end_offset);
863	errdefer backing_allocator.free(allocated_slice);
864	ev.* = .{
865	.backing_allocator_needs_mutex = options.backing_allocator_needs_mutex,
866	.backing_allocator_mutex = .init,
867	.backing_allocator = backing_allocator,
868	.main_fiber_buffer = undefined,
869	.log2_ring_entries = options.log2_ring_entries,
870	.threads = .{
871	.allocated = @ptrCast(allocated_slice[0..threads_size]),
872	.reserved = 1,
873	.active = 1,
874	},
875	.sync_limit = if (options.sync_limit.toInt()) \|sync_limit\| .{ .permits = sync_limit } else null,
876
877	.stderr_mutex = .init,
878	.stderr_writer = .{
879	.io = ev.io(),
880	.interface = Io.File.Writer.initInterface(&.{}),
881	.file = .stderr(),
882	.mode = .streaming,
883	},
884	.stderr_mode = .no_color,
885	.stderr_writer_initialized = false,
886
887	.environ_mutex = .init,
888	.environ = .{ .process_environ = options.environ },
889
890	.null_fd = .init,
891	.random_fd = .init,
892
893	.csprng_mutex = .init,
894	.csprng = .uninitialized,
895	};
896	const main_fiber: *Fiber = @ptrCast(&ev.main_fiber_buffer);
897	main_fiber.* = .{
898	.required_align = {},
899	.context = undefined,
900	.await_count = 0,
901	.link = .{ .awaiter = null },
902	.status = .{ .queue_next = null },
903	.cancel_status = .unrequested,
904	.cancel_protection = .unblocked,
905	.name = if (tracy.enable) "main task",
906	};
907	const main_thread = &ev.threads.allocated[0];
908	Thread.self = main_thread;
909	const idle_stack_end: [*]align(16) usize =
910	@ptrCast(@alignCast(allocated_slice[idle_stack_end_offset..].ptr));
911	(idle_stack_end - 1)[0..1].* = .{@intFromPtr(ev)};
912	main_thread.* = .{
913	.required_align = {},
914	.thread = undefined,
915	.idle_context = switch (builtin.cpu.arch) {
916	.aarch64 => .{
917	.sp = @intFromPtr(idle_stack_end),
918	.fp = 0,
919	.pc = @intFromPtr(&mainIdleEntry),
920	},
921	.x86_64 => .{
922	.rsp = @intFromPtr(idle_stack_end - 1),
923	.rbp = 0,
924	.rip = @intFromPtr(&mainIdleEntry),
925	},
926	else => @compileError("unimplemented architecture"),
927	},
928	.current_context = &main_fiber.context,
929	.ready_queue = null,
930	.free_queue = null,
931	.io_uring = try .init(
932	@as(u16, 1) << ev.log2_ring_entries,
933	linux.IORING_SETUP_COOP_TASKRUN \| linux.IORING_SETUP_SINGLE_ISSUER,
934	),
935	.idle_search_index = 1,
936	.steal_ready_search_index = 1,
937	.steal_free_search_index = 1,
938	.name_arena = .{},
939	.csprng = .uninitialized,
940	};
941	errdefer main_thread.io_uring.deinit();
942	if (tracy.enable) tracy.fiberEnter(main_fiber.name);
943	}
944
945	pub fn deinit(ev: *Evented) void {
946	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
947	for (ev.threads.allocated[0..active_threads]) \|*thread\| {
948	const ready_fiber = @atomicLoad(?*Fiber, &thread.ready_queue, .monotonic);
949	assert(ready_fiber == null or ready_fiber == Fiber.finished); // pending async
950	}
951	ev.yield(null, .exit);
952	ev.threads.allocated[0].deinit(ev.allocator());
953	ev.null_fd.close();
954	ev.random_fd.close();
955	const allocated_ptr: [*]align(@alignOf(Thread)) u8 = @ptrCast(@alignCast(ev.threads.allocated.ptr));
956	const idle_stack_end_offset = std.mem.alignForward(
957	usize,
958	ev.threads.allocated.len * @sizeOf(Thread) + idle_stack_size,
959	std.heap.page_size_max,
960	);
961	for (ev.threads.allocated[1..active_threads]) \|*thread\| thread.thread.join();
962	assert(active_threads == ev.threads.active); // spawned threads while there was no pending async?
963	ev.backing_allocator.free(allocated_ptr[0..idle_stack_end_offset]);
964	ev.* = undefined;
965	}
966
967	fn findReadyFiber(ev: Evented, thread: Thread) ?*Fiber {
968	if (@atomicRmw(?*Fiber, &thread.ready_queue, .Xchg, Fiber.finished, .acquire)) \|ready_fiber\| {
969	assert(ready_fiber != Fiber.finished);
970	@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.status.queue_next, .release);
971	ready_fiber.status.queue_next = null;
972	return ready_fiber;
973	}
974	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
975	for (0..@min(max_steal_ready_search, active_threads)) \|_\| {
976	defer thread.steal_ready_search_index += 1;
977	if (thread.steal_ready_search_index == active_threads) thread.steal_ready_search_index = 0;
978	const steal_ready_search_thread =
979	&ev.threads.allocated[0..active_threads][thread.steal_ready_search_index];
980	if (steal_ready_search_thread == thread) continue;
981	const ready_fiber =
982	@atomicLoad(?*Fiber, &steal_ready_search_thread.ready_queue, .monotonic) orelse continue;
983	if (ready_fiber == Fiber.finished) continue;
984	if (@cmpxchgWeak(
985	?*Fiber,
986	&steal_ready_search_thread.ready_queue,
987	ready_fiber,
988	null,
989	.acquire,
990	.monotonic,
991	)) \|_\| continue;
992	@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.status.queue_next, .release);
993	ready_fiber.status.queue_next = null;
994	return ready_fiber;
995	}
996	// couldn't find anything to do, so we are now open for business
997	@atomicStore(?*Fiber, &thread.ready_queue, null, .monotonic);
998	return null;
999	}
1000
1001	fn yield(ev: Evented, maybe_ready_fiber: ?Fiber, pending_task: SwitchMessage.PendingTask) void {
1002	const thread: *Thread = .current();
1003	const ready_context = if (maybe_ready_fiber orelse ev.findReadyFiber(thread)) \|ready_fiber\|
1004	&ready_fiber.context
1005	else
1006	&thread.idle_context;
1007	const message: SwitchMessage = .{
1008	.contexts = .{
1009	.prev = thread.current_context,
1010	.ready = ready_context,
1011	},
1012	.pending_task = pending_task,
1013	};
1014	contextSwitch(&message).handle(ev);
1015	}
1016
1017	fn schedule(ev: Evented, thread: Thread, ready_queue: Fiber.Queue) bool {
1018	// shared fields of previous `Thread` must be initialized before later ones are marked as active
1019	const new_thread_index = @atomicLoad(u32, &ev.threads.active, .acquire);
1020	for (0..@min(max_idle_search, new_thread_index)) \|_\| {
1021	defer thread.idle_search_index += 1;
1022	if (thread.idle_search_index == new_thread_index) thread.idle_search_index = 0;
1023	const idle_search_thread = &ev.threads.allocated[0..new_thread_index][thread.idle_search_index];
1024	if (idle_search_thread == thread) continue;
1025	if (@cmpxchgWeak(
1026	?*Fiber,
1027	&idle_search_thread.ready_queue,
1028	null,
1029	ready_queue.head,
1030	.release,
1031	.monotonic,
1032	)) \|_\| continue;
1033	thread.enqueue().* = .{
1034	.opcode = .MSG_RING,
1035	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1036	.ioprio = 0,
1037	.fd = idle_search_thread.io_uring.fd,
1038	.off = @intFromEnum(Completion.UserData.wakeup),
1039	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
1040	.len = 0,
1041	.rw_flags = 0,
1042	.user_data = @intFromEnum(Completion.UserData.wakeup),
1043	.buf_index = 0,
1044	.personality = 0,
1045	.splice_fd_in = 0,
1046	.addr3 = 0,
1047	.resv = 0,
1048	};
1049	return true;
1050	}
1051	spawn_thread: {
1052	// previous failed reservations must have completed before retrying
1053	if (new_thread_index == ev.threads.allocated.len or @cmpxchgWeak(
1054	u32,
1055	&ev.threads.reserved,
1056	new_thread_index,
1057	new_thread_index + 1,
1058	.acquire,
1059	.monotonic,
1060	) != null) break :spawn_thread;
1061	const new_thread = &ev.threads.allocated[new_thread_index];
1062	const next_thread_index = new_thread_index + 1;
1063	var params = std.mem.zeroInit(linux.io_uring_params, .{
1064	.flags = linux.IORING_SETUP_ATTACH_WQ \|
1065	linux.IORING_SETUP_R_DISABLED \|
1066	linux.IORING_SETUP_COOP_TASKRUN \|
1067	linux.IORING_SETUP_SINGLE_ISSUER,
1068	.wq_fd = @as(u32, @intCast(ev.threads.allocated[0].io_uring.fd)),
1069	});
1070	new_thread.* = .{
1071	.required_align = {},
1072	.thread = undefined,
1073	.idle_context = undefined,
1074	.current_context = &new_thread.idle_context,
1075	.ready_queue = ready_queue.head,
1076	.free_queue = null,
1077	.io_uring = IoUring.init_params(@as(u16, 1) << ev.log2_ring_entries, &params) catch \|err\| {
1078	@atomicStore(u32, &ev.threads.reserved, new_thread_index, .release);
1079	// no more access to `thread` after giving up reservation
1080	log.warn("unable to create worker thread due to io_uring init failure: {s}", .{
1081	@errorName(err),
1082	});
1083	break :spawn_thread;
1084	},
1085	.idle_search_index = 0,
1086	.steal_ready_search_index = 0,
1087	.steal_free_search_index = 0,
1088	.name_arena = .{},
1089	.csprng = .uninitialized,
1090	};
1091	new_thread.thread = std.Thread.spawn(.{
1092	.stack_size = idle_stack_size,
1093	.allocator = ev.allocator(),
1094	}, threadEntry, .{ ev, new_thread_index }) catch \|err\| {
1095	new_thread.io_uring.deinit();
1096	@atomicStore(u32, &ev.threads.reserved, new_thread_index, .release);
1097	// no more access to `thread` after giving up reservation
1098	log.warn("unable to create worker thread due spawn failure: {s}", .{@errorName(err)});
1099	break :spawn_thread;
1100	};
1101	// shared fields of `Thread` must be initialized before being marked active
1102	@atomicStore(u32, &ev.threads.active, next_thread_index, .release);
1103	return false;
1104	}
1105	// nobody wanted it, so just queue it on ourselves
1106	while (true) ready_queue.tail.status.queue_next = @cmpxchgWeak(
1107	?*Fiber,
1108	&thread.ready_queue,
1109	ready_queue.tail.status.queue_next,
1110	ready_queue.head,
1111	.acq_rel,
1112	.acquire,
1113	) orelse break;
1114	return false;
1115	}
1116
1117	fn mainIdle(
1118	ev: *Evented,
1119	message: *const SwitchMessage,
1120	) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Context)))) noreturn {
1121	message.handle(ev);
1122	ev.idle(&ev.threads.allocated[0]);
1123	ev.yield(@ptrCast(&ev.main_fiber_buffer), .nothing);
1124	unreachable; // switched to dead fiber
1125	}
1126
1127	fn threadEntry(ev: *Evented, index: u32) void {
1128	const thread: *Thread = &ev.threads.allocated[index];
1129	Thread.self = thread;
1130	defer thread.deinit(ev.allocator());
1131	switch (linux.errno(linux.io_uring_register(thread.io_uring.fd, .REGISTER_ENABLE_RINGS, null, 0))) {
1132	.SUCCESS => ev.idle(thread),
1133	else => \|err\| @panic(@tagName(err)),
1134	}
1135	}
1136
1137	const Completion = struct {
1138	result: i32,
1139	flags: u32,
1140
1141	const UserData = enum(usize) {
1142	unused,
1143	wakeup,
1144	futex_wake,
1145	cleanup,
1146	exit,
1147	/// If bit 0 is 1, a pointer to the `context` field of `Io.Batch.Storage.Pending`.
1148	/// If bits 0 and 1 are 0, a `*Fiber`.
1149	_,
1150	};
1151
1152	fn errno(completion: Completion) linux.E {
1153	return linux.errno(@bitCast(@as(isize, completion.result)));
1154	}
1155	};
1156
1157	fn idle(ev: Evented, thread: Thread) void {
1158	var maybe_ready_fiber: ?*Fiber = null;
1159	while (true) {
1160	while (maybe_ready_fiber orelse ev.findReadyFiber(thread)) \|ready_fiber\| {
1161	ev.yield(ready_fiber, .nothing);
1162	maybe_ready_fiber = null;
1163	}
1164	_ = thread.io_uring.submit_and_wait(1) catch \|err\| switch (err) {
1165	error.SignalInterrupt => {},
1166	else => \|e\| @panic(@errorName(e)),
1167	};
1168	var maybe_ready_queue: ?Fiber.Queue = null;
1169	while (true) {
1170	var cqes_buffer: [1 << 8]linux.io_uring_cqe = undefined;
1171	const cqes = cqes_buffer[0 .. thread.io_uring.copy_cqes(&cqes_buffer, 0) catch \|err\| switch (err) {
1172	error.SignalInterrupt => 0,
1173	else => \|e\| @panic(@errorName(e)),
1174	}];
1175	if (cqes.len == 0) break;
1176	for (cqes) \|cqe\| if (cqe.flags & linux.IORING_CQE_F_SKIP == 0) switch (@as(
1177	Completion.UserData,
1178	@enumFromInt(cqe.user_data),
1179	)) {
1180	.unused => unreachable, // bad submission queued?
1181	.wakeup => {},
1182	.futex_wake => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
1183	.SUCCESS => recoverableOsBugDetected(), // success is skipped
1184	.INVAL => {}, // invalid futex_wait() on ptr done elsewhere
1185	.INTR, .CANCELED => recoverableOsBugDetected(), // `Completion.UserData.futex_wake` is not cancelable
1186	.FAULT => {}, // pointer became invalid while doing the wake
1187	else => recoverableOsBugDetected(), // deadlock due to operating system bug
1188	},
1189	.cleanup => @panic("failed to notify other threads that we are exiting"),
1190	.exit => {
1191	assert(maybe_ready_fiber == null and maybe_ready_queue == null); // pending async
1192	return;
1193	},
1194	_ => if (@as(?*Fiber, ready_fiber: switch (@as(u2, @truncate(cqe.user_data))) {
1195	0b00 => {
1196	const ready_fiber: *Fiber = @ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
1197	ready_fiber.resultPointer(Completion).* = .{
1198	.result = cqe.res,
1199	.flags = cqe.flags,
1200	};
1201	break :ready_fiber ready_fiber;
1202	},
1203	0b01 => {
1204	thread.enqueue().* = .{
1205	.opcode = .ASYNC_CANCEL,
1206	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1207	.ioprio = 0,
1208	.fd = 0,
1209	.off = 0,
1210	.addr = cqe.user_data & ~@as(usize, 0b11),
1211	.len = 0,
1212	.rw_flags = 0,
1213	.user_data = @intFromEnum(Completion.UserData.wakeup),
1214	.buf_index = 0,
1215	.personality = 0,
1216	.splice_fd_in = 0,
1217	.addr3 = 0,
1218	.resv = 0,
1219	};
1220	break :ready_fiber null;
1221	},
1222	0b10 => {
1223	const context: *Io.Operation.Storage.Pending.Context =
1224	@ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
1225	const batch: *Io.Batch = @ptrFromInt(context[0]);
1226	var next: usize = 0b00;
1227	context[0..3].* = .{ next, @as(u32, @bitCast(cqe.res)), cqe.flags };
1228	while (true) {
1229	next = @cmpxchgWeak(
1230	usize,
1231	@as(*usize, @ptrCast(&batch.context)),
1232	next,
1233	cqe.user_data,
1234	.release,
1235	.acquire,
1236	) orelse break;
1237	context[0] = next;
1238	}
1239	break :ready_fiber switch (@as(u2, @truncate(next))) {
1240	0b00, 0b01 => @ptrFromInt(next & ~@as(usize, 0b11)),
1241	0b10, 0b11 => null,
1242	};
1243	},
1244	0b11 => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
1245	.SUCCESS => unreachable, // no event count specified
1246	.TIME => {
1247	const context: *usize = @ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
1248	const fiber = @atomicRmw(usize, context, .Add, 0b01, .acquire);
1249	break :ready_fiber switch (@as(u2, @truncate(fiber))) {
1250	else => unreachable, // timeout completed multiple times
1251	0b00 => @ptrFromInt(fiber & ~@as(usize, 0b11)),
1252	0b10 => null,
1253	};
1254	},
1255	.CANCELED => null, // user data may have been invalidated
1256	else => \|err\| unexpectedErrno(err) catch null,
1257	},
1258	})) \|ready_fiber\| {
1259	assert(ready_fiber.status.queue_next == null);
1260	if (maybe_ready_fiber == null) {
1261	maybe_ready_fiber = ready_fiber;
1262	} else if (maybe_ready_queue) \|*ready_queue\| {
1263	ready_queue.tail.status.queue_next = ready_fiber;
1264	ready_queue.tail = ready_fiber;
1265	} else maybe_ready_queue = .{ .head = ready_fiber, .tail = ready_fiber };
1266	},
1267	};
1268	}
1269	if (maybe_ready_queue) \|ready_queue\| _ = ev.schedule(thread, ready_queue);
1270	}
1271	}
1272
1273	const SwitchMessage = struct {
1274	contexts: extern struct {
1275	prev: *Context,
1276	ready: *Context,
1277	},
1278	pending_task: PendingTask,
1279
1280	const PendingTask = union(enum) {
1281	nothing,
1282	reschedule,
1283	await: u31,
1284	group_await: Group,
1285	group_cancel: Group,
1286	batch_await: *Io.Batch,
1287	destroy,
1288	exit,
1289	};
1290
1291	fn handle(message: const SwitchMessage, ev: Evented) void {
1292	const thread: *Thread = .current();
1293	thread.current_context = message.contexts.ready;
1294	if (tracy.enable) {
1295	if (message.contexts.ready != &thread.idle_context) {
1296	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.ready));
1297	tracy.fiberEnter(fiber.name);
1298	} else tracy.fiberLeave();
1299	}
1300	switch (message.pending_task) {
1301	.nothing => {},
1302	.reschedule => if (message.contexts.prev != &thread.idle_context) {
1303	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
1304	assert(fiber.status.queue_next == null);
1305	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1306	},
1307	.await => \|count\| {
1308	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
1309	if (@atomicRmw(i32, &fiber.await_count, .Sub, count, .monotonic) > 0)
1310	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1311	},
1312	.group_await => \|group\| {
1313	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
1314	if (group.await(ev, fiber))
1315	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1316	},
1317	.group_cancel => \|group\| {
1318	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
1319	if (group.cancel(ev, fiber))
1320	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1321	},
1322	.batch_await => \|batch\| {
1323	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
1324	if (@cmpxchgStrong(
1325	?*anyopaque,
1326	&batch.context,
1327	null,
1328	fiber,
1329	.release,
1330	.monotonic,
1331	)) \|head\| {
1332	assert(@as(u2, @truncate(@intFromPtr(head))) != 0b00);
1333	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1334	}
1335	},
1336	.destroy => {
1337	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
1338	fiber.destroy();
1339	},
1340	.exit => for (
1341	ev.threads.allocated[0..@atomicLoad(u32, &ev.threads.active, .acquire)],
1342	) \|*each_thread\| {
1343	thread.enqueue().* = .{
1344	.opcode = .MSG_RING,
1345	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1346	.ioprio = 0,
1347	.fd = each_thread.io_uring.fd,
1348	.off = @intFromEnum(Completion.UserData.exit),
1349	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
1350	.len = 0,
1351	.rw_flags = 0,
1352	.user_data = @intFromEnum(Completion.UserData.cleanup),
1353	.buf_index = 0,
1354	.personality = 0,
1355	.splice_fd_in = 0,
1356	.addr3 = 0,
1357	.resv = 0,
1358	};
1359	},
1360	}
1361	}
1362	};
1363
1364	const Context = switch (builtin.cpu.arch) {
1365	.aarch64 => extern struct {
1366	sp: u64,
1367	fp: u64,
1368	pc: u64,
1369	},
1370	.x86_64 => extern struct {
1371	rsp: u64,
1372	rbp: u64,
1373	rip: u64,
1374	},
1375	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1376	};
1377
1378	inline fn contextSwitch(message: const SwitchMessage) const SwitchMessage {
1379	return @fieldParentPtr("contexts", switch (builtin.cpu.arch) {
1380	.aarch64 => asm volatile (
1381	\\ ldp x0, x2, [x1]
1382	\\ ldr x3, [x2, #16]
1383	\\ mov x4, sp
1384	\\ stp x4, fp, [x0]
1385	\\ adr x5, 0f
1386	\\ ldp x4, fp, [x2]
1387	\\ str x5, [x0, #16]
1388	\\ mov sp, x4
1389	\\ br x3
1390	\\0:
1391	: [received_message] "={x1}" (-> *const @FieldType(SwitchMessage, "contexts")),
1392	: [message_to_send] "{x1}" (&message.contexts),
1393	: .{
1394	.x0 = true,
1395	.x1 = true,
1396	.x2 = true,
1397	.x3 = true,
1398	.x4 = true,
1399	.x5 = true,
1400	.x6 = true,
1401	.x7 = true,
1402	.x8 = true,
1403	.x9 = true,
1404	.x10 = true,
1405	.x11 = true,
1406	.x12 = true,
1407	.x13 = true,
1408	.x14 = true,
1409	.x15 = true,
1410	.x16 = true,
1411	.x17 = true,
1412	.x19 = true,
1413	.x20 = true,
1414	.x21 = true,
1415	.x22 = true,
1416	.x23 = true,
1417	.x24 = true,
1418	.x25 = true,
1419	.x26 = true,
1420	.x27 = true,
1421	.x28 = true,
1422	.x30 = true,
1423	.z0 = true,
1424	.z1 = true,
1425	.z2 = true,
1426	.z3 = true,
1427	.z4 = true,
1428	.z5 = true,
1429	.z6 = true,
1430	.z7 = true,
1431	.z8 = true,
1432	.z9 = true,
1433	.z10 = true,
1434	.z11 = true,
1435	.z12 = true,
1436	.z13 = true,
1437	.z14 = true,
1438	.z15 = true,
1439	.z16 = true,
1440	.z17 = true,
1441	.z18 = true,
1442	.z19 = true,
1443	.z20 = true,
1444	.z21 = true,
1445	.z22 = true,
1446	.z23 = true,
1447	.z24 = true,
1448	.z25 = true,
1449	.z26 = true,
1450	.z27 = true,
1451	.z28 = true,
1452	.z29 = true,
1453	.z30 = true,
1454	.z31 = true,
1455	.p0 = true,
1456	.p1 = true,
1457	.p2 = true,
1458	.p3 = true,
1459	.p4 = true,
1460	.p5 = true,
1461	.p6 = true,
1462	.p7 = true,
1463	.p8 = true,
1464	.p9 = true,
1465	.p10 = true,
1466	.p11 = true,
1467	.p12 = true,
1468	.p13 = true,
1469	.p14 = true,
1470	.p15 = true,
1471	.fpcr = true,
1472	.fpsr = true,
1473	.ffr = true,
1474	.memory = true,
1475	}),
1476	.x86_64 => asm volatile (
1477	\\ movq 0(%%rsi), %%rax
1478	\\ movq 8(%%rsi), %%rcx
1479	\\ leaq 0f(%%rip), %%rdx
1480	\\ movq %%rsp, 0(%%rax)
1481	\\ movq %%rbp, 8(%%rax)
1482	\\ movq %%rdx, 16(%%rax)
1483	\\ movq 0(%%rcx), %%rsp
1484	\\ movq 8(%%rcx), %%rbp
1485	\\ jmpq *16(%%rcx)
1486	\\0:
1487	: [received_message] "={rsi}" (-> *const @FieldType(SwitchMessage, "contexts")),
1488	: [message_to_send] "{rsi}" (&message.contexts),
1489	: .{
1490	.rax = true,
1491	.rcx = true,
1492	.rdx = true,
1493	.rbx = true,
1494	.rsi = true,
1495	.rdi = true,
1496	.r8 = true,
1497	.r9 = true,
1498	.r10 = true,
1499	.r11 = true,
1500	.r12 = true,
1501	.r13 = true,
1502	.r14 = true,
1503	.r15 = true,
1504	.mm0 = true,
1505	.mm1 = true,
1506	.mm2 = true,
1507	.mm3 = true,
1508	.mm4 = true,
1509	.mm5 = true,
1510	.mm6 = true,
1511	.mm7 = true,
1512	.zmm0 = true,
1513	.zmm1 = true,
1514	.zmm2 = true,
1515	.zmm3 = true,
1516	.zmm4 = true,
1517	.zmm5 = true,
1518	.zmm6 = true,
1519	.zmm7 = true,
1520	.zmm8 = true,
1521	.zmm9 = true,
1522	.zmm10 = true,
1523	.zmm11 = true,
1524	.zmm12 = true,
1525	.zmm13 = true,
1526	.zmm14 = true,
1527	.zmm15 = true,
1528	.zmm16 = true,
1529	.zmm17 = true,
1530	.zmm18 = true,
1531	.zmm19 = true,
1532	.zmm20 = true,
1533	.zmm21 = true,
1534	.zmm22 = true,
1535	.zmm23 = true,
1536	.zmm24 = true,
1537	.zmm25 = true,
1538	.zmm26 = true,
1539	.zmm27 = true,
1540	.zmm28 = true,
1541	.zmm29 = true,
1542	.zmm30 = true,
1543	.zmm31 = true,
1544	.fpsr = true,
1545	.fpcr = true,
1546	.mxcsr = true,
1547	.rflags = true,
1548	.dirflag = true,
1549	.memory = true,
1550	}),
1551	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1552	});
1553	}
1554
1555	fn mainIdleEntry() callconv(.naked) void {
1556	switch (builtin.cpu.arch) {
1557	.aarch64 => asm volatile (
1558	\\ ldr x0, [sp, #-8]
1559	\\ b %[mainIdle]
1560	:
1561	: [mainIdle] "X" (&mainIdle),
1562	),
1563	.x86_64 => asm volatile (
1564	\\ movq (%%rsp), %%rdi
1565	\\ jmp %[mainIdle:P]
1566	:
1567	: [mainIdle] "X" (&mainIdle),
1568	),
1569	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1570	}
1571	}
1572
1573	const AsyncClosure = struct {
1574	ev: *Evented,
1575	fiber: *Fiber,
1576	start: const fn (context: const anyopaque, result: *anyopaque) void,
1577	result_align: Alignment,
1578
1579	fn fromFiber(fiber: Fiber) AsyncClosure {
1580	return @ptrFromInt(Fiber.max_context_align.max(.of(AsyncClosure)).backward(
1581	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
1582	) - @sizeOf(AsyncClosure));
1583	}
1584
1585	fn contextPointer(closure: AsyncClosure) []align(Fiber.max_context_align.toByteUnits()) u8 {
1586	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(AsyncClosure));
1587	}
1588
1589	fn entry() callconv(.naked) void {
1590	switch (builtin.cpu.arch) {
1591	.aarch64 => asm volatile (
1592	\\ mov x0, sp
1593	\\ b %[call]
1594	:
1595	: [call] "X" (&call),
1596	),
1597	.x86_64 => asm volatile (
1598	\\ leaq 8(%%rsp), %%rdi
1599	\\ jmp %[call:P]
1600	:
1601	: [call] "X" (&call),
1602	),
1603	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1604	}
1605	}
1606
1607	fn call(
1608	closure: *AsyncClosure,
1609	message: *const SwitchMessage,
1610	) callconv(.withStackAlign(.c, @alignOf(AsyncClosure))) noreturn {
1611	message.handle(closure.ev);
1612	const fiber = closure.fiber;
1613	closure.start(closure.contextPointer(), fiber.resultBytes(closure.result_align));
1614	closure.ev.yield(
1615	if (@atomicRmw(?*Fiber, &fiber.link.awaiter, .Xchg, Fiber.finished, .acq_rel)) \|awaiter\|
1616	if (@atomicRmw(i32, &awaiter.await_count, .Add, 1, .monotonic) == -1) awaiter else null
1617	else
1618	null,
1619	.nothing,
1620	);
1621	unreachable; // switched to dead fiber
1622	}
1623	};
1624
1625	fn async(
1626	userdata: ?*anyopaque,
1627	result: []u8,
1628	result_alignment: Alignment,
1629	context: []const u8,
1630	context_alignment: Alignment,
1631	start: const fn (context: const anyopaque, result: *anyopaque) void,
1632	) ?*std.Io.AnyFuture {
1633	const ev: *Evented = @ptrCast(@alignCast(userdata));
1634	return concurrent(ev, result.len, result_alignment, context, context_alignment, start) catch {
1635	start(context.ptr, result.ptr);
1636	return null;
1637	};
1638	}
1639
1640	fn concurrent(
1641	userdata: ?*anyopaque,
1642	result_len: usize,
1643	result_alignment: Alignment,
1644	context: []const u8,
1645	context_alignment: Alignment,
1646	start: const fn (context: const anyopaque, result: *anyopaque) void,
1647	) Io.ConcurrentError!*std.Io.AnyFuture {
1648	assert(result_alignment.compare(.lte, Fiber.max_result_align)); // TODO
1649	assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
1650	assert(result_len <= Fiber.max_result_size); // TODO
1651	assert(context.len <= Fiber.max_context_size); // TODO
1652
1653	const ev: *Evented = @ptrCast(@alignCast(userdata));
1654	const fiber = Fiber.create(ev) catch \|err\| switch (err) {
1655	error.OutOfMemory => return error.ConcurrencyUnavailable,
1656	};
1657
1658	const closure: *AsyncClosure = .fromFiber(fiber);
1659	fiber.* = .{
1660	.required_align = {},
1661	.context = switch (builtin.cpu.arch) {
1662	.aarch64 => .{
1663	.sp = @intFromPtr(closure),
1664	.fp = 0,
1665	.pc = @intFromPtr(&AsyncClosure.entry),
1666	},
1667	.x86_64 => .{
1668	.rsp = @intFromPtr(closure) - @sizeOf(usize),
1669	.rbp = 0,
1670	.rip = @intFromPtr(&AsyncClosure.entry),
1671	},
1672	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1673	},
1674	.await_count = 0,
1675	.link = .{ .awaiter = null },
1676	.status = .{ .queue_next = null },
1677	.cancel_status = .unrequested,
1678	.cancel_protection = .unblocked,
1679	.name = if (tracy.enable) name: {
1680	const thread: *Thread = .current();
1681	var name_arena = thread.name_arena.promote(std.heap.page_allocator);
1682	defer thread.name_arena = name_arena.state;
1683	break :name std.fmt.allocPrintSentinel(
1684	name_arena.allocator(),
1685	"task {d}",
1686	.{@atomicRmw(u64, &Fiber.next_name, .Add, 1, .monotonic)},
1687	0,
1688	) catch return error.ConcurrencyUnavailable;
1689	},
1690	};
1691	closure.* = .{
1692	.ev = ev,
1693	.fiber = fiber,
1694	.start = start,
1695	.result_align = result_alignment,
1696	};
1697	@memcpy(closure.contextPointer(), context);
1698
1699	const thread: *Thread = .current();
1700	if (ev.schedule(thread, .{ .head = fiber, .tail = fiber })) thread.submit();
1701	return @ptrCast(fiber);
1702	}
1703
1704	fn await(
1705	userdata: ?*anyopaque,
1706	future: *std.Io.AnyFuture,
1707	result: []u8,
1708	result_alignment: Alignment,
1709	) void {
1710	const ev: *Evented = @ptrCast(@alignCast(userdata));
1711	const fiber = Thread.current().currentFiber();
1712	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
1713	if (@atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, fiber, .acq_rel)) \|awaiter\| {
1714	assert(awaiter == Fiber.finished);
1715	} else while (true) {
1716	ev.yield(null, .{ .await = 1 });
1717	const awaiter = @atomicLoad(?*Fiber, &future_fiber.link.awaiter, .acquire);
1718	if (awaiter == Fiber.finished) break;
1719	assert(awaiter == fiber); // spurious wakeup
1720	}
1721	@memcpy(result, future_fiber.resultBytes(result_alignment));
1722	future_fiber.destroy();
1723	}
1724
1725	fn cancel(
1726	userdata: ?*anyopaque,
1727	future: *std.Io.AnyFuture,
1728	result: []u8,
1729	result_alignment: Alignment,
1730	) void {
1731	const ev: *Evented = @ptrCast(@alignCast(userdata));
1732	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
1733	future_fiber.requestCancel(ev);
1734	await(ev, future, result, result_alignment);
1735	}
1736
1737	const Group = struct {
1738	ptr: *Io.Group,
1739
1740	const List = packed struct(usize) {
1741	cancel_requested: bool,
1742	awaiter_delayed: bool,
1743	fibers: Fiber.PackedPtr,
1744	};
1745	fn listPtr(group: Group) *List {
1746	return @ptrCast(&group.ptr.token);
1747	}
1748
1749	const Mutex = packed struct(u32) {
1750	locked: bool,
1751	contended: bool,
1752	shared2: u30,
1753	};
1754	fn mutexPtr(group: Group) *Mutex {
1755	return switch (comptime builtin.cpu.arch.endian()) {
1756	.little => @ptrCast(&group.ptr.state),
1757	.big => @ptrCast(@alignCast(
1758	@as([*]u8, @ptrCast(&group.ptr.state)) + @sizeOf(usize) - @sizeOf(u32),
1759	)),
1760	};
1761	}
1762
1763	const Awaiter = packed struct(usize) {
1764	locked: bool,
1765	contended: bool,
1766	awaiter: Fiber.PackedPtr,
1767	};
1768	fn awaiterPtr(group: Group) *Awaiter {
1769	return @ptrCast(&group.ptr.state);
1770	}
1771
1772	fn lock(group: Group, ev: *Evented) void {
1773	const mutex = group.mutexPtr();
1774	{
1775	const old_state = @atomicRmw(
1776	Mutex,
1777	mutex,
1778	.Or,
1779	.{ .locked = true, .contended = false, .shared2 = 0 },
1780	.acquire,
1781	);
1782	if (!old_state.locked) {
1783	@branchHint(.likely);
1784	return;
1785	}
1786	if (old_state.contended) {
1787	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
1788	}
1789	}
1790	while (true) {
1791	var old_state = @atomicRmw(
1792	Mutex,
1793	mutex,
1794	.Or,
1795	.{ .locked = true, .contended = true, .shared2 = 0 },
1796	.acquire,
1797	);
1798	if (!old_state.locked) {
1799	@branchHint(.likely);
1800	return;
1801	}
1802	old_state.contended = true;
1803	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
1804	}
1805	}
1806
1807	fn unlock(group: Group, ev: *Evented) void {
1808	const mutex = group.mutexPtr();
1809	const old_state = @atomicRmw(
1810	Mutex,
1811	mutex,
1812	.And,
1813	.{ .locked = false, .contended = false, .shared2 = std.math.maxInt(u30) },
1814	.release,
1815	);
1816	assert(old_state.locked);
1817	if (old_state.contended) futexWake(ev, @ptrCast(mutex), 1);
1818	}
1819
1820	fn addFiber(group: Group, ev: Evented, fiber: Fiber) void {
1821	group.lock(ev);
1822	defer group.unlock(ev);
1823	const list_ptr = group.listPtr();
1824	const list = @atomicLoad(List, list_ptr, .monotonic);
1825	if (list.cancel_requested) fiber.cancel_status = .{ .requested = true, .awaiting = .nothing };
1826	const old_head = list.fibers.unpack();
1827	if (old_head) \|head\| head.link.group.prev = fiber;
1828	fiber.link.group.next = old_head;
1829	@atomicStore(List, list_ptr, .{
1830	.cancel_requested = list.cancel_requested,
1831	.awaiter_delayed = list.awaiter_delayed,
1832	.fibers = .pack(fiber),
1833	}, .monotonic);
1834	}
1835
1836	fn removeFiber(group: Group, ev: Evented, fiber: Fiber) ?*Fiber {
1837	group.lock(ev);
1838	defer group.unlock(ev);
1839	const list_ptr = group.listPtr();
1840	const list = @atomicLoad(List, list_ptr, .monotonic);
1841	if (fiber.link.group.next) \|next\| next.link.group.prev = fiber.link.group.prev;
1842	if (fiber.link.group.prev) \|prev\| {
1843	prev.link.group.next = fiber.link.group.next;
1844	} else if (fiber.link.group.next) \|new_head\| {
1845	@atomicStore(List, list_ptr, .{
1846	.cancel_requested = list.cancel_requested,
1847	.awaiter_delayed = list.awaiter_delayed,
1848	.fibers = .pack(new_head),
1849	}, .monotonic);
1850	} else if (@atomicLoad(Awaiter, group.awaiterPtr(), .monotonic).awaiter.unpack()) \|awaiter\| {
1851	if (!awaiter.cancel_status.changeAwaiting(.group, .nothing) or list.cancel_requested) {
1852	@atomicStore(List, list_ptr, .{
1853	.cancel_requested = false,
1854	.awaiter_delayed = false,
1855	.fibers = .null,
1856	}, .release);
1857	assert(awaiter.status.awaiting_group.ptr == group.ptr);
1858	awaiter.status = .{ .queue_next = null };
1859	return awaiter;
1860	}
1861	// Race with `Fiber.requestCancel`
1862	@atomicStore(List, list_ptr, .{
1863	.cancel_requested = false,
1864	.awaiter_delayed = true,
1865	.fibers = .null,
1866	}, .monotonic);
1867	} else @atomicStore(List, list_ptr, .{
1868	.cancel_requested = false,
1869	.awaiter_delayed = false,
1870	.fibers = .null,
1871	}, .release);
1872	return null;
1873	}
1874
1875	fn await(group: Group, ev: Evented, awaiter: Fiber) bool {
1876	group.lock(ev);
1877	defer group.unlock(ev);
1878	if (@atomicLoad(List, group.listPtr(), .monotonic).fibers.unpack()) \|_\| {
1879	if (group.registerAwaiter(awaiter) and awaiter.cancel_protection.check() == .unblocked) {
1880	// The awaiter already had an unacknowledged cancelation request before
1881	// attempting to await a group, so propagate the cancelation to the group.
1882	assert(!group.cancelLocked(ev, null));
1883	}
1884	return false;
1885	}
1886	return true;
1887	}
1888
1889	fn cancel(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
1890	group.lock(ev);
1891	defer group.unlock(ev);
1892	return group.cancelLocked(ev, maybe_awaiter);
1893	}
1894
1895	/// Assumes the mutex is held.
1896	fn cancelLocked(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
1897	const list_ptr = group.listPtr();
1898	const list = @atomicRmw(
1899	List,
1900	list_ptr,
1901	.Add,
1902	.{ .cancel_requested = true, .awaiter_delayed = false, .fibers = .null },
1903	.monotonic,
1904	);
1905	assert(!list.cancel_requested);
1906	if (list.fibers.unpack()) \|head\| {
1907	var maybe_fiber: ?*Fiber = head;
1908	while (maybe_fiber) \|fiber\| {
1909	fiber.requestCancel(ev);
1910	maybe_fiber = fiber.link.group.next;
1911	}
1912	if (maybe_awaiter) \|awaiter\| _ = group.registerAwaiter(awaiter);
1913	return false;
1914	}
1915	@atomicStore(
1916	List,
1917	list_ptr,
1918	.{ .cancel_requested = false, .awaiter_delayed = false, .fibers = .null },
1919	.release,
1920	);
1921	return if (maybe_awaiter) \|_\| true else list.awaiter_delayed;
1922	}
1923
1924	/// Assumes the mutex is held.
1925	fn registerAwaiter(group: Group, awaiter: *Fiber) bool {
1926	assert(awaiter.status.queue_next == null);
1927	awaiter.status = .{ .awaiting_group = group };
1928	assert(@atomicRmw(
1929	Awaiter,
1930	group.awaiterPtr(),
1931	.Add,
1932	.{ .locked = false, .contended = false, .awaiter = .pack(awaiter) },
1933	.monotonic,
1934	).awaiter == .null);
1935	return awaiter.cancel_status.changeAwaiting(.nothing, .group);
1936	}
1937
1938	const AsyncClosure = struct {
1939	ev: *Evented,
1940	group: Group,
1941	fiber: *Fiber,
1942	start: const fn (context: const anyopaque) Io.Cancelable!void,
1943
1944	fn fromFiber(fiber: Fiber) Group.AsyncClosure {
1945	return @ptrFromInt(Fiber.max_context_align.max(.of(Group.AsyncClosure)).backward(
1946	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
1947	) - @sizeOf(Group.AsyncClosure));
1948	}
1949
1950	fn contextPointer(
1951	closure: *Group.AsyncClosure,
1952	) [*]align(Fiber.max_context_align.toByteUnits()) u8 {
1953	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(Group.AsyncClosure));
1954	}
1955
1956	fn entry() callconv(.naked) void {
1957	switch (builtin.cpu.arch) {
1958	.aarch64 => asm volatile (
1959	\\ mov x0, sp
1960	\\ b %[call]
1961	:
1962	: [call] "X" (&call),
1963	),
1964	.x86_64 => asm volatile (
1965	\\ leaq 8(%%rsp), %%rdi
1966	\\ jmp %[call:P]
1967	:
1968	: [call] "X" (&call),
1969	),
1970	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1971	}
1972	}
1973
1974	fn call(
1975	closure: *Group.AsyncClosure,
1976	message: *const SwitchMessage,
1977	) callconv(.withStackAlign(.c, @alignOf(Group.AsyncClosure))) noreturn {
1978	message.handle(closure.ev);
1979	assert(closure.fiber.status.queue_next == null);
1980	const result = closure.start(closure.contextPointer());
1981	const ev = closure.ev;
1982	const group = closure.group;
1983	const fiber = closure.fiber;
1984	const cancel_acknowledged = fiber.cancel_protection.acknowledged;
1985	if (result) {
1986	assert(!cancel_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
1987	} else \|err\| switch (err) {
1988	error.Canceled => assert(cancel_acknowledged), // group task returned `error.Canceled` but was never canceled
1989	}
1990	ev.yield(group.removeFiber(ev, fiber), .destroy);
1991	unreachable; // switched to dead fiber
1992	}
1993	};
1994	};
1995
1996	fn groupAsync(
1997	userdata: ?*anyopaque,
1998	type_erased: *Io.Group,
1999	context: []const u8,
2000	context_alignment: Alignment,
2001	start: const fn (context: const anyopaque) Io.Cancelable!void,
2002	) void {
2003	const ev: *Evented = @ptrCast(@alignCast(userdata));
2004	return groupConcurrent(ev, type_erased, context, context_alignment, start) catch {
2005	const fiber = Thread.current().currentFiber();
2006	const pre_acknowledged = fiber.cancel_protection.acknowledged;
2007	const result = start(context.ptr);
2008	const post_acknowledged = fiber.cancel_protection.acknowledged;
2009	if (result) {
2010	if (pre_acknowledged) {
2011	assert(post_acknowledged); // group task called `recancel` but was not canceled
2012	} else {
2013	assert(!post_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
2014	}
2015	} else \|err\| switch (err) {
2016	// Don't swallow the cancelation: make it visible to the `Group.async` caller.
2017	error.Canceled => {
2018	assert(!pre_acknowledged); // group task called `recancel` but was not canceled
2019	assert(post_acknowledged); // group task returned `error.Canceled` but was never canceled
2020	recancel(userdata);
2021	},
2022	}
2023	};
2024	}
2025
2026	fn groupConcurrent(
2027	userdata: ?*anyopaque,
2028	type_erased: *Io.Group,
2029	context: []const u8,
2030	context_alignment: Alignment,
2031	start: const fn (context: const anyopaque) Io.Cancelable!void,
2032	) Io.ConcurrentError!void {
2033	assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
2034	assert(context.len <= Fiber.max_context_size); // TODO
2035
2036	const ev: *Evented = @ptrCast(@alignCast(userdata));
2037	const group: Group = .{ .ptr = type_erased };
2038	const fiber = Fiber.create(ev) catch \|err\| switch (err) {
2039	error.OutOfMemory => return error.ConcurrencyUnavailable,
2040	};
2041
2042	const closure: *Group.AsyncClosure = .fromFiber(fiber);
2043	fiber.* = .{
2044	.required_align = {},
2045	.context = switch (builtin.cpu.arch) {
2046	.aarch64 => .{
2047	.sp = @intFromPtr(closure),
2048	.fp = 0,
2049	.pc = @intFromPtr(&Group.AsyncClosure.entry),
2050	},
2051	.x86_64 => .{
2052	.rsp = @intFromPtr(closure) - @sizeOf(usize),
2053	.rbp = 0,
2054	.rip = @intFromPtr(&Group.AsyncClosure.entry),
2055	},
2056	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
2057	},
2058	.await_count = 0,
2059	.link = .{ .group = .{ .prev = null, .next = null } },
2060	.status = .{ .queue_next = null },
2061	.cancel_status = .unrequested,
2062	.cancel_protection = .unblocked,
2063	.name = if (tracy.enable) name: {
2064	const thread: *Thread = .current();
2065	var name_arena = thread.name_arena.promote(std.heap.page_allocator);
2066	defer thread.name_arena = name_arena.state;
2067	break :name std.fmt.allocPrintSentinel(
2068	name_arena.allocator(),
2069	"group task {d}",
2070	.{@atomicRmw(u64, &Fiber.next_name, .Add, 1, .monotonic)},
2071	0,
2072	) catch return error.ConcurrencyUnavailable;
2073	},
2074	};
2075	closure.* = .{
2076	.ev = ev,
2077	.group = group,
2078	.fiber = fiber,
2079	.start = start,
2080	};
2081	@memcpy(closure.contextPointer(), context);
2082	group.addFiber(ev, fiber);
2083	const thread: *Thread = .current();
2084	if (ev.schedule(thread, .{ .head = fiber, .tail = fiber })) thread.submit();
2085	}
2086
2087	fn groupAwait(
2088	userdata: ?*anyopaque,
2089	type_erased: *Io.Group,
2090	initial_token: *anyopaque,
2091	) Io.Cancelable!void {
2092	const ev: *Evented = @ptrCast(@alignCast(userdata));
2093	_ = initial_token;
2094	ev.yield(null, .{ .group_await = .{ .ptr = type_erased } });
2095	}
2096
2097	fn groupCancel(userdata: ?anyopaque, type_erased: Io.Group, initial_token: *anyopaque) void {
2098	const ev: *Evented = @ptrCast(@alignCast(userdata));
2099	_ = initial_token;
2100	ev.yield(null, .{ .group_cancel = .{ .ptr = type_erased } });
2101	}
2102
2103	fn recancel(userdata: ?*anyopaque) void {
2104	const ev: *Evented = @ptrCast(@alignCast(userdata));
2105	_ = ev;
2106	const cancel_protection = &Thread.current().currentFiber().cancel_protection;
2107	assert(cancel_protection.acknowledged);
2108	cancel_protection.acknowledged = false;
2109	}
2110
2111	fn swapCancelProtection(userdata: ?*anyopaque, new: Io.CancelProtection) Io.CancelProtection {
2112	const ev: *Evented = @ptrCast(@alignCast(userdata));
2113	_ = ev;
2114	const cancel_protection = &Thread.current().currentFiber().cancel_protection;
2115	defer cancel_protection.user = new;
2116	return cancel_protection.user;
2117	}
2118
2119	fn checkCancel(userdata: ?*anyopaque) Io.Cancelable!void {
2120	const ev: *Evented = @ptrCast(@alignCast(userdata));
2121	_ = ev;
2122	const fiber = Thread.current().currentFiber();
2123	switch (fiber.cancel_protection.check()) {
2124	.blocked => {},
2125	.unblocked => if (@atomicLoad(Fiber.CancelStatus, &fiber.cancel_status, .monotonic).requested) {
2126	fiber.cancel_protection.acknowledge();
2127	return error.Canceled;
2128	},
2129	}
2130	}
2131
2132	fn select(userdata: ?anyopaque, futures: []const Io.AnyFuture) Io.Cancelable!usize {
2133	const ev: *Evented = @ptrCast(@alignCast(userdata));
2134	var cancel_region: CancelRegion = .init();
2135	defer cancel_region.deinit();
2136	var await_count: u31, var result = for (futures, 0..) \|future, future_index\| {
2137	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
2138	if (@atomicRmw(
2139	?*Fiber,
2140	&future_fiber.link.awaiter,
2141	.Xchg,
2142	cancel_region.fiber,
2143	.acq_rel,
2144	)) \|awaiter\| {
2145	assert(awaiter == Fiber.finished);
2146	break .{ @intCast(future_index), future_index };
2147	}
2148	} else result: {
2149	const await_count: u31 = @intCast(futures.len);
2150	cancel_region.await(.select) catch \|err\| switch (err) {
2151	error.Canceled => \|e\| break :result .{ await_count + 1, e },
2152	};
2153	ev.yield(null, .{ .await = 1 });
2154	cancel_region.await(.nothing) catch \|err\| switch (err) {
2155	error.Canceled => \|e\| break :result .{ await_count, e },
2156	};
2157	break :result .{ await_count - 1, futures.len };
2158	};
2159	for (futures[0 .. result catch futures.len], 0..) \|future, future_index\| {
2160	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
2161	const awaiter = @atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, null, .monotonic);
2162	if (awaiter == Fiber.finished) {
2163	@atomicStore(?*Fiber, &future_fiber.link.awaiter, Fiber.finished, .monotonic);
2164	result = if (result) \|finished_index\| @min(future_index, finished_index) else \|e\| e;
2165	} else {
2166	assert(awaiter == cancel_region.fiber);
2167	await_count -= 1;
2168	}
2169	}
2170	// Equivalent to `ev.yield(null, .{ .await = await_count });`,
2171	// but avoiding a context switch in the common case.
2172	switch (std.math.order(
2173	@atomicRmw(i32, &cancel_region.fiber.await_count, .Sub, await_count, .monotonic),
2174	await_count,
2175	)) {
2176	.lt => ev.yield(null, .{ .await = 0 }),
2177	.eq => {},
2178	.gt => unreachable,
2179	}
2180	return result;
2181	}
2182
2183	fn futexWait(
2184	userdata: ?*anyopaque,
2185	ptr: *const u32,
2186	expected: u32,
2187	timeout: Io.Timeout,
2188	) Io.Cancelable!void {
2189	const ev: *Evented = @ptrCast(@alignCast(userdata));
2190	const timespec: ?linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = timespec: switch (timeout) {
2191	.none => .{
2192	null,
2193	.awake,
2194	linux.IORING_TIMEOUT_ABS,
2195	},
2196	.duration => \|duration\| {
2197	const ns = duration.raw.toNanoseconds();
2198	break :timespec .{
2199	.{
2200	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
2201	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
2202	},
2203	duration.clock,
2204	0,
2205	};
2206	},
2207	.deadline => \|deadline\| {
2208	const ns = deadline.raw.toNanoseconds();
2209	break :timespec .{
2210	.{
2211	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
2212	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
2213	},
2214	deadline.clock,
2215	linux.IORING_TIMEOUT_ABS,
2216	};
2217	},
2218	};
2219	var cancel_region: CancelRegion = .init();
2220	defer cancel_region.deinit();
2221	const thread = try cancel_region.awaitIoUring();
2222	thread.enqueue().* = .{
2223	.opcode = .FUTEX_WAIT,
2224	.flags = if (timespec) \|_\| linux.IOSQE_IO_LINK else 0,
2225	.ioprio = 0,
2226	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
2227	.off = expected,
2228	.addr = @intFromPtr(ptr),
2229	.len = 0,
2230	.rw_flags = 0,
2231	.user_data = @intFromPtr(cancel_region.fiber),
2232	.buf_index = 0,
2233	.personality = 0,
2234	.splice_fd_in = 0,
2235	.addr3 = std.math.maxInt(u32),
2236	.resv = 0,
2237	};
2238	if (timespec) \|timespec_ptr\| thread.enqueue(). = .{
2239	.opcode = .LINK_TIMEOUT,
2240	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
2241	.ioprio = 0,
2242	.fd = 0,
2243	.off = 0,
2244	.addr = @intFromPtr(timespec_ptr),
2245	.len = 1,
2246	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
2247	.real => linux.IORING_TIMEOUT_REALTIME,
2248	else => 0,
2249	.boot => linux.IORING_TIMEOUT_BOOTTIME,
2250	}),
2251	.user_data = @intFromEnum(Completion.UserData.wakeup),
2252	.buf_index = 0,
2253	.personality = 0,
2254	.splice_fd_in = 0,
2255	.addr3 = 0,
2256	.resv = 0,
2257	};
2258	ev.yield(null, .nothing);
2259	switch (cancel_region.errno()) {
2260	.SUCCESS => {}, // notified by `wake()`
2261	.INTR, .CANCELED => {}, // caller's responsibility to retry
2262	.AGAIN => {}, // ptr.* != expect
2263	.INVAL => {}, // possibly timeout overflow
2264	.TIMEDOUT => unreachable,
2265	.FAULT => recoverableOsBugDetected(), // ptr was invalid
2266	else => recoverableOsBugDetected(),
2267	}
2268	}
2269
2270	fn futexWaitUncancelable(userdata: ?anyopaque, ptr: const u32, expected: u32) void {
2271	const ev: *Evented = @ptrCast(@alignCast(userdata));
2272	var cancel_region: CancelRegion = .initBlocked();
2273	defer cancel_region.deinit();
2274	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
2275	error.Canceled => unreachable, // blocked
2276	};
2277	thread.enqueue().* = .{
2278	.opcode = .FUTEX_WAIT,
2279	.flags = 0,
2280	.ioprio = 0,
2281	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
2282	.off = expected,
2283	.addr = @intFromPtr(ptr),
2284	.len = 0,
2285	.rw_flags = 0,
2286	.user_data = @intFromPtr(cancel_region.fiber),
2287	.buf_index = 0,
2288	.personality = 0,
2289	.splice_fd_in = 0,
2290	.addr3 = std.math.maxInt(u32),
2291	.resv = 0,
2292	};
2293	ev.yield(null, .nothing);
2294	switch (cancel_region.errno()) {
2295	.SUCCESS => {}, // notified by `wake()`
2296	.INTR, .CANCELED => {}, // caller's responsibility to retry
2297	.AGAIN => {}, // ptr.* != expect
2298	.INVAL => {}, // possibly timeout overflow
2299	.FAULT => recoverableOsBugDetected(), // ptr was invalid
2300	else => recoverableOsBugDetected(),
2301	}
2302	}
2303
2304	fn futexWake(userdata: ?anyopaque, ptr: const u32, max_waiters: u32) void {
2305	const ev: *Evented = @ptrCast(@alignCast(userdata));
2306	_ = ev;
2307	const thread: *Thread = .current();
2308	thread.enqueue().* = .{
2309	.opcode = .FUTEX_WAKE,
2310	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
2311	.ioprio = 0,
2312	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
2313	.off = max_waiters,
2314	.addr = @intFromPtr(ptr),
2315	.len = 0,
2316	.rw_flags = 0,
2317	.user_data = @intFromEnum(Completion.UserData.futex_wake),
2318	.buf_index = 0,
2319	.personality = 0,
2320	.splice_fd_in = 0,
2321	.addr3 = std.math.maxInt(u32),
2322	.resv = 0,
2323	};
2324	thread.submit();
2325	}
2326
2327	fn operate(userdata: ?*anyopaque, operation: Io.Operation) Io.Cancelable!Io.Operation.Result {
2328	const ev: *Evented = @ptrCast(@alignCast(userdata));
2329	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
2330	defer maybe_sync.deinit(ev);
2331	return switch (operation) {
2332	.file_read_streaming => \|o\| .{
2333	.file_read_streaming = ev.fileReadStreaming(
2334	&maybe_sync.cancel_region,
2335	o.file,
2336	o.data,
2337	) catch \|err\| switch (err) {
2338	error.Canceled => \|e\| return e,
2339	else => \|e\| e,
2340	},
2341	},
2342	.file_write_streaming => \|o\| .{
2343	.file_write_streaming = ev.fileWriteStreaming(
2344	&maybe_sync.cancel_region,
2345	o.file,
2346	o.header,
2347	o.data,
2348	o.splat,
2349	) catch \|err\| switch (err) {
2350	error.Canceled => \|e\| return e,
2351	else => \|e\| e,
2352	},
2353	},
2354	.device_io_control => \|o\| .{
2355	.device_io_control = try ev.deviceIoControl(try maybe_sync.enterSync(ev), o),
2356	},
2357	};
2358	}
2359
2360	fn fileReadStreaming(
2361	ev: *Evented,
2362	cancel_region: *CancelRegion,
2363	file: File,
2364	data: []const []u8,
2365	) File.ReadStreamingError!usize {
2366	var iovecs_buffer: [max_iovecs_len]iovec = undefined;
2367	var i: usize = 0;
2368	for (data) \|buf\| {
2369	if (iovecs_buffer.len - i == 0) break;
2370	if (buf.len != 0) {
2371	iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len };
2372	i += 1;
2373	}
2374	}
2375	const dest = iovecs_buffer[0..i];
2376	assert(dest[0].len > 0);
2377
2378	const n = try ev.preadv(cancel_region, file.handle, dest, null);
2379	return if (n == 0) error.EndOfStream else n;
2380	}
2381
2382	fn fileWriteStreaming(
2383	ev: *Evented,
2384	cancel_region: *CancelRegion,
2385	file: File,
2386	header: []const u8,
2387	data: []const []const u8,
2388	splat: usize,
2389	) File.Writer.Error!usize {
2390	var iovecs: [max_iovecs_len]iovec_const = undefined;
2391	var iovlen: iovlen_t = 0;
2392	addBuf(&iovecs, &iovlen, header);
2393	for (data[0 .. data.len - 1]) \|bytes\| addBuf(&iovecs, &iovlen, bytes);
2394	const pattern = data[data.len - 1];
2395	if (iovecs.len - iovlen != 0) switch (splat) {
2396	0 => {},
2397	1 => addBuf(&iovecs, &iovlen, pattern),
2398	else => switch (pattern.len) {
2399	0 => {},
2400	1 => {
2401	var backup_buffer: [splat_buffer_size]u8 = undefined;
2402	const splat_buffer = &backup_buffer;
2403	const memset_len = @min(splat_buffer.len, splat);
2404	const buf = splat_buffer[0..memset_len];
2405	@memset(buf, pattern[0]);
2406	addBuf(&iovecs, &iovlen, buf);
2407	var remaining_splat = splat - buf.len;
2408	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0) {
2409	assert(buf.len == splat_buffer.len);
2410	addBuf(&iovecs, &iovlen, splat_buffer);
2411	remaining_splat -= splat_buffer.len;
2412	}
2413	addBuf(&iovecs, &iovlen, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
2414	},
2415	else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
2416	addBuf(&iovecs, &iovlen, pattern);
2417	},
2418	},
2419	};
2420	return ev.pwritev(cancel_region, file.handle, iovecs[0..iovlen], null);
2421	}
2422
2423	fn deviceIoControl(
2424	ev: *Evented,
2425	sync: *CancelRegion.Sync,
2426	o: Io.Operation.DeviceIoControl,
2427	) Io.Cancelable!i32 {
2428	_ = ev;
2429	while (true) {
2430	try sync.cancel_region.await(.nothing);
2431	const rc = linux.ioctl(o.file.handle, @bitCast(o.code), @intFromPtr(o.arg));
2432	switch (linux.errno(rc)) {
2433	.SUCCESS => return @bitCast(@as(u32, @truncate(rc))),
2434	.INTR => continue,
2435	else => \|err\| return -@as(i32, @intFromEnum(err)),
2436	}
2437	}
2438	}
2439
2440	fn batchAwaitAsync(userdata: ?anyopaque, batch: Io.Batch) Io.Cancelable!void {
2441	const ev: *Evented = @ptrCast(@alignCast(userdata));
2442	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
2443	defer maybe_sync.deinit(ev);
2444	ev.batchDrainSubmitted(&maybe_sync, batch, false) catch \|err\| switch (err) {
2445	error.ConcurrencyUnavailable => unreachable, // passed concurrency=false
2446	else => \|e\| return e,
2447	};
2448	maybe_sync.leaveSync(ev);
2449	while (true) {
2450	batchDrainReady(batch) catch \|err\| switch (err) {
2451	error.Timeout => unreachable, // no timeout
2452	};
2453	if (batch.completed.head != .none) return;
2454	ev.yield(null, .{ .batch_await = batch });
2455	}
2456	}
2457
2458	fn batchAwaitConcurrent(
2459	userdata: ?*anyopaque,
2460	batch: *Io.Batch,
2461	timeout: Io.Timeout,
2462	) Io.Batch.AwaitConcurrentError!void {
2463	const ev: *Evented = @ptrCast(@alignCast(userdata));
2464	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
2465	defer maybe_sync.deinit(ev);
2466	try ev.batchDrainSubmitted(&maybe_sync, batch, true);
2467	maybe_sync.leaveSync(ev);
2468	const timespec: linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = while (true) {
2469	batchDrainReady(batch) catch \|err\| switch (err) {
2470	error.Timeout => unreachable, // no timeout
2471	};
2472	if (batch.completed.head != .none) return;
2473	switch (timeout) {
2474	.none => ev.yield(null, .{ .batch_await = batch }),
2475	.duration => \|duration\| {
2476	const ns = duration.raw.toNanoseconds();
2477	break .{
2478	.{
2479	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
2480	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
2481	},
2482	duration.clock,
2483	0,
2484	};
2485	},
2486	.deadline => \|deadline\| {
2487	const ns = deadline.raw.toNanoseconds();
2488	break .{
2489	.{
2490	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
2491	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
2492	},
2493	deadline.clock,
2494	linux.IORING_TIMEOUT_ABS,
2495	};
2496	},
2497	}
2498	};
2499	{
2500	const thread = try maybe_sync.cancel_region.awaitIoUring();
2501	thread.enqueue().* = .{
2502	.opcode = .TIMEOUT,
2503	.flags = 0,
2504	.ioprio = 0,
2505	.fd = 0,
2506	.off = 0,
2507	.addr = @intFromPtr(&timespec),
2508	.len = 1,
2509	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
2510	.real => linux.IORING_TIMEOUT_REALTIME,
2511	else => 0,
2512	.boot => linux.IORING_TIMEOUT_BOOTTIME,
2513	}),
2514	.user_data = @intFromPtr(&batch.context) \| 0b11,
2515	.buf_index = 0,
2516	.personality = 0,
2517	.splice_fd_in = 0,
2518	.addr3 = 0,
2519	.resv = 0,
2520	};
2521	}
2522	while (batch.completed.head == .none) {
2523	ev.yield(null, .{ .batch_await = batch });
2524	batchDrainReady(batch) catch \|err\| switch (err) {
2525	error.Timeout => \|e\| return if (batch.completed.head == .none) e,
2526	};
2527	if (batch.completed.head == .none) continue;
2528	}
2529	const thread = try maybe_sync.cancel_region.awaitIoUring();
2530	thread.enqueue().* = .{
2531	.opcode = .TIMEOUT_REMOVE,
2532	.flags = 0,
2533	.ioprio = 0,
2534	.fd = 0,
2535	.off = 0,
2536	.addr = @intFromPtr(&batch.context) \| 0b11,
2537	.len = 0,
2538	.rw_flags = 0,
2539	.user_data = @intFromPtr(maybe_sync.cancel_region.fiber),
2540	.buf_index = 0,
2541	.personality = 0,
2542	.splice_fd_in = 0,
2543	.addr3 = 0,
2544	.resv = 0,
2545	};
2546	ev.yield(null, .nothing);
2547	switch (maybe_sync.cancel_region.errno()) {
2548	.SUCCESS => return,
2549	.BUSY, .NOENT => {},
2550	else => \|err\| unexpectedErrno(err) catch {},
2551	}
2552	while (true) {
2553	batchDrainReady(batch) catch \|err\| switch (err) {
2554	error.Timeout => return,
2555	};
2556	ev.yield(null, .{ .batch_await = batch });
2557	}
2558	}
2559
2560	/// If `concurrency` is false, `error.ConcurrencyUnavailable` is unreachable.
2561	fn batchDrainSubmitted(
2562	ev: *Evented,
2563	maybe_sync: *CancelRegion.Sync.Maybe,
2564	batch: *Io.Batch,
2565	concurrency: bool,
2566	) (Io.ConcurrentError \|\| Io.Cancelable)!void {
2567	var index = batch.submitted.head;
2568	if (index == .none) return;
2569	errdefer batch.submitted.head = index;
2570	const thread = try maybe_sync.cancelRegion().awaitIoUring();
2571	while (index != .none) {
2572	const storage = &batch.storage[index.toIndex()];
2573	const next_index = storage.submission.node.next;
2574	if (@as(?Io.Operation.Result, result: switch (storage.submission.operation) {
2575	.file_read_streaming => \|o\| {
2576	const buffer = for (o.data) \|buffer\| {
2577	if (buffer.len != 0) break buffer;
2578	} else break :result .{ .file_read_streaming = 0 };
2579	const fd = o.file.handle;
2580	storage.* = .{ .pending = .{
2581	.node = .{ .prev = batch.pending.tail, .next = .none },
2582	.tag = .file_read_streaming,
2583	.context = undefined,
2584	} };
2585	thread.enqueue().* = .{
2586	.opcode = .READ,
2587	.flags = 0,
2588	.ioprio = 0,
2589	.fd = fd,
2590	.off = std.math.maxInt(u64),
2591	.addr = @intFromPtr(buffer.ptr),
2592	.len = @min(buffer.len, 0xfffff000),
2593	.rw_flags = 0,
2594	.user_data = @intFromPtr(&storage.pending.context) \| 0b10,
2595	.buf_index = 0,
2596	.personality = 0,
2597	.splice_fd_in = 0,
2598	.addr3 = 0,
2599	.resv = 0,
2600	};
2601	break :result null;
2602	},
2603	.file_write_streaming => \|o\| {
2604	const buffer = buffer: {
2605	if (o.header.len != 0) break :buffer o.header;
2606	for (o.data[0 .. o.data.len - 1]) \|buffer\| {
2607	if (buffer.len != 0) break :buffer buffer;
2608	}
2609	if (o.splat > 0) break :buffer o.data[o.data.len - 1];
2610	break :result .{ .file_write_streaming = 0 };
2611	};
2612	const fd = o.file.handle;
2613	storage.* = .{ .pending = .{
2614	.node = .{ .prev = batch.pending.tail, .next = .none },
2615	.tag = .file_write_streaming,
2616	.context = undefined,
2617	} };
2618	thread.enqueue().* = .{
2619	.opcode = .WRITE,
2620	.flags = 0,
2621	.ioprio = 0,
2622	.fd = fd,
2623	.off = std.math.maxInt(u64),
2624	.addr = @intFromPtr(buffer.ptr),
2625	.len = @min(buffer.len, 0xfffff000),
2626	.rw_flags = 0,
2627	.user_data = @intFromPtr(&storage.pending.context) \| 0b10,
2628	.buf_index = 0,
2629	.personality = 0,
2630	.splice_fd_in = 0,
2631	.addr3 = 0,
2632	.resv = 0,
2633	};
2634	break :result null;
2635	},
2636	.device_io_control => \|o\| if (concurrency)
2637	return error.ConcurrencyUnavailable
2638	else
2639	.{ .device_io_control = try ev.deviceIoControl(try maybe_sync.enterSync(ev), o) },
2640	})) \|result\| {
2641	switch (batch.completed.tail) {
2642	.none => batch.completed.head = index,
2643	else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next = index,
2644	}
2645	batch.completed.tail = index;
2646	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2647	} else {
2648	switch (batch.pending.tail) {
2649	.none => batch.pending.head = index,
2650	else => \|tail_index\| batch.storage[tail_index.toIndex()].pending.node.next = index,
2651	}
2652	batch.pending.tail = index;
2653	storage.pending.context[0] = @intFromPtr(batch);
2654	}
2655	index = next_index;
2656	}
2657	batch.submitted = .{ .head = .none, .tail = .none };
2658	}
2659
2660	fn batchDrainReady(batch: *Io.Batch) Io.Timeout.Error!void {
2661	while (@atomicRmw(?*anyopaque, &batch.context, .Xchg, null, .acquire)) \|head\| {
2662	var next: usize = @intFromPtr(head);
2663	var timeout = false;
2664	while (cond: switch (@as(u2, @truncate(next))) {
2665	0b00 => if (timeout) return error.Timeout else false,
2666	0b01 => {
2667	assert(!timeout);
2668	return error.Timeout;
2669	},
2670	0b10 => true,
2671	0b11 => {
2672	assert(!timeout);
2673	timeout = true;
2674	break :cond true;
2675	},
2676	}) {
2677	var context: *Io.Operation.Storage.Pending.Context = @ptrFromInt(next & ~@as(usize, 0b11));
2678	next = context[0];
2679	const completion: Completion = .{
2680	.result = @bitCast(@as(u32, @intCast(context[1]))),
2681	.flags = @intCast(context[2]),
2682	};
2683	const pending: *Io.Operation.Storage.Pending = @fieldParentPtr("context", context);
2684	const storage: *Io.Operation.Storage = @fieldParentPtr("pending", pending);
2685	const index: Io.Operation.OptionalIndex = .fromIndex(storage - batch.storage.ptr);
2686	assert(completion.flags & linux.IORING_CQE_F_SKIP == 0);
2687	switch (pending.node.prev) {
2688	.none => batch.pending.head = pending.node.next,
2689	else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.next =
2690	pending.node.next,
2691	}
2692	switch (pending.node.next) {
2693	.none => batch.pending.tail = pending.node.prev,
2694	else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.prev =
2695	pending.node.prev,
2696	}
2697	if (@as(?Io.Operation.Result, result: switch (pending.tag) {
2698	.file_read_streaming => .{
2699	.file_read_streaming = switch (completion.errno()) {
2700	.SUCCESS => @as(u32, @bitCast(completion.result)),
2701	.INTR => 0,
2702	.CANCELED => break :result null,
2703	.INVAL => \|err\| errnoBug(err),
2704	.FAULT => \|err\| errnoBug(err),
2705	.AGAIN => error.WouldBlock,
2706	.BADF => \|err\| errnoBug(err), // File descriptor used after closed
2707	.IO => error.InputOutput,
2708	.ISDIR => error.IsDir,
2709	.NOBUFS => error.SystemResources,
2710	.NOMEM => error.SystemResources,
2711	.NOTCONN => error.SocketUnconnected,
2712	.CONNRESET => error.ConnectionResetByPeer,
2713	else => \|err\| unexpectedErrno(err),
2714	},
2715	},
2716	.file_write_streaming => .{
2717	.file_write_streaming = switch (completion.errno()) {
2718	.SUCCESS => @as(u32, @bitCast(completion.result)),
2719	.INTR => 0,
2720	.CANCELED => break :result null,
2721	.INVAL => \|err\| errnoBug(err),
2722	.FAULT => \|err\| errnoBug(err),
2723	.AGAIN => error.WouldBlock,
2724	.BADF => error.NotOpenForWriting, // Can be a race condition.
2725	.DESTADDRREQ => \|err\| errnoBug(err), // `connect` was never called.
2726	.DQUOT => error.DiskQuota,
2727	.FBIG => error.FileTooBig,
2728	.IO => error.InputOutput,
2729	.NOSPC => error.NoSpaceLeft,
2730	.PERM => error.PermissionDenied,
2731	.PIPE => error.BrokenPipe,
2732	.CONNRESET => \|err\| errnoBug(err), // Not a socket handle.
2733	.BUSY => error.DeviceBusy,
2734	else => \|err\| unexpectedErrno(err),
2735	},
2736	},
2737	.device_io_control => unreachable,
2738	})) \|result\| {
2739	switch (batch.completed.tail) {
2740	.none => batch.completed.head = index,
2741	else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next =
2742	index,
2743	}
2744	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2745	batch.completed.tail = index;
2746	} else {
2747	switch (batch.unused.tail) {
2748	.none => batch.unused.head = index,
2749	else => \|tail_index\| batch.storage[tail_index.toIndex()].unused.next = index,
2750	}
2751	storage.* = .{ .unused = .{ .prev = batch.unused.tail, .next = .none } };
2752	batch.unused.tail = index;
2753	}
2754	}
2755	}
2756	}
2757
2758	fn batchCancel(userdata: ?anyopaque, batch: Io.Batch) void {
2759	const ev: *Evented = @ptrCast(@alignCast(userdata));
2760	_ = ev;
2761	batchDrainReady(batch) catch \|err\| switch (err) {
2762	error.Timeout => unreachable, // no timeout
2763	};
2764	var index = batch.pending.head;
2765	if (index == .none) return;
2766	var cancel_region: CancelRegion = .initBlocked();
2767	defer cancel_region.deinit();
2768	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
2769	error.Canceled => unreachable, // blocked
2770	};
2771	while (index != .none) {
2772	const pending = &batch.storage[index.toIndex()].pending;
2773	thread.enqueue().* = .{
2774	.opcode = .ASYNC_CANCEL,
2775	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
2776	.ioprio = 0,
2777	.fd = 0,
2778	.off = 0,
2779	.addr = @intFromPtr(&pending.context) \| 0b10,
2780	.len = 0,
2781	.rw_flags = 0,
2782	.user_data = @intFromEnum(Completion.UserData.wakeup),
2783	.buf_index = 0,
2784	.personality = 0,
2785	.splice_fd_in = 0,
2786	.addr3 = 0,
2787	.resv = 0,
2788	};
2789	index = pending.node.next;
2790	}
2791	while (batch.pending.head != .none) batchDrainReady(batch) catch \|err\| switch (err) {
2792	error.Timeout => unreachable, // no timeout
2793	};
2794	}
2795
2796	fn dirCreateDir(
2797	userdata: ?*anyopaque,
2798	dir: Dir,
2799	sub_path: []const u8,
2800	permissions: Dir.Permissions,
2801	) Dir.CreateDirError!void {
2802	const ev: *Evented = @ptrCast(@alignCast(userdata));
2803
2804	var path_buffer: [PATH_MAX]u8 = undefined;
2805	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2806
2807	var cancel_region: CancelRegion = .init();
2808	defer cancel_region.deinit();
2809	while (true) {
2810	const thread = try cancel_region.awaitIoUring();
2811	thread.enqueue().* = .{
2812	.opcode = .MKDIRAT,
2813	.flags = 0,
2814	.ioprio = 0,
2815	.fd = dir.handle,
2816	.off = 0,
2817	.addr = @intFromPtr(sub_path_posix.ptr),
2818	.len = permissions.toMode(),
2819	.rw_flags = 0,
2820	.user_data = @intFromPtr(cancel_region.fiber),
2821	.buf_index = 0,
2822	.personality = 0,
2823	.splice_fd_in = 0,
2824	.addr3 = 0,
2825	.resv = 0,
2826	};
2827	ev.yield(null, .nothing);
2828	switch (cancel_region.errno()) {
2829	.SUCCESS => return,
2830	.INTR, .CANCELED => continue,
2831	.ACCES => return error.AccessDenied,
2832	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
2833	.PERM => return error.PermissionDenied,
2834	.DQUOT => return error.DiskQuota,
2835	.EXIST => return error.PathAlreadyExists,
2836	.FAULT => \|err\| return errnoBug(err),
2837	.LOOP => return error.SymLinkLoop,
2838	.MLINK => return error.LinkQuotaExceeded,
2839	.NAMETOOLONG => return error.NameTooLong,
2840	.NOENT => return error.FileNotFound,
2841	.NOMEM => return error.SystemResources,
2842	.NOSPC => return error.NoSpaceLeft,
2843	.NOTDIR => return error.NotDir,
2844	.ROFS => return error.ReadOnlyFileSystem,
2845	// dragonfly: when dir_fd is unlinked from filesystem
2846	.NOTCONN => return error.FileNotFound,
2847	.ILSEQ => return error.BadPathName,
2848	else => \|err\| return unexpectedErrno(err),
2849	}
2850	}
2851	}
2852
2853	fn dirCreateDirPath(
2854	userdata: ?*anyopaque,
2855	dir: Dir,
2856	sub_path: []const u8,
2857	permissions: Dir.Permissions,
2858	) Dir.CreateDirPathError!Dir.CreatePathStatus {
2859	const ev: *Evented = @ptrCast(@alignCast(userdata));
2860
2861	var it = Dir.path.componentIterator(sub_path);
2862	var status: Dir.CreatePathStatus = .existed;
2863	var component = it.last() orelse return error.BadPathName;
2864	while (true) {
2865	if (dirCreateDir(ev, dir, component.path, permissions)) \|_\| {
2866	status = .created;
2867	} else \|err\| switch (err) {
2868	error.PathAlreadyExists => {
2869	// stat the file and return an error if it's not a directory
2870	// this is important because otherwise a dangling symlink
2871	// could cause an infinite loop
2872	const fstat = try dirStatFile(ev, dir, component.path, .{});
2873	if (fstat.kind != .directory) return error.NotDir;
2874	},
2875	error.FileNotFound => \|e\| {
2876	component = it.previous() orelse return e;
2877	continue;
2878	},
2879	else => \|e\| return e,
2880	}
2881	component = it.next() orelse return status;
2882	}
2883	}
2884
2885	fn dirCreateDirPathOpen(
2886	userdata: ?*anyopaque,
2887	dir: Dir,
2888	sub_path: []const u8,
2889	permissions: Dir.Permissions,
2890	options: Dir.OpenOptions,
2891	) Dir.CreateDirPathOpenError!Dir {
2892	const ev: *Evented = @ptrCast(@alignCast(userdata));
2893	return dirOpenDir(ev, dir, sub_path, options) catch \|err\| switch (err) {
2894	error.FileNotFound => {
2895	_ = try dirCreateDirPath(ev, dir, sub_path, permissions);
2896	return dirOpenDir(ev, dir, sub_path, options);
2897	},
2898	else => \|e\| return e,
2899	};
2900	}
2901
2902	fn dirOpenDir(
2903	userdata: ?*anyopaque,
2904	dir: Dir,
2905	sub_path: []const u8,
2906	options: Dir.OpenOptions,
2907	) Dir.OpenError!Dir {
2908	const ev: *Evented = @ptrCast(@alignCast(userdata));
2909
2910	var path_buffer: [PATH_MAX]u8 = undefined;
2911	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2912
2913	var cancel_region: CancelRegion = .init();
2914	defer cancel_region.deinit();
2915	return .{
2916	.handle = ev.openat(&cancel_region, dir.handle, sub_path_posix, .{
2917	.ACCMODE = .RDONLY,
2918	.DIRECTORY = true,
2919	.NOFOLLOW = !options.follow_symlinks,
2920	.CLOEXEC = true,
2921	.PATH = !options.iterate,
2922	}, 0) catch \|err\| switch (err) {
2923	error.IsDir => return errnoBug(.ISDIR),
2924	error.WouldBlock => return errnoBug(.AGAIN),
2925	error.FileTooBig => return errnoBug(.FBIG),
2926	error.NoSpaceLeft => return errnoBug(.NOSPC),
2927	error.DeviceBusy => return errnoBug(.BUSY), // O_EXCL not passed
2928	error.FileBusy => return errnoBug(.TXTBSY),
2929	error.PathAlreadyExists => return errnoBug(.EXIST), // Not creating.
2930	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
2931	error.AntivirusInterference => unreachable, // Windows-only
2932	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
2933	else => \|e\| return e,
2934	},
2935	};
2936	}
2937
2938	fn dirStat(userdata: ?*anyopaque, dir: Dir) Dir.StatError!Dir.Stat {
2939	const ev: *Evented = @ptrCast(@alignCast(userdata));
2940	var cancel_region: CancelRegion = .init();
2941	defer cancel_region.deinit();
2942	return ev.stat(&cancel_region, dir.handle);
2943	}
2944
2945	fn dirStatFile(
2946	userdata: ?*anyopaque,
2947	dir: Dir,
2948	sub_path: []const u8,
2949	options: Dir.StatFileOptions,
2950	) Dir.StatFileError!File.Stat {
2951	const ev: *Evented = @ptrCast(@alignCast(userdata));
2952	var path_buffer: [PATH_MAX]u8 = undefined;
2953	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2954	var cancel_region: CancelRegion = .init();
2955	defer cancel_region.deinit();
2956	return ev.statx(&cancel_region, dir.handle, sub_path_posix, linux.AT.NO_AUTOMOUNT \|
2957	@as(u32, if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW));
2958	}
2959
2960	fn dirAccess(
2961	userdata: ?*anyopaque,
2962	dir: Dir,
2963	sub_path: []const u8,
2964	options: Dir.AccessOptions,
2965	) Dir.AccessError!void {
2966	const ev: *Evented = @ptrCast(@alignCast(userdata));
2967
2968	var path_buffer: [PATH_MAX]u8 = undefined;
2969	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2970
2971	const mode: u32 =
2972	@as(u32, if (options.read) linux.R_OK else 0) \|
2973	@as(u32, if (options.write) linux.W_OK else 0) \|
2974	@as(u32, if (options.execute) linux.X_OK else 0);
2975	const flags: u32 = if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW;
2976
2977	var sync: CancelRegion.Sync = try .init(ev);
2978	defer sync.deinit(ev);
2979	while (true) {
2980	try sync.cancel_region.await(.nothing);
2981	switch (linux.errno(linux.faccessat(dir.handle, sub_path_posix, mode, flags))) {
2982	.SUCCESS => return,
2983	.INTR => continue,
2984	.ACCES => return error.AccessDenied,
2985	.PERM => return error.PermissionDenied,
2986	.ROFS => return error.ReadOnlyFileSystem,
2987	.LOOP => return error.SymLinkLoop,
2988	.TXTBSY => return error.FileBusy,
2989	.NOTDIR => return error.FileNotFound,
2990	.NOENT => return error.FileNotFound,
2991	.NAMETOOLONG => return error.NameTooLong,
2992	.INVAL => \|err\| return errnoBug(err),
2993	.FAULT => \|err\| return errnoBug(err),
2994	.IO => return error.InputOutput,
2995	.NOMEM => return error.SystemResources,
2996	.ILSEQ => return error.BadPathName,
2997	else => \|err\| return unexpectedErrno(err),
2998	}
2999	}
3000	}
3001
3002	fn dirCreateFile(
3003	userdata: ?*anyopaque,
3004	dir: Dir,
3005	sub_path: []const u8,
3006	flags: File.CreateFlags,
3007	) File.OpenError!File {
3008	const ev: *Evented = @ptrCast(@alignCast(userdata));
3009
3010	var path_buffer: [PATH_MAX]u8 = undefined;
3011	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3012
3013	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
3014	defer maybe_sync.deinit(ev);
3015	const fd = try ev.openat(&maybe_sync.cancel_region, dir.handle, sub_path_posix, .{
3016	.ACCMODE = if (flags.read) .RDWR else .WRONLY,
3017	.CREAT = true,
3018	.TRUNC = flags.truncate,
3019	.EXCL = flags.exclusive,
3020	.CLOEXEC = true,
3021	}, flags.permissions.toMode());
3022	errdefer ev.close(maybe_sync.cancelRegion(), fd);
3023
3024	switch (flags.lock) {
3025	.none => {},
3026	.shared, .exclusive => try ev.flock(
3027	try maybe_sync.enterSync(ev),
3028	fd,
3029	flags.lock,
3030	if (flags.lock_nonblocking) .nonblocking else .blocking,
3031	),
3032	}
3033
3034	return .{ .handle = fd, .flags = .{ .nonblocking = false } };
3035	}
3036
3037	fn dirCreateFileAtomic(
3038	userdata: ?*anyopaque,
3039	dir: Dir,
3040	dest_path: []const u8,
3041	options: Dir.CreateFileAtomicOptions,
3042	) Dir.CreateFileAtomicError!File.Atomic {
3043	const ev: *Evented = @ptrCast(@alignCast(userdata));
3044	// Linux has O_TMPFILE, but linkat() does not support AT_REPLACE, so it's
3045	// useless when we have to make up a bogus path name to do the rename()
3046	// anyway.
3047	if (!options.replace) tmpfile: {
3048	const flags: linux.O = if (@hasField(linux.O, "TMPFILE")) .{
3049	.ACCMODE = .RDWR,
3050	.TMPFILE = true,
3051	.DIRECTORY = true,
3052	.CLOEXEC = true,
3053	} else if (@hasField(linux.O, "TMPFILE0") and !@hasField(linux.O, "TMPFILE2")) .{
3054	.ACCMODE = .RDWR,
3055	.TMPFILE0 = true,
3056	.TMPFILE1 = true,
3057	.DIRECTORY = true,
3058	.CLOEXEC = true,
3059	} else break :tmpfile;
3060
3061	const dest_dirname = Dir.path.dirname(dest_path);
3062	if (dest_dirname) \|dirname\| {
3063	// This has a nice side effect of preemptively triggering EISDIR or
3064	// ENOENT, avoiding the ambiguity below.
3065	_ = dirCreateDirPath(ev, dir, dirname, .default_dir) catch \|err\| switch (err) {
3066	// None of these make sense in this context.
3067	error.IsDir,
3068	error.Streaming,
3069	error.DiskQuota,
3070	error.PathAlreadyExists,
3071	error.LinkQuotaExceeded,
3072	error.PipeBusy,
3073	error.FileTooBig,
3074	error.DeviceBusy,
3075	error.FileLocksUnsupported,
3076	error.FileBusy,
3077	=> return error.Unexpected,
3078
3079	else => \|e\| return e,
3080	};
3081	}
3082
3083	var path_buffer: [PATH_MAX]u8 = undefined;
3084	const sub_path_posix = try pathToPosix(dest_dirname orelse ".", &path_buffer);
3085
3086	var cancel_region: CancelRegion = .init();
3087	defer cancel_region.deinit();
3088	return .{
3089	.file = .{
3090	.handle = ev.openat(
3091	&cancel_region,
3092	dir.handle,
3093	sub_path_posix,
3094	flags,
3095	options.permissions.toMode(),
3096	) catch \|err\| switch (err) {
3097	error.IsDir, error.FileNotFound => {
3098	// Ambiguous error code. It might mean the file system
3099	// does not support O_TMPFILE. Therefore, we must fall
3100	// back to not using O_TMPFILE.
3101	break :tmpfile;
3102	},
3103	error.FileTooBig => return errnoBug(.FBIG),
3104	error.DeviceBusy => return errnoBug(.BUSY), // O_EXCL not passed
3105	error.PathAlreadyExists => return errnoBug(.EXIST), // Not creating.
3106	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
3107	error.AntivirusInterference => unreachable, // Windows-only
3108	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
3109	else => \|e\| return e,
3110	},
3111	.flags = .{ .nonblocking = false },
3112	},
3113	.file_basename_hex = 0,
3114	.dest_sub_path = dest_path,
3115	.file_open = true,
3116	.file_exists = false,
3117	.close_dir_on_deinit = false,
3118	.dir = dir,
3119	};
3120	}
3121
3122	if (Dir.path.dirname(dest_path)) \|dirname\| {
3123	const new_dir = if (options.make_path)
3124	dirCreateDirPathOpen(ev, dir, dirname, .default_dir, .{}) catch \|err\| switch (err) {
3125	// None of these make sense in this context.
3126	error.IsDir,
3127	error.Streaming,
3128	error.DiskQuota,
3129	error.PathAlreadyExists,
3130	error.LinkQuotaExceeded,
3131	error.PipeBusy,
3132	error.FileTooBig,
3133	error.FileLocksUnsupported,
3134	error.DeviceBusy,
3135	=> return error.Unexpected,
3136
3137	else => \|e\| return e,
3138	}
3139	else
3140	try dirOpenDir(ev, dir, dirname, .{});
3141
3142	return ev.atomicFileInit(Dir.path.basename(dest_path), options.permissions, new_dir, true);
3143	}
3144
3145	return ev.atomicFileInit(dest_path, options.permissions, dir, false);
3146	}
3147
3148	fn atomicFileInit(
3149	ev: *Evented,
3150	dest_basename: []const u8,
3151	permissions: File.Permissions,
3152	dir: Dir,
3153	close_dir_on_deinit: bool,
3154	) Dir.CreateFileAtomicError!File.Atomic {
3155	while (true) {
3156	var random_integer: u64 = undefined;
3157	random(ev, @ptrCast(&random_integer));
3158	const tmp_sub_path = std.fmt.hex(random_integer);
3159	const file = dirCreateFile(ev, dir, &tmp_sub_path, .{
3160	.permissions = permissions,
3161	.exclusive = true,
3162	}) catch \|err\| switch (err) {
3163	error.PathAlreadyExists => continue,
3164	error.DeviceBusy => continue,
3165	error.FileBusy => continue,
3166
3167	error.IsDir => return error.Unexpected, // No path components.
3168	error.FileTooBig => return error.Unexpected, // Creating, not opening.
3169	error.FileLocksUnsupported => return error.Unexpected, // Not asking for locks.
3170	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
3171
3172	else => \|e\| return e,
3173	};
3174	return .{
3175	.file = file,
3176	.file_basename_hex = random_integer,
3177	.dest_sub_path = dest_basename,
3178	.file_open = true,
3179	.file_exists = true,
3180	.close_dir_on_deinit = close_dir_on_deinit,
3181	.dir = dir,
3182	};
3183	}
3184	}
3185
3186	fn dirOpenFile(
3187	userdata: ?*anyopaque,
3188	dir: Dir,
3189	sub_path: []const u8,
3190	flags: File.OpenFlags,
3191	) File.OpenError!File {
3192	const ev: *Evented = @ptrCast(@alignCast(userdata));
3193
3194	var path_buffer: [PATH_MAX]u8 = undefined;
3195	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3196
3197	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
3198	defer maybe_sync.deinit(ev);
3199	const fd = try ev.openat(&maybe_sync.cancel_region, dir.handle, sub_path_posix, .{
3200	.ACCMODE = switch (flags.mode) {
3201	.read_only => .RDONLY,
3202	.write_only => .WRONLY,
3203	.read_write => .RDWR,
3204	},
3205	.NOCTTY = !flags.allow_ctty,
3206	.NOFOLLOW = !flags.follow_symlinks,
3207	.CLOEXEC = true,
3208	.PATH = flags.path_only,
3209	}, 0);
3210	errdefer ev.close(maybe_sync.cancelRegion(), fd);
3211
3212	if (!flags.allow_directory) {
3213	const is_dir = is_dir: {
3214	const s = ev.stat(&maybe_sync.cancel_region, fd) catch \|err\| switch (err) {
3215	// The directory-ness is either unknown or unknowable
3216	error.Streaming => break :is_dir false,
3217	else => \|e\| return e,
3218	};
3219	break :is_dir s.kind == .directory;
3220	};
3221	if (is_dir) return error.IsDir;
3222	}
3223
3224	switch (flags.lock) {
3225	.none => {},
3226	.shared, .exclusive => try ev.flock(
3227	try maybe_sync.enterSync(ev),
3228	fd,
3229	flags.lock,
3230	if (flags.lock_nonblocking) .nonblocking else .blocking,
3231	),
3232	}
3233
3234	return .{ .handle = fd, .flags = .{ .nonblocking = false } };
3235	}
3236
3237	fn dirClose(userdata: ?*anyopaque, dirs: []const Dir) void {
3238	const ev: *Evented = @ptrCast(@alignCast(userdata));
3239	var cancel_region: CancelRegion = .init();
3240	defer cancel_region.deinit();
3241	for (dirs) \|dir\| ev.close(&cancel_region, dir.handle);
3242	}
3243
3244	fn dirRead(userdata: ?anyopaque, dr: Dir.Reader, buffer: []Dir.Entry) Dir.Reader.Error!usize {
3245	const ev: *Evented = @ptrCast(@alignCast(userdata));
3246	var buffer_index: usize = 0;
3247	while (buffer.len - buffer_index != 0) {
3248	if (dr.end - dr.index == 0) {
3249	// Refill the buffer, unless we've already created references to
3250	// buffered data.
3251	if (buffer_index != 0) break;
3252	var sync: CancelRegion.Sync = try .init(ev);
3253	defer sync.deinit(ev);
3254	if (dr.state == .reset) {
3255	ev.lseek(&sync, dr.dir.handle, 0, linux.SEEK.SET) catch \|err\| switch (err) {
3256	error.Unseekable => return error.Unexpected,
3257	else => \|e\| return e,
3258	};
3259	dr.state = .reading;
3260	}
3261	const n = while (true) {
3262	try sync.cancel_region.await(.nothing);
3263	const rc = linux.getdents64(dr.dir.handle, dr.buffer.ptr, dr.buffer.len);
3264	switch (linux.errno(rc)) {
3265	.SUCCESS => break rc,
3266	.INTR => continue,
3267	.BADF => \|err\| return errnoBug(err), // Dir is invalid or was opened without iteration ability.
3268	.FAULT => \|err\| return errnoBug(err),
3269	.NOTDIR => \|err\| return errnoBug(err),
3270	// To be consistent across platforms, iteration
3271	// ends if the directory being iterated is deleted
3272	// during iteration. This matches the behavior of
3273	// non-Linux, non-WASI UNIX platforms.
3274	.NOENT => {
3275	dr.state = .finished;
3276	return 0;
3277	},
3278	// This can occur when reading /proc/$PID/net, or
3279	// if the provided buffer is too small. Neither
3280	// scenario is intended to be handled by this API.
3281	.INVAL => return error.Unexpected,
3282	.ACCES => return error.AccessDenied, // Lacking permission to iterate this directory.
3283	else => \|err\| return unexpectedErrno(err),
3284	}
3285	};
3286	if (n == 0) {
3287	dr.state = .finished;
3288	return 0;
3289	}
3290	dr.index = 0;
3291	dr.end = n;
3292	}
3293	// Linux aligns the header by padding after the null byte of the name
3294	// to align the next entry. This means we can find the end of the name
3295	// by looking at only the 8 bytes before the next record. However since
3296	// file names are usually short it's better to keep the machine code
3297	// simpler.
3298	//
3299	// Furthermore, I observed qemu user mode to not align this struct, so
3300	// this code makes the conservative choice to not assume alignment.
3301	const linux_entry: *align(1) linux.dirent64 = @ptrCast(&dr.buffer[dr.index]);
3302	const next_index = dr.index + linux_entry.reclen;
3303	dr.index = next_index;
3304	const name_ptr: [*]u8 = &linux_entry.name;
3305	const padded_name = name_ptr[0 .. linux_entry.reclen - @offsetOf(linux.dirent64, "name")];
3306	const name_len = std.mem.findScalar(u8, padded_name, 0).?;
3307	const name = name_ptr[0..name_len :0];
3308
3309	if (std.mem.eql(u8, name, ".") or std.mem.eql(u8, name, "..")) continue;
3310
3311	const entry_kind: File.Kind = switch (linux_entry.type) {
3312	linux.DT.BLK => .block_device,
3313	linux.DT.CHR => .character_device,
3314	linux.DT.DIR => .directory,
3315	linux.DT.FIFO => .named_pipe,
3316	linux.DT.LNK => .sym_link,
3317	linux.DT.REG => .file,
3318	linux.DT.SOCK => .unix_domain_socket,
3319	else => .unknown,
3320	};
3321	buffer[buffer_index] = .{
3322	.name = name,
3323	.kind = entry_kind,
3324	.inode = linux_entry.ino,
3325	};
3326	buffer_index += 1;
3327	}
3328	return buffer_index;
3329	}
3330
3331	fn dirRealPath(userdata: ?*anyopaque, dir: Dir, out_buffer: []u8) Dir.RealPathError!usize {
3332	const ev: *Evented = @ptrCast(@alignCast(userdata));
3333	var sync: CancelRegion.Sync = try .init(ev);
3334	defer sync.deinit(ev);
3335	return ev.realPath(&sync, dir.handle, out_buffer);
3336	}
3337
3338	fn dirRealPathFile(
3339	userdata: ?*anyopaque,
3340	dir: Dir,
3341	sub_path: []const u8,
3342	out_buffer: []u8,
3343	) Dir.RealPathFileError!usize {
3344	const ev: *Evented = @ptrCast(@alignCast(userdata));
3345
3346	var path_buffer: [PATH_MAX]u8 = undefined;
3347	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3348
3349	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
3350	defer maybe_sync.deinit(ev);
3351	const fd = ev.openat(&maybe_sync.cancel_region, dir.handle, sub_path_posix, .{
3352	.CLOEXEC = true,
3353	.PATH = true,
3354	}, 0) catch \|err\| switch (err) {
3355	error.WouldBlock => return errnoBug(.AGAIN),
3356	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
3357	else => \|e\| return e,
3358	};
3359	defer ev.close(maybe_sync.cancelRegion(), fd);
3360	return ev.realPath(try maybe_sync.enterSync(ev), fd, out_buffer);
3361	}
3362
3363	fn dirDeleteFile(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteFileError!void {
3364	const ev: *Evented = @ptrCast(@alignCast(userdata));
3365
3366	var path_buffer: [PATH_MAX]u8 = undefined;
3367	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3368
3369	var cancel_region: CancelRegion = .init();
3370	defer cancel_region.deinit();
3371	while (true) {
3372	const thread = try cancel_region.awaitIoUring();
3373	thread.enqueue().* = .{
3374	.opcode = .UNLINKAT,
3375	.flags = 0,
3376	.ioprio = 0,
3377	.fd = dir.handle,
3378	.off = 0,
3379	.addr = @intFromPtr(sub_path_posix.ptr),
3380	.len = 0,
3381	.rw_flags = 0,
3382	.user_data = @intFromPtr(cancel_region.fiber),
3383	.buf_index = 0,
3384	.personality = 0,
3385	.splice_fd_in = 0,
3386	.addr3 = 0,
3387	.resv = 0,
3388	};
3389	ev.yield(null, .nothing);
3390	switch (cancel_region.errno()) {
3391	.SUCCESS => return,
3392	.INTR, .CANCELED => continue,
3393	.PERM => return error.PermissionDenied,
3394	.ACCES => return error.AccessDenied,
3395	.BUSY => return error.FileBusy,
3396	.FAULT => \|err\| return errnoBug(err),
3397	.IO => return error.FileSystem,
3398	.ISDIR => return error.IsDir,
3399	.LOOP => return error.SymLinkLoop,
3400	.NAMETOOLONG => return error.NameTooLong,
3401	.NOENT => return error.FileNotFound,
3402	.NOTDIR => return error.NotDir,
3403	.NOMEM => return error.SystemResources,
3404	.ROFS => return error.ReadOnlyFileSystem,
3405	.EXIST => \|err\| return errnoBug(err),
3406	.NOTEMPTY => \|err\| return errnoBug(err), // Not passing AT.REMOVEDIR
3407	.ILSEQ => return error.BadPathName,
3408	.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
3409	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
3410	else => \|err\| return unexpectedErrno(err),
3411	}
3412	}
3413	}
3414
3415	fn dirDeleteDir(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteDirError!void {
3416	const ev: *Evented = @ptrCast(@alignCast(userdata));
3417
3418	var path_buffer: [PATH_MAX]u8 = undefined;
3419	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3420
3421	var cancel_region: CancelRegion = .init();
3422	defer cancel_region.deinit();
3423	while (true) {
3424	const thread = try cancel_region.awaitIoUring();
3425	thread.enqueue().* = .{
3426	.opcode = .UNLINKAT,
3427	.flags = 0,
3428	.ioprio = 0,
3429	.fd = dir.handle,
3430	.off = 0,
3431	.addr = @intFromPtr(sub_path_posix.ptr),
3432	.len = 0,
3433	.rw_flags = linux.AT.REMOVEDIR,
3434	.user_data = @intFromPtr(cancel_region.fiber),
3435	.buf_index = 0,
3436	.personality = 0,
3437	.splice_fd_in = 0,
3438	.addr3 = 0,
3439	.resv = 0,
3440	};
3441	ev.yield(null, .nothing);
3442	switch (cancel_region.errno()) {
3443	.SUCCESS => return,
3444	.INTR, .CANCELED => continue,
3445	.ACCES => return error.AccessDenied,
3446	.PERM => return error.PermissionDenied,
3447	.BUSY => return error.FileBusy,
3448	.FAULT => \|err\| return errnoBug(err),
3449	.IO => return error.FileSystem,
3450	.ISDIR => \|err\| return errnoBug(err),
3451	.LOOP => return error.SymLinkLoop,
3452	.NAMETOOLONG => return error.NameTooLong,
3453	.NOENT => return error.FileNotFound,
3454	.NOTDIR => return error.NotDir,
3455	.NOMEM => return error.SystemResources,
3456	.ROFS => return error.ReadOnlyFileSystem,
3457	.EXIST => \|err\| return errnoBug(err),
3458	.NOTEMPTY => return error.DirNotEmpty,
3459	.ILSEQ => return error.BadPathName,
3460	.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
3461	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
3462	else => \|err\| return unexpectedErrno(err),
3463	}
3464	}
3465	}
3466
3467	fn dirRename(
3468	userdata: ?*anyopaque,
3469	old_dir: Dir,
3470	old_sub_path: []const u8,
3471	new_dir: Dir,
3472	new_sub_path: []const u8,
3473	) Dir.RenameError!void {
3474	const ev: *Evented = @ptrCast(@alignCast(userdata));
3475
3476	var old_path_buffer: [PATH_MAX]u8 = undefined;
3477	var new_path_buffer: [PATH_MAX]u8 = undefined;
3478
3479	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
3480	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
3481
3482	var cancel_region: CancelRegion = .init();
3483	defer cancel_region.deinit();
3484	return ev.renameat(
3485	&cancel_region,
3486	old_dir.handle,
3487	old_sub_path_posix,
3488	new_dir.handle,
3489	new_sub_path_posix,
3490	.{},
3491	);
3492	}
3493
3494	fn dirRenamePreserve(
3495	userdata: ?*anyopaque,
3496	old_dir: Dir,
3497	old_sub_path: []const u8,
3498	new_dir: Dir,
3499	new_sub_path: []const u8,
3500	) Dir.RenamePreserveError!void {
3501	const ev: *Evented = @ptrCast(@alignCast(userdata));
3502
3503	var old_path_buffer: [PATH_MAX]u8 = undefined;
3504	var new_path_buffer: [PATH_MAX]u8 = undefined;
3505
3506	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
3507	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
3508
3509	var cancel_region: CancelRegion = .init();
3510	defer cancel_region.deinit();
3511	return ev.renameat(
3512	&cancel_region,
3513	old_dir.handle,
3514	old_sub_path_posix,
3515	new_dir.handle,
3516	new_sub_path_posix,
3517	.{ .NOREPLACE = true },
3518	);
3519	}
3520
3521	fn dirSymLink(
3522	userdata: ?*anyopaque,
3523	dir: Dir,
3524	target_path: []const u8,
3525	sym_link_path: []const u8,
3526	flags: Dir.SymLinkFlags,
3527	) Dir.SymLinkError!void {
3528	const ev: *Evented = @ptrCast(@alignCast(userdata));
3529	_ = flags;
3530
3531	var target_path_buffer: [PATH_MAX]u8 = undefined;
3532	var sym_link_path_buffer: [PATH_MAX]u8 = undefined;
3533
3534	const target_path_posix = try pathToPosix(target_path, &target_path_buffer);
3535	const sym_link_path_posix = try pathToPosix(sym_link_path, &sym_link_path_buffer);
3536
3537	var cancel_region: CancelRegion = .init();
3538	defer cancel_region.deinit();
3539	while (true) {
3540	const thread = try cancel_region.awaitIoUring();
3541	thread.enqueue().* = .{
3542	.opcode = .SYMLINKAT,
3543	.flags = 0,
3544	.ioprio = 0,
3545	.fd = dir.handle,
3546	.off = @intFromPtr(sym_link_path_posix.ptr),
3547	.addr = @intFromPtr(target_path_posix.ptr),
3548	.len = 0,
3549	.rw_flags = 0,
3550	.user_data = @intFromPtr(cancel_region.fiber),
3551	.buf_index = 0,
3552	.personality = 0,
3553	.splice_fd_in = 0,
3554	.addr3 = 0,
3555	.resv = 0,
3556	};
3557	ev.yield(null, .nothing);
3558	switch (cancel_region.errno()) {
3559	.SUCCESS => return,
3560	.INTR, .CANCELED => continue,
3561	.FAULT => \|err\| return errnoBug(err),
3562	.INVAL => \|err\| return errnoBug(err),
3563	.ACCES => return error.AccessDenied,
3564	.PERM => return error.PermissionDenied,
3565	.DQUOT => return error.DiskQuota,
3566	.EXIST => return error.PathAlreadyExists,
3567	.IO => return error.FileSystem,
3568	.LOOP => return error.SymLinkLoop,
3569	.NAMETOOLONG => return error.NameTooLong,
3570	.NOENT => return error.FileNotFound,
3571	.NOTDIR => return error.NotDir,
3572	.NOMEM => return error.SystemResources,
3573	.NOSPC => return error.NoSpaceLeft,
3574	.ROFS => return error.ReadOnlyFileSystem,
3575	.ILSEQ => return error.BadPathName,
3576	else => \|err\| return unexpectedErrno(err),
3577	}
3578	}
3579	}
3580
3581	fn dirReadLink(
3582	userdata: ?*anyopaque,
3583	dir: Dir,
3584	sub_path: []const u8,
3585	buffer: []u8,
3586	) Dir.ReadLinkError!usize {
3587	const ev: *Evented = @ptrCast(@alignCast(userdata));
3588
3589	var sub_path_buffer: [PATH_MAX]u8 = undefined;
3590	const sub_path_posix = try pathToPosix(sub_path, &sub_path_buffer);
3591
3592	var sync: CancelRegion.Sync = try .init(ev);
3593	defer sync.deinit(ev);
3594	while (true) {
3595	try sync.cancel_region.await(.nothing);
3596	const rc = linux.readlinkat(dir.handle, sub_path_posix, buffer.ptr, buffer.len);
3597	switch (linux.errno(rc)) {
3598	.SUCCESS => {
3599	const len: usize = @bitCast(rc);
3600	return len;
3601	},
3602	.INTR => continue,
3603	.ACCES => return error.AccessDenied,
3604	.FAULT => \|err\| return errnoBug(err),
3605	.INVAL => return error.NotLink,
3606	.IO => return error.FileSystem,
3607	.LOOP => return error.SymLinkLoop,
3608	.NAMETOOLONG => return error.NameTooLong,
3609	.NOENT => return error.FileNotFound,
3610	.NOMEM => return error.SystemResources,
3611	.NOTDIR => return error.NotDir,
3612	.ILSEQ => return error.BadPathName,
3613	else => \|err\| return unexpectedErrno(err),
3614	}
3615	}
3616	}
3617
3618	fn dirSetOwner(
3619	userdata: ?*anyopaque,
3620	dir: Dir,
3621	owner: ?File.Uid,
3622	group: ?File.Gid,
3623	) Dir.SetOwnerError!void {
3624	const ev: *Evented = @ptrCast(@alignCast(userdata));
3625	var sync: CancelRegion.Sync = try .init(ev);
3626	defer sync.deinit(ev);
3627	try ev.fchownat(
3628	&sync,
3629	dir.handle,
3630	"",
3631	owner orelse std.math.maxInt(linux.uid_t),
3632	group orelse std.math.maxInt(linux.gid_t),
3633	linux.AT.EMPTY_PATH,
3634	);
3635	}
3636
3637	fn dirSetFileOwner(
3638	userdata: ?*anyopaque,
3639	dir: Dir,
3640	sub_path: []const u8,
3641	owner: ?File.Uid,
3642	group: ?File.Gid,
3643	options: Dir.SetFileOwnerOptions,
3644	) Dir.SetFileOwnerError!void {
3645	const ev: *Evented = @ptrCast(@alignCast(userdata));
3646	var path_buffer: [PATH_MAX]u8 = undefined;
3647	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3648	var sync: CancelRegion.Sync = try .init(ev);
3649	defer sync.deinit(ev);
3650	try ev.fchownat(
3651	&sync,
3652	dir.handle,
3653	sub_path_posix,
3654	owner orelse std.math.maxInt(linux.uid_t),
3655	group orelse std.math.maxInt(linux.gid_t),
3656	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
3657	);
3658	}
3659
3660	fn dirSetPermissions(
3661	userdata: ?*anyopaque,
3662	dir: Dir,
3663	permissions: Dir.Permissions,
3664	) Dir.SetPermissionsError!void {
3665	const ev: *Evented = @ptrCast(@alignCast(userdata));
3666	var sync: CancelRegion.Sync = try .init(ev);
3667	defer sync.deinit(ev);
3668	ev.fchmodat(
3669	&sync,
3670	dir.handle,
3671	"",
3672	permissions.toMode(),
3673	linux.AT.EMPTY_PATH,
3674	) catch \|err\| switch (err) {
3675	error.NameTooLong => return errnoBug(.NAMETOOLONG),
3676	error.BadPathName => return errnoBug(.ILSEQ),
3677	error.ProcessFdQuotaExceeded => return errnoBug(.MFILE),
3678	error.SystemFdQuotaExceeded => return errnoBug(.NFILE),
3679	error.OperationUnsupported => return errnoBug(.OPNOTSUPP),
3680	else => \|e\| return e,
3681	};
3682	}
3683
3684	fn dirSetFilePermissions(
3685	userdata: ?*anyopaque,
3686	dir: Dir,
3687	sub_path: []const u8,
3688	permissions: Dir.Permissions,
3689	options: Dir.SetFilePermissionsOptions,
3690	) Dir.SetFilePermissionsError!void {
3691	const ev: *Evented = @ptrCast(@alignCast(userdata));
3692	var path_buffer: [PATH_MAX]u8 = undefined;
3693	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3694	var sync: CancelRegion.Sync = try .init(ev);
3695	defer sync.deinit(ev);
3696	try ev.fchmodat(
3697	&sync,
3698	dir.handle,
3699	sub_path_posix,
3700	permissions.toMode(),
3701	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
3702	);
3703	}
3704
3705	fn dirSetTimestamps(
3706	userdata: ?*anyopaque,
3707	dir: Dir,
3708	sub_path: []const u8,
3709	options: Dir.SetTimestampsOptions,
3710	) Dir.SetTimestampsError!void {
3711	const ev: *Evented = @ptrCast(@alignCast(userdata));
3712	var path_buffer: [PATH_MAX]u8 = undefined;
3713	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3714	var cancel_region: CancelRegion.Sync = try .init(ev);
3715	defer cancel_region.deinit(ev);
3716	try ev.utimensat(
3717	&cancel_region,
3718	dir.handle,
3719	sub_path_posix,
3720	if (options.modify_timestamp != .now or options.access_timestamp != .now) &.{
3721	setTimestampToPosix(options.access_timestamp),
3722	setTimestampToPosix(options.modify_timestamp),
3723	} else null,
3724	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
3725	);
3726	}
3727
3728	fn dirHardLink(
3729	userdata: ?*anyopaque,
3730	old_dir: Dir,
3731	old_sub_path: []const u8,
3732	new_dir: Dir,
3733	new_sub_path: []const u8,
3734	options: Dir.HardLinkOptions,
3735	) Dir.HardLinkError!void {
3736	const ev: *Evented = @ptrCast(@alignCast(userdata));
3737
3738	var old_path_buffer: [PATH_MAX]u8 = undefined;
3739	var new_path_buffer: [PATH_MAX]u8 = undefined;
3740
3741	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
3742	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
3743
3744	var cancel_region: CancelRegion = .init();
3745	defer cancel_region.deinit();
3746	return ev.linkat(
3747	&cancel_region,
3748	old_dir.handle,
3749	old_sub_path_posix,
3750	new_dir.handle,
3751	new_sub_path_posix,
3752	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
3753	);
3754	}
3755
3756	fn fileStat(userdata: ?*anyopaque, file: File) File.StatError!File.Stat {
3757	const ev: *Evented = @ptrCast(@alignCast(userdata));
3758	var cancel_region: CancelRegion = .init();
3759	defer cancel_region.deinit();
3760	return ev.stat(&cancel_region, file.handle);
3761	}
3762
3763	fn fileLength(userdata: ?*anyopaque, file: File) File.LengthError!u64 {
3764	const ev: *Evented = @ptrCast(@alignCast(userdata));
3765	var cancel_region: CancelRegion = .init();
3766	defer cancel_region.deinit();
3767	while (true) {
3768	var statx_buf = std.mem.zeroes(linux.Statx);
3769	const thread = try cancel_region.awaitIoUring();
3770	thread.enqueue().* = .{
3771	.opcode = .STATX,
3772	.flags = 0,
3773	.ioprio = 0,
3774	.fd = file.handle,
3775	.off = @intFromPtr(&statx_buf),
3776	.addr = @intFromPtr(""),
3777	.len = @bitCast(linux.STATX{ .SIZE = true }),
3778	.rw_flags = linux.AT.EMPTY_PATH,
3779	.user_data = @intFromPtr(cancel_region.fiber),
3780	.buf_index = 0,
3781	.personality = 0,
3782	.splice_fd_in = 0,
3783	.addr3 = 0,
3784	.resv = 0,
3785	};
3786	ev.yield(null, .nothing);
3787	switch (cancel_region.errno()) {
3788	.SUCCESS => {
3789	if (!statx_buf.mask.SIZE) return error.Unexpected;
3790	return statx_buf.size;
3791	},
3792	.INTR, .CANCELED => continue,
3793	.ACCES => \|err\| return errnoBug(err),
3794	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
3795	.FAULT => \|err\| return errnoBug(err),
3796	.INVAL => \|err\| return errnoBug(err),
3797	.LOOP => \|err\| return errnoBug(err),
3798	.NAMETOOLONG => \|err\| return errnoBug(err),
3799	.NOENT => \|err\| return errnoBug(err),
3800	.NOMEM => return error.SystemResources,
3801	.NOTDIR => \|err\| return errnoBug(err),
3802	else => \|err\| return unexpectedErrno(err),
3803	}
3804	}
3805	}
3806
3807	fn fileClose(userdata: ?*anyopaque, files: []const File) void {
3808	const ev: *Evented = @ptrCast(@alignCast(userdata));
3809	var cancel_region: CancelRegion = .init();
3810	defer cancel_region.deinit();
3811	for (files) \|file\| ev.close(&cancel_region, file.handle);
3812	}
3813
3814	fn fileWritePositional(
3815	userdata: ?*anyopaque,
3816	file: File,
3817	header: []const u8,
3818	data: []const []const u8,
3819	splat: usize,
3820	offset: u64,
3821	) File.WritePositionalError!usize {
3822	const ev: *Evented = @ptrCast(@alignCast(userdata));
3823
3824	var iovecs: [max_iovecs_len]iovec_const = undefined;
3825	var iovlen: iovlen_t = 0;
3826	addBuf(&iovecs, &iovlen, header);
3827	for (data[0 .. data.len - 1]) \|bytes\| addBuf(&iovecs, &iovlen, bytes);
3828	const pattern = data[data.len - 1];
3829	if (iovecs.len - iovlen != 0) switch (splat) {
3830	0 => {},
3831	1 => addBuf(&iovecs, &iovlen, pattern),
3832	else => switch (pattern.len) {
3833	0 => {},
3834	1 => {
3835	var backup_buffer: [splat_buffer_size]u8 = undefined;
3836	const splat_buffer = &backup_buffer;
3837	const memset_len = @min(splat_buffer.len, splat);
3838	const buf = splat_buffer[0..memset_len];
3839	@memset(buf, pattern[0]);
3840	addBuf(&iovecs, &iovlen, buf);
3841	var remaining_splat = splat - buf.len;
3842	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0) {
3843	assert(buf.len == splat_buffer.len);
3844	addBuf(&iovecs, &iovlen, splat_buffer);
3845	remaining_splat -= splat_buffer.len;
3846	}
3847	addBuf(&iovecs, &iovlen, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
3848	},
3849	else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
3850	addBuf(&iovecs, &iovlen, pattern);
3851	},
3852	},
3853	};
3854
3855	var cancel_region: CancelRegion = .init();
3856	defer cancel_region.deinit();
3857	return ev.pwritev(&cancel_region, file.handle, iovecs[0..iovlen], offset);
3858	}
3859
3860	/// This is either usize or u32. Since, either is fine, let's use the same
3861	/// `addBuf` function for both writing to a file and sending network messages.
3862	const iovlen_t = @FieldType(linux.msghdr_const, "iovlen");
3863
3864	fn addBuf(v: []iovec_const, i: *iovlen_t, bytes: []const u8) void {
3865	// OS checks ptr addr before length so zero length vectors must be omitted.
3866	if (bytes.len == 0) return;
3867	if (v.len - i.* == 0) return;
3868	v[i.*] = .{ .base = bytes.ptr, .len = bytes.len };
3869	i.* += 1;
3870	}
3871
3872	fn fileWriteFileStreaming(
3873	userdata: ?*anyopaque,
3874	file: File,
3875	header: []const u8,
3876	file_reader: *File.Reader,
3877	limit: Io.Limit,
3878	) File.Writer.WriteFileError!usize {
3879	const ev: *Evented = @ptrCast(@alignCast(userdata));
3880	_ = ev;
3881	_ = file;
3882	_ = header;
3883	_ = file_reader;
3884	_ = limit;
3885	return error.Unimplemented;
3886	}
3887
3888	fn fileWriteFilePositional(
3889	userdata: ?*anyopaque,
3890	file: File,
3891	header: []const u8,
3892	file_reader: *File.Reader,
3893	limit: Io.Limit,
3894	offset: u64,
3895	) File.WriteFilePositionalError!usize {
3896	const ev: *Evented = @ptrCast(@alignCast(userdata));
3897	_ = ev;
3898	_ = file;
3899	_ = header;
3900	_ = file_reader;
3901	_ = limit;
3902	_ = offset;
3903	return error.Unimplemented;
3904	}
3905
3906	fn fileReadPositional(
3907	userdata: ?*anyopaque,
3908	file: File,
3909	data: []const []u8,
3910	offset: u64,
3911	) File.ReadPositionalError!usize {
3912	const ev: *Evented = @ptrCast(@alignCast(userdata));
3913
3914	var iovecs_buffer: [max_iovecs_len]iovec = undefined;
3915	var i: usize = 0;
3916	for (data) \|buf\| {
3917	if (iovecs_buffer.len - i == 0) break;
3918	if (buf.len != 0) {
3919	iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len };
3920	i += 1;
3921	}
3922	}
3923	if (i == 0) return 0;
3924	const dest = iovecs_buffer[0..i];
3925	assert(dest[0].len > 0);
3926
3927	var cancel_region: CancelRegion = .init();
3928	defer cancel_region.deinit();
3929	return ev.preadv(&cancel_region, file.handle, dest, offset) catch \|err\| switch (err) {
3930	error.SocketUnconnected => errnoBug(.NOTCONN), // not a socket
3931	error.ConnectionResetByPeer => errnoBug(.CONNRESET), // not a socket
3932	else => \|e\| e,
3933	};
3934	}
3935
3936	fn fileSeekBy(userdata: ?*anyopaque, file: File, offset: i64) File.SeekError!void {
3937	const ev: *Evented = @ptrCast(@alignCast(userdata));
3938	var sync: CancelRegion.Sync = try .init(ev);
3939	defer sync.deinit(ev);
3940	try ev.lseek(&sync, file.handle, @bitCast(offset), linux.SEEK.CUR);
3941	}
3942
3943	fn fileSeekTo(userdata: ?*anyopaque, file: File, offset: u64) File.SeekError!void {
3944	const ev: *Evented = @ptrCast(@alignCast(userdata));
3945	var sync: CancelRegion.Sync = try .init(ev);
3946	defer sync.deinit(ev);
3947	try ev.lseek(&sync, file.handle, offset, linux.SEEK.SET);
3948	}
3949
3950	fn fileSync(userdata: ?*anyopaque, file: File) File.SyncError!void {
3951	const ev: *Evented = @ptrCast(@alignCast(userdata));
3952	var cancel_region: CancelRegion = .init();
3953	defer cancel_region.deinit();
3954	while (true) {
3955	const thread = try cancel_region.awaitIoUring();
3956	thread.enqueue().* = .{
3957	.opcode = .FSYNC,
3958	.flags = 0,
3959	.ioprio = 0,
3960	.fd = file.handle,
3961	.off = 0,
3962	.addr = 0,
3963	.len = 0,
3964	.rw_flags = 0,
3965	.user_data = @intFromPtr(cancel_region.fiber),
3966	.buf_index = 0,
3967	.personality = 0,
3968	.splice_fd_in = 0,
3969	.addr3 = 0,
3970	.resv = 0,
3971	};
3972	ev.yield(null, .nothing);
3973	switch (cancel_region.errno()) {
3974	.SUCCESS => return,
3975	.INTR, .CANCELED => continue,
3976	.BADF => \|err\| return errnoBug(err),
3977	.INVAL => \|err\| return errnoBug(err),
3978	.ROFS => \|err\| return errnoBug(err),
3979	.IO => return error.InputOutput,
3980	.NOSPC => return error.NoSpaceLeft,
3981	.DQUOT => return error.DiskQuota,
3982	else => \|err\| return unexpectedErrno(err),
3983	}
3984	}
3985	}
3986
3987	fn fileIsTty(userdata: ?*anyopaque, file: File) Io.Cancelable!bool {
3988	const ev: *Evented = @ptrCast(@alignCast(userdata));
3989	var sync: CancelRegion.Sync = try .init(ev);
3990	defer sync.deinit(ev);
3991	while (true) {
3992	try sync.cancel_region.await(.nothing);
3993	var wsz: winsize = undefined;
3994	const rc = linux.ioctl(file.handle, linux.T.IOCGWINSZ, @intFromPtr(&wsz));
3995	switch (linux.errno(rc)) {
3996	.SUCCESS => return true,
3997	.INTR => continue,
3998	else => return false,
3999	}
4000	}
4001	}
4002
4003	fn fileEnableAnsiEscapeCodes(userdata: ?*anyopaque, file: File) File.EnableAnsiEscapeCodesError!void {
4004	const ev: *Evented = @ptrCast(@alignCast(userdata));
4005	if (!try fileIsTty(ev, file)) return error.NotTerminalDevice;
4006	}
4007
4008	fn fileSetLength(userdata: ?*anyopaque, file: File, length: u64) File.SetLengthError!void {
4009	const ev: *Evented = @ptrCast(@alignCast(userdata));
4010	var cancel_region: CancelRegion = .init();
4011	defer cancel_region.deinit();
4012	while (true) {
4013	const thread = try cancel_region.awaitIoUring();
4014	thread.enqueue().* = .{
4015	.opcode = .FTRUNCATE,
4016	.flags = 0,
4017	.ioprio = 0,
4018	.fd = file.handle,
4019	.off = length,
4020	.addr = 0,
4021	.len = 0,
4022	.rw_flags = 0,
4023	.user_data = @intFromPtr(cancel_region.fiber),
4024	.buf_index = 0,
4025	.personality = 0,
4026	.splice_fd_in = 0,
4027	.addr3 = 0,
4028	.resv = 0,
4029	};
4030	ev.yield(null, .nothing);
4031	switch (cancel_region.errno()) {
4032	.SUCCESS => return,
4033	.INTR, .CANCELED => continue,
4034	.FBIG => return error.FileTooBig,
4035	.IO => return error.InputOutput,
4036	.PERM => return error.PermissionDenied,
4037	.TXTBSY => return error.FileBusy,
4038	.BADF => \|err\| return errnoBug(err), // Handle not open for writing.
4039	.INVAL => return error.NonResizable, // This is returned for /dev/null for example.
4040	else => \|err\| return unexpectedErrno(err),
4041	}
4042	}
4043	}
4044
4045	fn fileSetOwner(
4046	userdata: ?*anyopaque,
4047	file: File,
4048	owner: ?File.Uid,
4049	group: ?File.Gid,
4050	) File.SetOwnerError!void {
4051	const ev: *Evented = @ptrCast(@alignCast(userdata));
4052	var sync: CancelRegion.Sync = try .init(ev);
4053	defer sync.deinit(ev);
4054	try ev.fchownat(
4055	&sync,
4056	file.handle,
4057	"",
4058	owner orelse std.math.maxInt(linux.uid_t),
4059	group orelse std.math.maxInt(linux.gid_t),
4060	linux.AT.EMPTY_PATH,
4061	);
4062	}
4063
4064	fn fileSetPermissions(
4065	userdata: ?*anyopaque,
4066	file: File,
4067	permissions: File.Permissions,
4068	) File.SetPermissionsError!void {
4069	const ev: *Evented = @ptrCast(@alignCast(userdata));
4070	var sync: CancelRegion.Sync = try .init(ev);
4071	defer sync.deinit(ev);
4072	ev.fchmodat(
4073	&sync,
4074	file.handle,
4075	"",
4076	permissions.toMode(),
4077	linux.AT.EMPTY_PATH,
4078	) catch \|err\| switch (err) {
4079	error.NameTooLong => return errnoBug(.NAMETOOLONG),
4080	error.BadPathName => return errnoBug(.ILSEQ),
4081	error.ProcessFdQuotaExceeded => return errnoBug(.MFILE),
4082	error.SystemFdQuotaExceeded => return errnoBug(.NFILE),
4083	error.OperationUnsupported => return errnoBug(.OPNOTSUPP),
4084	else => \|e\| return e,
4085	};
4086	}
4087
4088	fn fileSetTimestamps(
4089	userdata: ?*anyopaque,
4090	file: File,
4091	options: File.SetTimestampsOptions,
4092	) File.SetTimestampsError!void {
4093	const ev: *Evented = @ptrCast(@alignCast(userdata));
4094	var sync: CancelRegion.Sync = try .init(ev);
4095	defer sync.deinit(ev);
4096	try ev.utimensat(
4097	&sync,
4098	file.handle,
4099	"",
4100	if (options.modify_timestamp != .now or options.access_timestamp != .now) &.{
4101	setTimestampToPosix(options.access_timestamp),
4102	setTimestampToPosix(options.modify_timestamp),
4103	} else null,
4104	linux.AT.EMPTY_PATH,
4105	);
4106	}
4107
4108	fn fileLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!void {
4109	const ev: *Evented = @ptrCast(@alignCast(userdata));
4110	var sync: CancelRegion.Sync = try .init(ev);
4111	defer sync.deinit(ev);
4112	ev.flock(&sync, file.handle, lock, .blocking) catch \|err\| switch (err) {
4113	error.WouldBlock => unreachable, // blocking
4114	else => \|e\| return e,
4115	};
4116	}
4117
4118	fn fileTryLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!bool {
4119	const ev: *Evented = @ptrCast(@alignCast(userdata));
4120	var sync: CancelRegion.Sync = try .init(ev);
4121	defer sync.deinit(ev);
4122	ev.flock(&sync, file.handle, lock, switch (lock) {
4123	.none => .blocking,
4124	.shared, .exclusive => .nonblocking,
4125	}) catch \|err\| switch (err) {
4126	error.WouldBlock => return false,
4127	else => \|e\| return e,
4128	};
4129	return true;
4130	}
4131
4132	fn fileUnlock(userdata: ?*anyopaque, file: File) void {
4133	const ev: *Evented = @ptrCast(@alignCast(userdata));
4134	var sync: CancelRegion.Sync = .initBlocked(ev);
4135	defer sync.deinit(ev);
4136	ev.flock(&sync, file.handle, .none, .blocking) catch \|err\| switch (err) {
4137	error.Canceled => unreachable, // blocked
4138	error.WouldBlock => unreachable, // blocking
4139	error.SystemResources => return recoverableOsBugDetected(), // Resource deallocation.
4140	error.FileLocksUnsupported => return recoverableOsBugDetected(), // We already got the lock.
4141	error.Unexpected => return recoverableOsBugDetected(), // Resource deallocation must succeed.
4142	};
4143	}
4144
4145	fn fileDowngradeLock(userdata: ?*anyopaque, file: File) File.DowngradeLockError!void {
4146	const ev: *Evented = @ptrCast(@alignCast(userdata));
4147	var sync: CancelRegion.Sync = try .init(ev);
4148	defer sync.deinit(ev);
4149	ev.flock(&sync, file.handle, .shared, .nonblocking) catch \|err\| switch (err) {
4150	error.WouldBlock => return errnoBug(.AGAIN), // File was not locked in exclusive mode.
4151	error.SystemResources => return errnoBug(.NOLCK), // Lock already obtained.
4152	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Lock already obtained.
4153	else => \|e\| return e,
4154	};
4155	}
4156
4157	fn fileRealPath(userdata: ?*anyopaque, file: File, out_buffer: []u8) File.RealPathError!usize {
4158	const ev: *Evented = @ptrCast(@alignCast(userdata));
4159	var sync: CancelRegion.Sync = try .init(ev);
4160	defer sync.deinit(ev);
4161	return ev.realPath(&sync, file.handle, out_buffer);
4162	}
4163
4164	fn fileHardLink(
4165	userdata: ?*anyopaque,
4166	file: File,
4167	new_dir: Dir,
4168	new_sub_path: []const u8,
4169	options: File.HardLinkOptions,
4170	) File.HardLinkError!void {
4171	const ev: *Evented = @ptrCast(@alignCast(userdata));
4172
4173	var new_path_buffer: [PATH_MAX]u8 = undefined;
4174	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
4175
4176	var cancel_region: CancelRegion = .init();
4177	defer cancel_region.deinit();
4178	return ev.linkat(
4179	&cancel_region,
4180	file.handle,
4181	"",
4182	new_dir.handle,
4183	new_sub_path_posix,
4184	linux.AT.EMPTY_PATH \| @as(u32, if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW),
4185	);
4186	}
4187
4188	fn fileMemoryMapCreate(
4189	userdata: ?*anyopaque,
4190	file: File,
4191	options: File.MemoryMap.CreateOptions,
4192	) File.MemoryMap.CreateError!File.MemoryMap {
4193	const ev: *Evented = @ptrCast(@alignCast(userdata));
4194
4195	const prot: linux.PROT = .{
4196	.READ = options.protection.read,
4197	.WRITE = options.protection.write,
4198	.EXEC = options.protection.execute,
4199	};
4200	const flags: linux.MAP = .{
4201	.TYPE = .SHARED_VALIDATE,
4202	.POPULATE = options.populate,
4203	};
4204
4205	const page_align = std.heap.page_size_min;
4206
4207	var sync: CancelRegion.Sync = try .init(ev);
4208	defer sync.deinit(ev);
4209	const contents = while (true) {
4210	try sync.cancel_region.await(.nothing);
4211	const casted_offset = std.math.cast(i64, options.offset) orelse return error.Unseekable;
4212	const rc = linux.mmap(null, options.len, prot, flags, file.handle, casted_offset);
4213	switch (linux.errno(rc)) {
4214	.SUCCESS => break @as([*]align(page_align) u8, @ptrFromInt(rc))[0..options.len],
4215	.INTR => continue,
4216	.ACCES => return error.AccessDenied,
4217	.AGAIN => return error.LockedMemoryLimitExceeded,
4218	.MFILE => return error.ProcessFdQuotaExceeded,
4219	.NFILE => return error.SystemFdQuotaExceeded,
4220	.NOMEM => return error.OutOfMemory,
4221	.PERM => return error.PermissionDenied,
4222	.OVERFLOW => return error.Unseekable,
4223	.BADF => \|err\| return errnoBug(err), // Always a race condition.
4224	.INVAL => \|err\| return errnoBug(err), // Invalid parameters to mmap()
4225	.OPNOTSUPP => \|err\| return errnoBug(err), // Bad flags with MAP.SHARED_VALIDATE on Linux.
4226	else => \|err\| return unexpectedErrno(err),
4227	}
4228	};
4229	return .{
4230	.file = file,
4231	.offset = options.offset,
4232	.memory = contents,
4233	.section = {},
4234	};
4235	}
4236
4237	fn fileMemoryMapDestroy(userdata: ?anyopaque, mm: File.MemoryMap) void {
4238	const ev: *Evented = @ptrCast(@alignCast(userdata));
4239	_ = ev;
4240	const memory = mm.memory;
4241	if (memory.len == 0) return;
4242	switch (linux.errno(linux.munmap(memory.ptr, memory.len))) {
4243	.SUCCESS => {},
4244	else => \|err\| if (builtin.mode == .Debug)
4245	std.log.err("failed to unmap {d} bytes at {*}: {t}", .{ memory.len, memory.ptr, err }),
4246	}
4247	mm.* = undefined;
4248	}
4249
4250	fn processExecutableOpen(
4251	userdata: ?*anyopaque,
4252	flags: File.OpenFlags,
4253	) process.OpenExecutableError!File {
4254	const ev: *Evented = @ptrCast(@alignCast(userdata));
4255	return dirOpenFile(ev, .{ .handle = linux.AT.FDCWD }, "/proc/self/exe", flags);
4256	}
4257
4258	fn processExecutablePath(userdata: ?*anyopaque, out_buffer: []u8) process.ExecutablePathError!usize {
4259	const ev: *Evented = @ptrCast(@alignCast(userdata));
4260	return dirReadLink(ev, .cwd(), "/proc/self/exe", out_buffer) catch \|err\| switch (err) {
4261	error.UnsupportedReparsePointType => unreachable, // Windows-only
4262	error.NetworkNotFound => unreachable, // Windows-only
4263	error.FileBusy => unreachable, // Windows-only
4264	else => \|e\| return e,
4265	};
4266	}
4267
4268	fn lockStderr(userdata: ?*anyopaque, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
4269	const ev: *Evented = @ptrCast(@alignCast(userdata));
4270	const ev_io = ev.io();
4271	ev.stderr_mutex.lockUncancelable(ev_io);
4272	errdefer ev.stderr_mutex.unlock(ev_io);
4273	return ev.initLockedStderr(terminal_mode);
4274	}
4275
4276	fn tryLockStderr(
4277	userdata: ?*anyopaque,
4278	terminal_mode: ?Io.Terminal.Mode,
4279	) Io.Cancelable!?Io.LockedStderr {
4280	const ev: *Evented = @ptrCast(@alignCast(userdata));
4281	const ev_io = ev.io();
4282	if (!ev.stderr_mutex.tryLock()) return null;
4283	errdefer ev.stderr_mutex.unlock(ev_io);
4284	return try ev.initLockedStderr(terminal_mode);
4285	}
4286
4287	fn initLockedStderr(ev: *Evented, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
4288	if (!ev.stderr_writer_initialized) {
4289	const ev_io = ev.io();
4290	try ev.scanEnviron();
4291	const NO_COLOR = ev.environ.exist.NO_COLOR;
4292	const CLICOLOR_FORCE = ev.environ.exist.CLICOLOR_FORCE;
4293	ev.stderr_mode = terminal_mode orelse
4294	try .detect(ev_io, ev.stderr_writer.file, NO_COLOR, CLICOLOR_FORCE);
4295	ev.stderr_writer_initialized = true;
4296	}
4297	return .{
4298	.file_writer = &ev.stderr_writer,
4299	.terminal_mode = terminal_mode orelse ev.stderr_mode,
4300	};
4301	}
4302
4303	fn unlockStderr(userdata: ?*anyopaque) void {
4304	const ev: *Evented = @ptrCast(@alignCast(userdata));
4305	ev.stderr_writer.interface.flush() catch \|err\| switch (err) {
4306	error.WriteFailed => switch (ev.stderr_writer.err.?) {
4307	error.Canceled => recancel(ev),
4308	else => {},
4309	},
4310	};
4311	ev.stderr_writer.interface.end = 0;
4312	ev.stderr_writer.interface.buffer = &.{};
4313	ev.stderr_mutex.unlock(ev.io());
4314	}
4315
4316	fn processCurrentPath(userdata: ?*anyopaque, buffer: []u8) process.CurrentPathError!usize {
4317	const ev: *Evented = @ptrCast(@alignCast(userdata));
4318	var sync: CancelRegion.Sync = try .init(ev);
4319	defer sync.deinit(ev);
4320	while (true) {
4321	try sync.cancel_region.await(.nothing);
4322	switch (linux.errno(linux.getcwd(buffer.ptr, buffer.len))) {
4323	.SUCCESS => return std.mem.findScalar(u8, buffer, 0).?,
4324	.INTR => continue,
4325	.NOENT => return error.CurrentDirUnlinked,
4326	.RANGE => return error.NameTooLong,
4327	.FAULT => \|err\| return errnoBug(err),
4328	.INVAL => \|err\| return errnoBug(err),
4329	else => \|err\| return unexpectedErrno(err),
4330	}
4331	}
4332	}
4333
4334	fn processSetCurrentDir(userdata: ?*anyopaque, dir: Dir) process.SetCurrentDirError!void {
4335	const ev: *Evented = @ptrCast(@alignCast(userdata));
4336	if (dir.handle == linux.AT.FDCWD) return;
4337	var sync: CancelRegion.Sync = try .init(ev);
4338	defer sync.deinit(ev);
4339	return ev.fchdir(&sync, dir.handle);
4340	}
4341
4342	fn processSetCurrentPath(userdata: ?*anyopaque, dir_path: []const u8) ChdirError!void {
4343	const ev: *Evented = @ptrCast(@alignCast(userdata));
4344	var path_buffer: [PATH_MAX]u8 = undefined;
4345	const dir_path_posix = try pathToPosix(dir_path, &path_buffer);
4346	var sync: CancelRegion.Sync = try .init(ev);
4347	defer sync.deinit(ev);
4348	return ev.chdir(&sync, dir_path_posix);
4349	}
4350
4351	fn processReplace(userdata: ?*anyopaque, options: process.ReplaceOptions) process.ReplaceError {
4352	const ev: *Evented = @ptrCast(@alignCast(userdata));
4353
4354	try ev.scanEnviron(); // for PATH
4355	const PATH = ev.environ.string.PATH orelse default_PATH;
4356
4357	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
4358	defer arena_allocator.deinit();
4359	const arena = arena_allocator.allocator();
4360
4361	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
4362	for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
4363
4364	const env_block = env_block: {
4365	const prog_fd: i32 = -1;
4366	if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
4367	.zig_progress_fd = prog_fd,
4368	});
4369	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
4370	.zig_progress_fd = prog_fd,
4371	});
4372	};
4373
4374	var sync: CancelRegion.Sync = try .init(ev);
4375	defer sync.deinit(ev);
4376	return ev.execv(&sync, options.expand_arg0, argv_buf.ptr[0].?, argv_buf.ptr, env_block, PATH);
4377	}
4378
4379	fn processReplacePath(
4380	userdata: ?*anyopaque,
4381	dir: Dir,
4382	options: process.ReplaceOptions,
4383	) process.ReplaceError {
4384	const ev: *Evented = @ptrCast(@alignCast(userdata));
4385	_ = ev;
4386	_ = dir;
4387	_ = options;
4388	@panic("TODO processReplacePath");
4389	}
4390
4391	fn processSpawn(userdata: ?*anyopaque, options: process.SpawnOptions) process.SpawnError!process.Child {
4392	const ev: *Evented = @ptrCast(@alignCast(userdata));
4393	const spawned = try ev.spawn(options);
4394	var cancel_region: CancelRegion = .initBlocked();
4395	defer cancel_region.deinit();
4396	defer ev.close(&cancel_region, spawned.err_fd);
4397
4398	// Wait for the child to report any errors in or before `execvpe`.
4399	var child_err: ForkBailError = undefined;
4400	ev.readAll(&cancel_region, spawned.err_fd, @ptrCast(&child_err)) catch \|read_err\| {
4401	switch (read_err) {
4402	error.Canceled => unreachable, // blocked
4403	error.EndOfStream => {
4404	// Write end closed by CLOEXEC at the time of the `execvpe` call,
4405	// indicating success.
4406	},
4407	else => {
4408	// Problem reading the error from the error reporting pipe. We
4409	// don't know if the child is alive or dead. Better to assume it is
4410	// alive so the resource does not risk being leaked.
4411	},
4412	}
4413	return .{
4414	.id = spawned.pid,
4415	.thread_handle = {},
4416	.stdin = spawned.stdin,
4417	.stdout = spawned.stdout,
4418	.stderr = spawned.stderr,
4419	.request_resource_usage_statistics = options.request_resource_usage_statistics,
4420	};
4421	};
4422	return child_err;
4423	}
4424
4425	fn processSpawnPath(
4426	userdata: ?*anyopaque,
4427	dir: Dir,
4428	options: process.SpawnOptions,
4429	) process.SpawnError!process.Child {
4430	const ev: *Evented = @ptrCast(@alignCast(userdata));
4431	_ = ev;
4432	_ = dir;
4433	_ = options;
4434	@panic("TODO processSpawnPath");
4435	}
4436
4437	const prog_fileno = 3;
4438
4439	const Spawned = struct {
4440	pid: pid_t,
4441	err_fd: fd_t,
4442	stdin: ?File,
4443	stdout: ?File,
4444	stderr: ?File,
4445	};
4446	fn spawn(ev: *Evented, options: process.SpawnOptions) process.SpawnError!Spawned {
4447	var cancel_region: CancelRegion = .init();
4448	defer cancel_region.deinit();
4449
4450	// The child process does need to access (one end of) these pipes. However,
4451	// we must initially set CLOEXEC to avoid a race condition. If another thread
4452	// is racing to spawn a different child process, we don't want it to inherit
4453	// these FDs in any scenario; that would mean that, for instance, calls to
4454	// `poll` from the parent would not report the child's stdout as closing when
4455	// expected, since the other child may retain a reference to the write end of
4456	// the pipe. So, we create the pipes with CLOEXEC initially. After fork, we
4457	// need to do something in the new child to make sure we preserve the reference
4458	// we want. We could use `fcntl` to remove CLOEXEC from the FD, but as it
4459	// turns out, we `dup2` everything anyway, so there's no need!
4460	const pipe_flags: linux.O = .{ .CLOEXEC = true };
4461
4462	const stdin_pipe = if (options.stdin == .pipe) try pipe2(pipe_flags) else undefined;
4463	errdefer if (options.stdin == .pipe) {
4464	ev.destroyPipe(&cancel_region, stdin_pipe);
4465	};
4466
4467	const stdout_pipe = if (options.stdout == .pipe) try pipe2(pipe_flags) else undefined;
4468	errdefer if (options.stdout == .pipe) {
4469	ev.destroyPipe(&cancel_region, stdout_pipe);
4470	};
4471
4472	const stderr_pipe = if (options.stderr == .pipe) try pipe2(pipe_flags) else undefined;
4473	errdefer if (options.stderr == .pipe) {
4474	ev.destroyPipe(&cancel_region, stderr_pipe);
4475	};
4476
4477	const any_ignore =
4478	options.stdin == .ignore or options.stdout == .ignore or options.stderr == .ignore;
4479	const dev_null_fd = if (any_ignore) try ev.null_fd.open(ev, &cancel_region, "/dev/null", .{
4480	.ACCMODE = .RDWR,
4481	}) else undefined;
4482
4483	const prog_pipe: [2]fd_t = if (options.progress_node.index != .none) pipe: {
4484	// We use CLOEXEC for the same reason as in `pipe_flags`.
4485	const pipe = try pipe2(.{ .NONBLOCK = true, .CLOEXEC = true });
4486	_ = linux.fcntl(pipe[0], linux.F.SETPIPE_SZ, @as(u32, std.Progress.max_packet_len * 2));
4487	break :pipe pipe;
4488	} else .{ -1, -1 };
4489	errdefer ev.destroyPipe(&cancel_region, prog_pipe);
4490
4491	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
4492	defer arena_allocator.deinit();
4493	const arena = arena_allocator.allocator();
4494
4495	// The POSIX standard does not allow malloc() between fork() and execve(),
4496	// and this allocator may be a libc allocator.
4497	// I have personally observed the child process deadlocking when it tries
4498	// to call malloc() due to a heap allocation between fork() and execve(),
4499	// in musl v1.1.24.
4500	// Additionally, we want to reduce the number of possible ways things
4501	// can fail between fork() and execve().
4502	// Therefore, we do all the allocation for the execve() before the fork().
4503	// This means we must do the null-termination of argv and env vars here.
4504	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
4505	for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
4506
4507	comptime assert(@max(linux.STDIN_FILENO, linux.STDOUT_FILENO, linux.STDERR_FILENO) + 1 == prog_fileno);
4508
4509	const env_block = env_block: {
4510	const prog_fd: i32 = if (prog_pipe[1] == -1) -1 else prog_fileno;
4511	if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
4512	.zig_progress_fd = prog_fd,
4513	});
4514	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
4515	.zig_progress_fd = prog_fd,
4516	});
4517	};
4518
4519	// This pipe communicates to the parent errors in the child between `fork` and `execvpe`.
4520	// It is closed by the child (via CLOEXEC) without writing if `execvpe` succeeds.
4521	const err_pipe: [2]fd_t = try pipe2(.{ .CLOEXEC = true });
4522	errdefer ev.destroyPipe(&cancel_region, err_pipe);
4523
4524	try ev.scanEnviron(); // for PATH
4525	const PATH = ev.environ.string.PATH orelse default_PATH;
4526
4527	const pid_result: pid_t = fork: {
4528	const rc = linux.fork();
4529	switch (linux.errno(rc)) {
4530	.SUCCESS => break :fork @intCast(rc),
4531	.AGAIN => return error.SystemResources,
4532	.NOMEM => return error.SystemResources,
4533	.NOSYS => return error.OperationUnsupported,
4534	else => \|err\| return unexpectedErrno(err),
4535	}
4536	};
4537
4538	if (pid_result == 0) {
4539	defer comptime unreachable; // We are the child.
4540	var sync: CancelRegion.Sync = .{ .cancel_region = .initBlocked() };
4541	const err = ev.setUpChild(&sync, .{
4542	.stdin_pipe = stdin_pipe[0],
4543	.stdout_pipe = stdout_pipe[1],
4544	.stderr_pipe = stderr_pipe[1],
4545	.dev_null_fd = dev_null_fd,
4546	.prog_pipe = prog_pipe[1],
4547	.argv_buf = argv_buf,
4548	.env_block = env_block,
4549	.PATH = PATH,
4550	.spawn = options,
4551	});
4552	ev.writeAll(&sync.cancel_region, err_pipe[1], @ptrCast(&err)) catch {};
4553	const exit = if (builtin.single_threaded) linux.exit else linux.exit_group;
4554	exit(1);
4555	}
4556
4557	const pid: pid_t = @intCast(pid_result); // We are the parent.
4558	errdefer comptime unreachable; // The child is forked; we must not error from now on
4559
4560	ev.close(&cancel_region, err_pipe[1]); // make sure only the child holds the write end open
4561
4562	if (options.stdin == .pipe) ev.close(&cancel_region, stdin_pipe[0]);
4563	if (options.stdout == .pipe) ev.close(&cancel_region, stdout_pipe[1]);
4564	if (options.stderr == .pipe) ev.close(&cancel_region, stderr_pipe[1]);
4565
4566	if (prog_pipe[1] != -1) ev.close(&cancel_region, prog_pipe[1]);
4567
4568	options.progress_node.setIpcFile(ev, .{ .handle = prog_pipe[0], .flags = .{ .nonblocking = true } });
4569
4570	return .{
4571	.pid = pid,
4572	.err_fd = err_pipe[0],
4573	.stdin = switch (options.stdin) {
4574	.pipe => .{ .handle = stdin_pipe[1], .flags = .{ .nonblocking = false } },
4575	else => null,
4576	},
4577	.stdout = switch (options.stdout) {
4578	.pipe => .{ .handle = stdout_pipe[0], .flags = .{ .nonblocking = false } },
4579	else => null,
4580	},
4581	.stderr = switch (options.stderr) {
4582	.pipe => .{ .handle = stderr_pipe[0], .flags = .{ .nonblocking = false } },
4583	else => null,
4584	},
4585	};
4586	}
4587
4588	pub const PipeError = error{
4589	SystemFdQuotaExceeded,
4590	ProcessFdQuotaExceeded,
4591	} \|\| Io.UnexpectedError;
4592	pub fn pipe2(flags: linux.O) PipeError![2]fd_t {
4593	var fds: [2]fd_t = undefined;
4594	switch (linux.errno(linux.pipe2(&fds, flags))) {
4595	.SUCCESS => return fds,
4596	.INVAL => \|err\| return errnoBug(err), // Invalid flags
4597	.NFILE => return error.SystemFdQuotaExceeded,
4598	.MFILE => return error.ProcessFdQuotaExceeded,
4599	else => \|err\| return unexpectedErrno(err),
4600	}
4601	}
4602	fn destroyPipe(ev: Evented, cancel_region: CancelRegion, pipe: [2]fd_t) void {
4603	if (pipe[0] != -1) ev.close(cancel_region, pipe[0]);
4604	if (pipe[0] != pipe[1]) ev.close(cancel_region, pipe[1]);
4605	}
4606
4607	/// Errors that can occur between fork() and execv()
4608	const ForkBailError = process.SetCurrentDirError \|\| ChdirError \|\|
4609	process.SpawnError \|\| process.ReplaceError;
4610	fn setUpChild(
4611	ev: *Evented,
4612	sync: *CancelRegion.Sync,
4613	options: struct {
4614	stdin_pipe: fd_t,
4615	stdout_pipe: fd_t,
4616	stderr_pipe: fd_t,
4617	dev_null_fd: fd_t,
4618	prog_pipe: fd_t,
4619	argv_buf: [:null]?[*:0]const u8,
4620	env_block: process.Environ.Block,
4621	PATH: []const u8,
4622	spawn: process.SpawnOptions,
4623	},
4624	) ForkBailError {
4625	try ev.setUpChildIo(
4626	sync,
4627	options.spawn.stdin,
4628	options.stdin_pipe,
4629	linux.STDIN_FILENO,
4630	options.dev_null_fd,
4631	);
4632	try ev.setUpChildIo(
4633	sync,
4634	options.spawn.stdout,
4635	options.stdout_pipe,
4636	linux.STDOUT_FILENO,
4637	options.dev_null_fd,
4638	);
4639	try ev.setUpChildIo(
4640	sync,
4641	options.spawn.stderr,
4642	options.stderr_pipe,
4643	linux.STDERR_FILENO,
4644	options.dev_null_fd,
4645	);
4646
4647	switch (options.spawn.cwd) {
4648	.inherit => {},
4649	.dir => \|cwd_dir\| try ev.fchdir(sync, cwd_dir.handle),
4650	.path => \|cwd_path\| {
4651	var cwd_path_buffer: [PATH_MAX]u8 = undefined;
4652	const cwd_path_posix = try pathToPosix(cwd_path, &cwd_path_buffer);
4653	try ev.chdir(sync, cwd_path_posix);
4654	},
4655	}
4656
4657	// Must happen after fchdir above, the cwd file descriptor might be
4658	// equal to prog_fileno and be clobbered by this dup2 call.
4659	if (options.prog_pipe != -1) try ev.dup2(sync, options.prog_pipe, prog_fileno);
4660
4661	if (options.spawn.gid) \|gid\| {
4662	switch (linux.errno(linux.setregid(gid, gid))) {
4663	.SUCCESS => {},
4664	.AGAIN => return error.ResourceLimitReached,
4665	.INVAL => return error.InvalidUserId,
4666	.PERM => return error.PermissionDenied,
4667	else => return error.Unexpected,
4668	}
4669	}
4670
4671	if (options.spawn.uid) \|uid\| {
4672	switch (linux.errno(linux.setreuid(uid, uid))) {
4673	.SUCCESS => {},
4674	.AGAIN => return error.ResourceLimitReached,
4675	.INVAL => return error.InvalidUserId,
4676	.PERM => return error.PermissionDenied,
4677	else => return error.Unexpected,
4678	}
4679	}
4680
4681	if (options.spawn.pgid) \|pid\| {
4682	switch (linux.errno(linux.setpgid(0, pid))) {
4683	.SUCCESS => {},
4684	.ACCES => return error.ProcessAlreadyExec,
4685	.INVAL => return error.InvalidProcessGroupId,
4686	.PERM => return error.PermissionDenied,
4687	else => return error.Unexpected,
4688	}
4689	}
4690
4691	if (options.spawn.start_suspended) {
4692	switch (linux.errno(linux.kill(linux.getpid(), .STOP))) {
4693	.SUCCESS => {},
4694	.PERM => return error.PermissionDenied,
4695	else => return error.Unexpected,
4696	}
4697	}
4698
4699	return ev.execv(
4700	sync,
4701	options.spawn.expand_arg0,
4702	options.argv_buf.ptr[0].?,
4703	options.argv_buf.ptr,
4704	options.env_block,
4705	options.PATH,
4706	);
4707	}
4708
4709	fn setUpChildIo(
4710	ev: *Evented,
4711	sync: *CancelRegion.Sync,
4712	stdio: process.SpawnOptions.StdIo,
4713	pipe_fd: fd_t,
4714	std_fileno: i32,
4715	dev_null_fd: fd_t,
4716	) !void {
4717	switch (stdio) {
4718	.pipe => try ev.dup2(sync, pipe_fd, std_fileno),
4719	.close => ev.close(&sync.cancel_region, std_fileno),
4720	.inherit => {},
4721	.ignore => try ev.dup2(sync, dev_null_fd, std_fileno),
4722	.file => \|file\| try ev.dup2(sync, file.handle, std_fileno),
4723	}
4724	}
4725
4726	pub const DupError = error{
4727	ProcessFdQuotaExceeded,
4728	SystemResources,
4729	} \|\| Io.UnexpectedError \|\| Io.Cancelable;
4730	pub fn dup2(ev: Evented, sync: CancelRegion.Sync, old_fd: fd_t, new_fd: fd_t) DupError!void {
4731	_ = ev;
4732	while (true) {
4733	try sync.cancel_region.await(.nothing);
4734	switch (linux.errno(linux.dup2(old_fd, new_fd))) {
4735	.SUCCESS => {},
4736	.BUSY, .INTR => continue,
4737	.INVAL => \|err\| return errnoBug(err), // invalid parameters
4738	.BADF => \|err\| return errnoBug(err), // use after free
4739	.MFILE => return error.ProcessFdQuotaExceeded,
4740	.NOMEM => return error.SystemResources,
4741	else => \|err\| return unexpectedErrno(err),
4742	}
4743	}
4744	}
4745
4746	fn execv(
4747	ev: *Evented,
4748	sync: *CancelRegion.Sync,
4749	arg0_expand: process.ArgExpansion,
4750	file: [*:0]const u8,
4751	child_argv: [:null]?[:0]const u8,
4752	env_block: process.Environ.PosixBlock,
4753	PATH: []const u8,
4754	) process.ReplaceError {
4755	const file_slice = std.mem.sliceTo(file, 0);
4756	if (std.mem.findScalar(u8, file_slice, '/') != null) return ev.execvPath(sync, file, child_argv, env_block);
4757
4758	// Use of PATH_MAX here is valid as the path_buf will be passed
4759	// directly to the operating system in posixExecvPath.
4760	var path_buf: [PATH_MAX]u8 = undefined;
4761	var it = std.mem.tokenizeScalar(u8, PATH, ':');
4762	var seen_eacces = false;
4763	var err: process.ReplaceError = error.FileNotFound;
4764
4765	// In case of expanding arg0 we must put it back if we return with an error.
4766	const prev_arg0 = child_argv[0];
4767	defer switch (arg0_expand) {
4768	.expand => child_argv[0] = prev_arg0,
4769	.no_expand => {},
4770	};
4771
4772	while (it.next()) \|search_path\| {
4773	const path_len = search_path.len + file_slice.len + 1;
4774	if (path_buf.len < path_len + 1) return error.NameTooLong;
4775	@memcpy(path_buf[0..search_path.len], search_path);
4776	path_buf[search_path.len] = '/';
4777	@memcpy(path_buf[search_path.len + 1 ..][0..file_slice.len], file_slice);
4778	path_buf[path_len] = 0;
4779	const full_path = path_buf[0..path_len :0].ptr;
4780	switch (arg0_expand) {
4781	.expand => child_argv[0] = full_path,
4782	.no_expand => {},
4783	}
4784	err = ev.execvPath(sync, full_path, child_argv, env_block);
4785	switch (err) {
4786	error.AccessDenied => seen_eacces = true,
4787	error.FileNotFound, error.NotDir => {},
4788	else => \|e\| return e,
4789	}
4790	}
4791	if (seen_eacces) return error.AccessDenied;
4792	return err;
4793	}
4794	/// This function ignores PATH environment variable.
4795	pub fn execvPath(
4796	ev: *Evented,
4797	sync: *CancelRegion.Sync,
4798	path: [*:0]const u8,
4799	child_argv: [:null]const ?[:0]const u8,
4800	env_block: process.Environ.PosixBlock,
4801	) process.ReplaceError {
4802	_ = ev;
4803	try sync.cancel_region.await(.nothing);
4804	switch (linux.errno(linux.execve(path, child_argv, env_block.slice.ptr))) {
4805	.FAULT => \|err\| return errnoBug(err), // Bad pointer parameter.
4806	.@"2BIG" => return error.SystemResources,
4807	.MFILE => return error.ProcessFdQuotaExceeded,
4808	.NAMETOOLONG => return error.NameTooLong,
4809	.NFILE => return error.SystemFdQuotaExceeded,
4810	.NOMEM => return error.SystemResources,
4811	.ACCES => return error.AccessDenied,
4812	.PERM => return error.PermissionDenied,
4813	.INVAL => return error.InvalidExe,
4814	.NOEXEC => return error.InvalidExe,
4815	.IO => return error.FileSystem,
4816	.LOOP => return error.FileSystem,
4817	.ISDIR => return error.IsDir,
4818	.NOENT => return error.FileNotFound,
4819	.NOTDIR => return error.NotDir,
4820	.TXTBSY => return error.FileBusy,
4821	.LIBBAD => return error.InvalidExe,
4822	else => \|err\| return unexpectedErrno(err),
4823	}
4824	}
4825
4826	fn childWait(userdata: ?anyopaque, child: process.Child) process.Child.WaitError!process.Child.Term {
4827	const ev: *Evented = @ptrCast(@alignCast(userdata));
4828
4829	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
4830	defer maybe_sync.deinit(ev);
4831	defer ev.childCleanup(maybe_sync.cancelRegion(), child);
4832
4833	const pid = child.id.?;
4834	var info: linux.siginfo_t = undefined;
4835	while (true) {
4836	const thread = try maybe_sync.cancel_region.awaitIoUring();
4837	thread.enqueue().* = .{
4838	.opcode = .WAITID,
4839	.flags = 0,
4840	.ioprio = 0,
4841	.fd = pid,
4842	.off = @intFromPtr(&info),
4843	.addr = 0,
4844	.len = @intFromEnum(linux.P.PID),
4845	.rw_flags = 0,
4846	.user_data = @intFromPtr(maybe_sync.cancel_region.fiber),
4847	.buf_index = 0,
4848	.personality = 0,
4849	.splice_fd_in = linux.W.EXITED \|
4850	@as(i32, if (child.request_resource_usage_statistics) linux.W.NOWAIT else 0),
4851	.addr3 = 0,
4852	.resv = 0,
4853	};
4854	ev.yield(null, .nothing);
4855	switch (maybe_sync.cancel_region.errno()) {
4856	.SUCCESS => {
4857	if (child.request_resource_usage_statistics) {
4858	const sync = try maybe_sync.enterSync(ev);
4859	while (true) {
4860	try sync.cancel_region.await(.nothing);
4861	var rusage: linux.rusage = undefined;
4862	switch (linux.errno(linux.waitid(
4863	.PID,
4864	pid,
4865	&info,
4866	linux.W.EXITED \| linux.W.NOHANG,
4867	&rusage,
4868	))) {
4869	.SUCCESS => {
4870	child.resource_usage_statistics.rusage = rusage;
4871	break;
4872	},
4873	.INTR, .CANCELED => continue,
4874	.CHILD => \|err\| return errnoBug(err), // Double-free.
4875	else => \|err\| return unexpectedErrno(err),
4876	}
4877	}
4878	}
4879	const status: u32 = @bitCast(info.fields.common.second.sigchld.status);
4880	const code: linux.CLD = @enumFromInt(info.code);
4881	return switch (code) {
4882	.EXITED => .{ .exited = @truncate(status) },
4883	.KILLED, .DUMPED => .{ .signal = @enumFromInt(status) },
4884	.TRAPPED, .STOPPED => .{ .stopped = status },
4885	_, .CONTINUED => .{ .unknown = status },
4886	};
4887	},
4888	.INTR, .CANCELED => continue,
4889	.CHILD => \|err\| return errnoBug(err), // Double-free.
4890	else => \|err\| return unexpectedErrno(err),
4891	}
4892	}
4893	}
4894
4895	fn childKill(userdata: ?anyopaque, child: process.Child) void {
4896	const ev: *Evented = @ptrCast(@alignCast(userdata));
4897
4898	var maybe_sync: CancelRegion.Sync.Maybe = .{ .sync = .initBlocked(ev) };
4899	defer maybe_sync.deinit(ev);
4900	defer ev.childCleanup(maybe_sync.cancelRegion(), child);
4901
4902	const pid = child.id.?;
4903	while (true) switch (linux.errno(linux.kill(pid, .TERM))) {
4904	.SUCCESS => break,
4905	.INTR => continue,
4906	.PERM => return,
4907	.INVAL => \|err\| return errnoBug(err) catch {},
4908	.SRCH => \|err\| return errnoBug(err) catch {},
4909	else => \|err\| return unexpectedErrno(err) catch {},
4910	};
4911	maybe_sync.leaveSync(ev);
4912
4913	var info: linux.siginfo_t = undefined;
4914	while (true) {
4915	const thread = maybe_sync.cancel_region.awaitIoUring() catch \|err\| switch (err) {
4916	error.Canceled => unreachable, // blocked
4917	};
4918	thread.enqueue().* = .{
4919	.opcode = .WAITID,
4920	.flags = 0,
4921	.ioprio = 0,
4922	.fd = pid,
4923	.off = @intFromPtr(&info),
4924	.addr = 0,
4925	.len = @intFromEnum(linux.P.PID),
4926	.rw_flags = 0,
4927	.user_data = @intFromPtr(maybe_sync.cancel_region.fiber),
4928	.buf_index = 0,
4929	.personality = 0,
4930	.splice_fd_in = linux.W.EXITED,
4931	.addr3 = 0,
4932	.resv = 0,
4933	};
4934	ev.yield(null, .nothing);
4935	switch (maybe_sync.cancel_region.errno()) {
4936	.SUCCESS => return,
4937	.INTR, .CANCELED => continue,
4938	.CHILD => \|err\| return errnoBug(err) catch {}, // Double-free.
4939	else => \|err\| return unexpectedErrno(err) catch {},
4940	}
4941	}
4942	}
4943
4944	fn childCleanup(ev: Evented, cancel_region: CancelRegion, child: *process.Child) void {
4945	if (child.stdin) \|*stdin\| {
4946	ev.close(cancel_region, stdin.handle);
4947	child.stdin = null;
4948	}
4949	if (child.stdout) \|*stdout\| {
4950	ev.close(cancel_region, stdout.handle);
4951	child.stdout = null;
4952	}
4953	if (child.stderr) \|*stderr\| {
4954	ev.close(cancel_region, stderr.handle);
4955	child.stderr = null;
4956	}
4957	child.id = null;
4958	}
4959
4960	fn progressParentFile(userdata: ?*anyopaque) std.Progress.ParentFileError!File {
4961	const ev: *Evented = @ptrCast(@alignCast(userdata));
4962	const cancel_protection = swapCancelProtection(ev, .blocked);
4963	defer assert(swapCancelProtection(ev, cancel_protection) == .blocked);
4964	ev.scanEnviron() catch \|err\| switch (err) {
4965	error.Canceled => unreachable, // blocked
4966	};
4967	return ev.environ.zig_progress_file;
4968	}
4969
4970	fn scanEnviron(ev: *Evented) Io.Cancelable!void {
4971	const ev_io = ev.io();
4972	try ev.environ_mutex.lock(ev_io);
4973	defer ev.environ_mutex.unlock(ev_io);
4974	ev.environ.scan(ev.allocator());
4975	}
4976
4977	fn clockResolution(userdata: ?*anyopaque, clock: Io.Clock) Io.Clock.ResolutionError!Io.Duration {
4978	const ev: *Evented = @ptrCast(@alignCast(userdata));
4979	_ = ev;
4980	const clock_id = clockToPosix(clock);
4981	var timespec: linux.timespec = undefined;
4982	return switch (linux.errno(linux.clock_getres(clock_id, &timespec))) {
4983	.SUCCESS => .fromNanoseconds(nanosecondsFromPosix(&timespec)),
4984	.INVAL => return error.ClockUnavailable,
4985	else => \|err\| return unexpectedErrno(err),
4986	};
4987	}
4988
4989	fn now(userdata: ?*anyopaque, clock: Io.Clock) Io.Timestamp {
4990	const ev: *Evented = @ptrCast(@alignCast(userdata));
4991	_ = ev;
4992	var tp: linux.timespec = undefined;
4993	switch (linux.errno(linux.clock_gettime(clockToPosix(clock), &tp))) {
4994	.SUCCESS => return timestampFromPosix(&tp),
4995	else => return .zero,
4996	}
4997	}
4998
4999	fn sleep(userdata: ?*anyopaque, timeout: Io.Timeout) Io.Cancelable!void {
5000	const ev: *Evented = @ptrCast(@alignCast(userdata));
5001
5002	const timespec: linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = timespec: switch (timeout) {
5003	.none => .{
5004	.{
5005	.sec = std.math.maxInt(i64),
5006	.nsec = std.time.ns_per_s - 1,
5007	},
5008	.awake,
5009	linux.IORING_TIMEOUT_ABS,
5010	},
5011	.duration => \|duration\| {
5012	const ns = duration.raw.toNanoseconds();
5013	break :timespec .{
5014	.{
5015	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
5016	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
5017	},
5018	duration.clock,
5019	0,
5020	};
5021	},
5022	.deadline => \|deadline\| {
5023	const ns = deadline.raw.toNanoseconds();
5024	break :timespec .{
5025	.{
5026	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
5027	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
5028	},
5029	deadline.clock,
5030	linux.IORING_TIMEOUT_ABS,
5031	};
5032	},
5033	};
5034	var cancel_region: CancelRegion = .init();
5035	defer cancel_region.deinit();
5036	const thread = try cancel_region.awaitIoUring();
5037	thread.enqueue().* = .{
5038	.opcode = .TIMEOUT,
5039	.flags = 0,
5040	.ioprio = 0,
5041	.fd = 0,
5042	.off = 0,
5043	.addr = @intFromPtr(&timespec),
5044	.len = 1,
5045	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
5046	.real => linux.IORING_TIMEOUT_REALTIME,
5047	else => 0,
5048	.boot => linux.IORING_TIMEOUT_BOOTTIME,
5049	}),
5050	.user_data = @intFromPtr(cancel_region.fiber),
5051	.buf_index = 0,
5052	.personality = 0,
5053	.splice_fd_in = 0,
5054	.addr3 = 0,
5055	.resv = 0,
5056	};
5057	ev.yield(null, .nothing);
5058	switch (cancel_region.errno()) {
5059	// Handles SUCCESS as well as clock not available and unexpected
5060	// errors. The user had a chance to check clock resolution before
5061	// getting here, which would have reported 0, making this a legal
5062	// amount of time to sleep.
5063	else => return,
5064	.INTR, .CANCELED => return error.Canceled,
5065	}
5066	}
5067
5068	fn random(userdata: ?*anyopaque, buffer: []u8) void {
5069	const ev: *Evented = @ptrCast(@alignCast(userdata));
5070	var thread: *Thread = .current();
5071	if (!thread.csprng.isInitialized()) {
5072	@branchHint(.unlikely);
5073	var seed: [Csprng.seed_len]u8 = undefined;
5074	{
5075	const ev_io = ev.io();
5076	ev.csprng_mutex.lockUncancelable(ev_io);
5077	defer ev.csprng_mutex.unlock(ev_io);
5078	if (!ev.csprng.isInitialized()) {
5079	@branchHint(.unlikely);
5080	var cancel_region: CancelRegion = .initBlocked();
5081	defer cancel_region.deinit();
5082	ev.urandomReadAll(&cancel_region, &seed) catch \|err\| switch (err) {
5083	error.Canceled => unreachable, // blocked
5084	else => fallbackSeed(ev, &seed),
5085	};
5086	ev.csprng.rng = .init(seed);
5087	thread = .current();
5088	}
5089	ev.csprng.rng.fill(&seed);
5090	}
5091	if (!thread.csprng.isInitialized()) {
5092	@branchHint(.likely);
5093	thread.csprng.rng = .init(seed);
5094	} else thread.csprng.rng.addEntropy(&seed);
5095	}
5096	thread.csprng.rng.fill(buffer);
5097	}
5098
5099	fn randomSecure(userdata: ?*anyopaque, buffer: []u8) Io.RandomSecureError!void {
5100	const ev: *Evented = @ptrCast(@alignCast(userdata));
5101	if (buffer.len == 0) return;
5102	var cancel_region: CancelRegion = .init();
5103	defer cancel_region.deinit();
5104	ev.urandomReadAll(&cancel_region, buffer) catch \|err\| switch (err) {
5105	error.Canceled => return error.Canceled,
5106	else => return error.EntropyUnavailable,
5107	};
5108	}
5109
5110	fn netListenIpUnavailable(
5111	userdata: ?*anyopaque,
5112	address: net.IpAddress,
5113	options: net.IpAddress.ListenOptions,
5114	) net.IpAddress.ListenError!net.Server {
5115	const ev: *Evented = @ptrCast(@alignCast(userdata));
5116	_ = ev;
5117	_ = address;
5118	_ = options;
5119	return error.NetworkDown;
5120	}
5121
5122	fn netAcceptUnavailable(
5123	userdata: ?*anyopaque,
5124	listen_handle: net.Socket.Handle,
5125	) net.Server.AcceptError!net.Stream {
5126	const ev: *Evented = @ptrCast(@alignCast(userdata));
5127	_ = ev;
5128	_ = listen_handle;
5129	return error.NetworkDown;
5130	}
5131
5132	fn netBindIp(
5133	userdata: ?*anyopaque,
5134	address: *const net.IpAddress,
5135	options: net.IpAddress.BindOptions,
5136	) net.IpAddress.BindError!net.Socket {
5137	const ev: *Evented = @ptrCast(@alignCast(userdata));
5138	const family = posixAddressFamily(address);
5139	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
5140	defer maybe_sync.deinit(ev);
5141	const socket_fd = try ev.socket(&maybe_sync.cancel_region, family, options);
5142	errdefer ev.close(maybe_sync.cancelRegion(), socket_fd);
5143	var storage: PosixAddress = undefined;
5144	var addr_len = addressToPosix(address, &storage);
5145	try ev.bind(&maybe_sync.cancel_region, socket_fd, &storage.any, addr_len);
5146	try ev.getsockname(try maybe_sync.enterSync(ev), socket_fd, &storage.any, &addr_len);
5147	return .{
5148	.handle = socket_fd,
5149	.address = addressFromPosix(&storage),
5150	};
5151	}
5152
5153	fn netBindIpUnavailable(
5154	userdata: ?*anyopaque,
5155	address: *const net.IpAddress,
5156	options: net.IpAddress.BindOptions,
5157	) net.IpAddress.BindError!net.Socket {
5158	const ev: *Evented = @ptrCast(@alignCast(userdata));
5159	_ = ev;
5160	_ = address;
5161	_ = options;
5162	return error.NetworkDown;
5163	}
5164
5165	fn netConnectIpUnavailable(
5166	userdata: ?*anyopaque,
5167	address: *const net.IpAddress,
5168	options: net.IpAddress.ConnectOptions,
5169	) net.IpAddress.ConnectError!net.Stream {
5170	const ev: *Evented = @ptrCast(@alignCast(userdata));
5171	_ = ev;
5172	_ = address;
5173	_ = options;
5174	return error.NetworkDown;
5175	}
5176
5177	fn netListenUnixUnavailable(
5178	userdata: ?*anyopaque,
5179	address: *const net.UnixAddress,
5180	options: net.UnixAddress.ListenOptions,
5181	) net.UnixAddress.ListenError!net.Socket.Handle {
5182	const ev: *Evented = @ptrCast(@alignCast(userdata));
5183	_ = ev;
5184	_ = address;
5185	_ = options;
5186	return error.AddressFamilyUnsupported;
5187	}
5188
5189	fn netConnectUnixUnavailable(
5190	userdata: ?*anyopaque,
5191	address: *const net.UnixAddress,
5192	) net.UnixAddress.ConnectError!net.Socket.Handle {
5193	const ev: *Evented = @ptrCast(@alignCast(userdata));
5194	_ = ev;
5195	_ = address;
5196	return error.AddressFamilyUnsupported;
5197	}
5198
5199	fn netSocketCreatePairUnavailable(
5200	userdata: ?*anyopaque,
5201	options: net.Socket.CreatePairOptions,
5202	) net.Socket.CreatePairError![2]net.Socket {
5203	_ = userdata;
5204	_ = options;
5205	return error.OperationUnsupported;
5206	}
5207
5208	fn netSendUnavailable(
5209	userdata: ?*anyopaque,
5210	handle: net.Socket.Handle,
5211	messages: []net.OutgoingMessage,
5212	flags: net.SendFlags,
5213	) struct { ?net.Socket.SendError, usize } {
5214	const ev: *Evented = @ptrCast(@alignCast(userdata));
5215	_ = ev;
5216	_ = handle;
5217	_ = messages;
5218	_ = flags;
5219	return .{ error.NetworkDown, 0 };
5220	}
5221
5222	fn netReceive(
5223	userdata: ?*anyopaque,
5224	handle: net.Socket.Handle,
5225	message_buffer: []net.IncomingMessage,
5226	data_buffer: []u8,
5227	flags: net.ReceiveFlags,
5228	timeout: Io.Timeout,
5229	) struct { ?net.Socket.ReceiveTimeoutError, usize } {
5230	const ev: *Evented = @ptrCast(@alignCast(userdata));
5231	const ev_io = ev.io();
5232
5233	var message_i: usize = 0;
5234	var data_i: usize = 0;
5235
5236	const deadline: ?struct {
5237	raw: Io.Timestamp,
5238	timespec: linux.kernel_timespec,
5239	clock: Io.Clock,
5240	} = if (timeout.toTimestamp(ev_io)) \|deadline\| deadline: {
5241	const ns = deadline.raw.toNanoseconds();
5242	break :deadline .{
5243	.raw = deadline.raw,
5244	.timespec = .{
5245	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
5246	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
5247	},
5248	.clock = deadline.clock,
5249	};
5250	} else null;
5251
5252	var cancel_region: CancelRegion = .init();
5253	defer cancel_region.deinit();
5254	while (true) {
5255	if (message_buffer.len - message_i == 0) return .{ null, message_i };
5256	const message = &message_buffer[message_i];
5257	const remaining_data_buffer = data_buffer[data_i..];
5258	var storage: PosixAddress = undefined;
5259	var iov: iovec = .{ .base = remaining_data_buffer.ptr, .len = remaining_data_buffer.len };
5260	var msg: linux.msghdr = .{
5261	.name = &storage.any,
5262	.namelen = @sizeOf(PosixAddress),
5263	.iov = (&iov)[0..1],
5264	.iovlen = 1,
5265	.control = message.control.ptr,
5266	.controllen = @intCast(message.control.len),
5267	.flags = undefined,
5268	};
5269
5270	const thread = cancel_region.awaitIoUring() catch \|err\| return .{ err, message_i };
5271	thread.enqueue().* = .{
5272	.opcode = .RECVMSG,
5273	.flags = if (deadline) \|_\| linux.IOSQE_IO_LINK else 0,
5274	.ioprio = 0,
5275	.fd = handle,
5276	.off = 0,
5277	.addr = @intFromPtr(&msg),
5278	.len = 0,
5279	.rw_flags = linux.MSG.NOSIGNAL \|
5280	@as(u32, if (flags.oob) linux.MSG.OOB else 0) \|
5281	@as(u32, if (flags.peek) linux.MSG.PEEK else 0) \|
5282	@as(u32, if (flags.trunc) linux.MSG.TRUNC else 0),
5283	.user_data = @intFromPtr(cancel_region.fiber),
5284	.buf_index = 0,
5285	.personality = 0,
5286	.splice_fd_in = 0,
5287	.addr3 = 0,
5288	.resv = 0,
5289	};
5290	if (deadline) \|deadline_ptr\| thread.enqueue(). = .{
5291	.opcode = .LINK_TIMEOUT,
5292	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
5293	.ioprio = 0,
5294	.fd = 0,
5295	.off = 0,
5296	.addr = @intFromPtr(&deadline_ptr.timespec),
5297	.len = 1,
5298	.rw_flags = linux.IORING_TIMEOUT_ABS \| @as(u32, switch (deadline_ptr.clock) {
5299	.real => linux.IORING_TIMEOUT_REALTIME,
5300	else => 0,
5301	.boot => linux.IORING_TIMEOUT_BOOTTIME,
5302	}),
5303	.user_data = @intFromEnum(Completion.UserData.wakeup),
5304	.buf_index = 0,
5305	.personality = 0,
5306	.splice_fd_in = 0,
5307	.addr3 = 0,
5308	.resv = 0,
5309	};
5310	ev.yield(null, .nothing);
5311	const completion = cancel_region.completion();
5312	switch (completion.errno()) {
5313	.SUCCESS => {
5314	const data = remaining_data_buffer[0..@intCast(completion.result)];
5315	data_i += data.len;
5316	message.* = .{
5317	.from = addressFromPosix(&storage),
5318	.data = data,
5319	.control = if (msg.control) \|ptr\| @as([*]u8, @ptrCast(ptr))[0..msg.controllen] else message.control,
5320	.flags = .{
5321	.eor = (msg.flags & linux.MSG.EOR) != 0,
5322	.trunc = (msg.flags & linux.MSG.TRUNC) != 0,
5323	.ctrunc = (msg.flags & linux.MSG.CTRUNC) != 0,
5324	.oob = (msg.flags & linux.MSG.OOB) != 0,
5325	.errqueue = if (@hasDecl(linux.MSG, "ERRQUEUE")) (msg.flags & linux.MSG.ERRQUEUE) != 0 else false,
5326	},
5327	};
5328	message_i += 1;
5329	continue;
5330	},
5331	.AGAIN => unreachable,
5332	.INTR, .CANCELED => {
5333	if (deadline) \|d\| {
5334	if (now(ev, d.clock).nanoseconds >= d.raw.nanoseconds) return .{ error.Timeout, message_i };
5335	}
5336	continue;
5337	},
5338
5339	.BADF => \|err\| return .{ errnoBug(err), message_i },
5340	.NFILE => return .{ error.SystemFdQuotaExceeded, message_i },
5341	.MFILE => return .{ error.ProcessFdQuotaExceeded, message_i },
5342	.FAULT => \|err\| return .{ errnoBug(err), message_i },
5343	.INVAL => \|err\| return .{ errnoBug(err), message_i },
5344	.NOBUFS => return .{ error.SystemResources, message_i },
5345	.NOMEM => return .{ error.SystemResources, message_i },
5346	.NOTCONN => return .{ error.SocketUnconnected, message_i },
5347	.NOTSOCK => \|err\| return .{ errnoBug(err), message_i },
5348	.MSGSIZE => return .{ error.MessageOversize, message_i },
5349	.PIPE => return .{ error.SocketUnconnected, message_i },
5350	.OPNOTSUPP => \|err\| return .{ errnoBug(err), message_i },
5351	.CONNRESET => return .{ error.ConnectionResetByPeer, message_i },
5352	.NETDOWN => return .{ error.NetworkDown, message_i },
5353	else => \|err\| return .{ unexpectedErrno(err), message_i },
5354	}
5355	}
5356	}
5357
5358	fn netReceiveUnavailable(
5359	userdata: ?*anyopaque,
5360	handle: net.Socket.Handle,
5361	message_buffer: []net.IncomingMessage,
5362	data_buffer: []u8,
5363	flags: net.ReceiveFlags,
5364	timeout: Io.Timeout,
5365	) struct { ?net.Socket.ReceiveTimeoutError, usize } {
5366	const ev: *Evented = @ptrCast(@alignCast(userdata));
5367	_ = ev;
5368	_ = handle;
5369	_ = message_buffer;
5370	_ = data_buffer;
5371	_ = flags;
5372	_ = timeout;
5373	return .{ error.NetworkDown, 0 };
5374	}
5375
5376	fn netReadUnavailable(
5377	userdata: ?*anyopaque,
5378	fd: net.Socket.Handle,
5379	data: [][]u8,
5380	) net.Stream.Reader.Error!usize {
5381	const ev: *Evented = @ptrCast(@alignCast(userdata));
5382	_ = ev;
5383	_ = fd;
5384	_ = data;
5385	return error.NetworkDown;
5386	}
5387
5388	fn netWriteUnavailable(
5389	userdata: ?*anyopaque,
5390	handle: net.Socket.Handle,
5391	header: []const u8,
5392	data: []const []const u8,
5393	splat: usize,
5394	) net.Stream.Writer.Error!usize {
5395	const ev: *Evented = @ptrCast(@alignCast(userdata));
5396	_ = ev;
5397	_ = handle;
5398	_ = header;
5399	_ = data;
5400	_ = splat;
5401	return error.NetworkDown;
5402	}
5403
5404	fn netWriteFileUnavailable(
5405	userdata: ?*anyopaque,
5406	socket_handle: net.Socket.Handle,
5407	header: []const u8,
5408	file_reader: *File.Reader,
5409	limit: Io.Limit,
5410	) net.Stream.Writer.WriteFileError!usize {
5411	const ev: *Evented = @ptrCast(@alignCast(userdata));
5412	_ = ev;
5413	_ = socket_handle;
5414	_ = header;
5415	_ = file_reader;
5416	_ = limit;
5417	return error.NetworkDown;
5418	}
5419
5420	fn netClose(userdata: ?*anyopaque, handles: []const net.Socket.Handle) void {
5421	const ev: *Evented = @ptrCast(@alignCast(userdata));
5422	var cancel_region: CancelRegion = .init();
5423	defer cancel_region.deinit();
5424	for (handles) \|handle\| ev.close(&cancel_region, handle);
5425	}
5426
5427	fn netCloseUnavailable(userdata: ?*anyopaque, handles: []const net.Socket.Handle) void {
5428	const ev: *Evented = @ptrCast(@alignCast(userdata));
5429	_ = ev;
5430	_ = handles;
5431	unreachable; // How you gonna close something that was impossible to open?
5432	}
5433
5434	fn netShutdown(
5435	userdata: ?*anyopaque,
5436	handle: net.Socket.Handle,
5437	how: net.ShutdownHow,
5438	) net.ShutdownError!void {
5439	const ev: *Evented = @ptrCast(@alignCast(userdata));
5440	var cancel_region: CancelRegion = .init();
5441	defer cancel_region.deinit();
5442	while (true) {
5443	const thread = try cancel_region.awaitIoUring();
5444	thread.enqueue().* = .{
5445	.opcode = .SHUTDOWN,
5446	.flags = 0,
5447	.ioprio = 0,
5448	.fd = handle,
5449	.off = 0,
5450	.addr = 0,
5451	.len = switch (how) {
5452	.recv => linux.SHUT.RD,
5453	.send => linux.SHUT.WR,
5454	.both => linux.SHUT.RDWR,
5455	},
5456	.rw_flags = 0,
5457	.user_data = @intFromPtr(cancel_region.fiber),
5458	.buf_index = 0,
5459	.personality = 0,
5460	.splice_fd_in = 0,
5461	.addr3 = 0,
5462	.resv = 0,
5463	};
5464	ev.yield(null, .nothing);
5465	switch (cancel_region.errno()) {
5466	.SUCCESS => return,
5467	.INTR, .CANCELED => continue,
5468	.BADF, .NOTSOCK, .INVAL => \|err\| return errnoBug(err),
5469	.NOTCONN => return error.SocketUnconnected,
5470	.NOBUFS => return error.SystemResources,
5471	else => \|err\| return unexpectedErrno(err),
5472	}
5473	}
5474	}
5475
5476	fn netShutdownUnavailable(
5477	userdata: ?*anyopaque,
5478	handle: net.Socket.Handle,
5479	how: net.ShutdownHow,
5480	) net.ShutdownError!void {
5481	const ev: *Evented = @ptrCast(@alignCast(userdata));
5482	_ = ev;
5483	_ = handle;
5484	_ = how;
5485	unreachable; // How you gonna shutdown something that was impossible to open?
5486	}
5487
5488	fn netInterfaceNameResolveUnavailable(
5489	userdata: ?*anyopaque,
5490	name: *const net.Interface.Name,
5491	) net.Interface.Name.ResolveError!net.Interface {
5492	const ev: *Evented = @ptrCast(@alignCast(userdata));
5493	_ = ev;
5494	_ = name;
5495	return error.InterfaceNotFound;
5496	}
5497
5498	fn netInterfaceNameUnavailable(
5499	userdata: ?*anyopaque,
5500	interface: net.Interface,
5501	) net.Interface.NameError!net.Interface.Name {
5502	const ev: *Evented = @ptrCast(@alignCast(userdata));
5503	_ = ev;
5504	_ = interface;
5505	return error.Unexpected;
5506	}
5507
5508	fn netLookupUnavailable(
5509	userdata: ?*anyopaque,
5510	host_name: net.HostName,
5511	resolved: *Io.Queue(net.HostName.LookupResult),
5512	options: net.HostName.LookupOptions,
5513	) net.HostName.LookupError!void {
5514	const ev: *Evented = @ptrCast(@alignCast(userdata));
5515	_ = host_name;
5516	_ = options;
5517	resolved.close(ev.io());
5518	return error.NetworkDown;
5519	}
5520
5521	fn bind(
5522	ev: *Evented,
5523	cancel_region: *CancelRegion,
5524	socket_fd: fd_t,
5525	addr: *const linux.sockaddr,
5526	addr_len: linux.socklen_t,
5527	) !void {
5528	while (true) {
5529	const thread = try cancel_region.awaitIoUring();
5530	thread.enqueue().* = .{
5531	.opcode = .BIND,
5532	.flags = 0,
5533	.ioprio = 0,
5534	.fd = socket_fd,
5535	.off = addr_len,
5536	.addr = @intFromPtr(addr),
5537	.len = 0,
5538	.rw_flags = 0,
5539	.user_data = @intFromPtr(cancel_region.fiber),
5540	.buf_index = 0,
5541	.personality = 0,
5542	.splice_fd_in = 0,
5543	.addr3 = 0,
5544	.resv = 0,
5545	};
5546	ev.yield(null, .nothing);
5547	switch (cancel_region.errno()) {
5548	.SUCCESS => return,
5549	.INTR, .CANCELED => continue,
5550	.ADDRINUSE => return error.AddressInUse,
5551	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
5552	.INVAL => \|err\| return errnoBug(err), // invalid parameters
5553	.NOTSOCK => \|err\| return errnoBug(err), // invalid `sockfd`
5554	.AFNOSUPPORT => return error.AddressFamilyUnsupported,
5555	.ADDRNOTAVAIL => return error.AddressUnavailable,
5556	.FAULT => \|err\| return errnoBug(err), // invalid `addr` pointer
5557	.NOMEM => return error.SystemResources,
5558	else => \|err\| return unexpectedErrno(err),
5559	}
5560	}
5561	}
5562
5563	fn chdir(ev: Evented, sync: CancelRegion.Sync, path: [*:0]const u8) ChdirError!void {
5564	_ = ev;
5565	while (true) {
5566	try sync.cancel_region.await(.nothing);
5567	switch (linux.errno(linux.chdir(path))) {
5568	.SUCCESS => return,
5569	.INTR => continue,
5570	.ACCES => return error.AccessDenied,
5571	.IO => return error.FileSystem,
5572	.LOOP => return error.SymLinkLoop,
5573	.NAMETOOLONG => return error.NameTooLong,
5574	.NOENT => return error.FileNotFound,
5575	.NOMEM => return error.SystemResources,
5576	.NOTDIR => return error.NotDir,
5577	.ILSEQ => return error.BadPathName,
5578	.FAULT => \|err\| return errnoBug(err),
5579	else => \|err\| return unexpectedErrno(err),
5580	}
5581	}
5582	}
5583
5584	fn close(ev: Evented, cancel_region: CancelRegion, fd: fd_t) void {
5585	while (true) {
5586	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
5587	error.Canceled => unreachable, // blocked
5588	};
5589	thread.enqueue().* = .{
5590	.opcode = .CLOSE,
5591	.flags = 0,
5592	.ioprio = 0,
5593	.fd = fd,
5594	.off = 0,
5595	.addr = 0,
5596	.len = 0,
5597	.rw_flags = 0,
5598	.user_data = @intFromPtr(cancel_region.fiber),
5599	.buf_index = 0,
5600	.personality = 0,
5601	.splice_fd_in = 0,
5602	.addr3 = 0,
5603	.resv = 0,
5604	};
5605	ev.yield(null, .nothing);
5606	switch (cancel_region.errno()) {
5607	.SUCCESS => return,
5608	.INTR, .CANCELED => continue,
5609	.BADF => unreachable, // Always a race condition.
5610	else => break,
5611	}
5612	}
5613	}
5614
5615	fn fchdir(ev: Evented, sync: CancelRegion.Sync, dir: fd_t) process.SetCurrentDirError!void {
5616	_ = ev;
5617	if (dir == linux.AT.FDCWD) return;
5618	while (true) {
5619	try sync.cancel_region.await(.nothing);
5620	switch (linux.errno(linux.fchdir(dir))) {
5621	.SUCCESS => return,
5622	.INTR => continue,
5623	.ACCES => return error.AccessDenied,
5624	.NOTDIR => return error.NotDir,
5625	.IO => return error.FileSystem,
5626	.BADF => \|err\| return errnoBug(err),
5627	else => \|err\| return unexpectedErrno(err),
5628	}
5629	}
5630	}
5631
5632	fn fchmodat(
5633	ev: *Evented,
5634	sync: *CancelRegion.Sync,
5635	dir: fd_t,
5636	path: [*:0]const u8,
5637	mode: linux.mode_t,
5638	flags: u32,
5639	) Dir.SetFilePermissionsError!void {
5640	_ = ev;
5641	while (true) {
5642	try sync.cancel_region.await(.nothing);
5643	switch (linux.errno(linux.fchmodat2(dir, path, mode, flags))) {
5644	.SUCCESS => return,
5645	.INTR => continue,
5646	.BADF => \|err\| return errnoBug(err),
5647	.FAULT => \|err\| return errnoBug(err),
5648	.INVAL => \|err\| return errnoBug(err),
5649	.ACCES => return error.AccessDenied,
5650	.IO => return error.InputOutput,
5651	.LOOP => return error.SymLinkLoop,
5652	.NOENT => return error.FileNotFound,
5653	.NOMEM => return error.SystemResources,
5654	.NOTDIR => return error.FileNotFound,
5655	.OPNOTSUPP => return error.OperationUnsupported,
5656	.PERM => return error.PermissionDenied,
5657	.ROFS => return error.ReadOnlyFileSystem,
5658	else => \|err\| return unexpectedErrno(err),
5659	}
5660	}
5661	}
5662
5663	fn fchownat(
5664	ev: *Evented,
5665	sync: *CancelRegion.Sync,
5666	dir: fd_t,
5667	path: [*:0]const u8,
5668	owner: linux.uid_t,
5669	group: linux.gid_t,
5670	flags: u32,
5671	) File.SetOwnerError!void {
5672	_ = ev;
5673	while (true) {
5674	try sync.cancel_region.await(.nothing);
5675	switch (linux.errno(linux.fchownat(dir, path, owner, group, flags))) {
5676	.SUCCESS => return,
5677	.INTR => continue,
5678	.BADF => \|err\| return errnoBug(err), // likely fd refers to directory opened without `Dir.OpenOptions.iterate`
5679	.FAULT => \|err\| return errnoBug(err),
5680	.INVAL => \|err\| return errnoBug(err),
5681	.ACCES => return error.AccessDenied,
5682	.IO => return error.InputOutput,
5683	.LOOP => return error.SymLinkLoop,
5684	.NOENT => return error.FileNotFound,
5685	.NOMEM => return error.SystemResources,
5686	.NOTDIR => return error.FileNotFound,
5687	.PERM => return error.PermissionDenied,
5688	.ROFS => return error.ReadOnlyFileSystem,
5689	else => \|err\| return unexpectedErrno(err),
5690	}
5691	}
5692	}
5693
5694	fn flock(
5695	ev: *Evented,
5696	sync: *CancelRegion.Sync,
5697	fd: fd_t,
5698	op: File.Lock,
5699	blocking: enum { blocking, nonblocking },
5700	) (File.LockError \|\| error{WouldBlock})!void {
5701	while (true) {
5702	try sync.cancel_region.await(.nothing);
5703	switch (linux.errno(linux.flock(fd, LOCK.NB \| @as(i32, switch (op) {
5704	.none => LOCK.UN,
5705	.shared => LOCK.SH,
5706	.exclusive => LOCK.EX,
5707	})))) {
5708	.SUCCESS => return,
5709	.INTR => continue,
5710	.BADF => \|err\| return errnoBug(err),
5711	.INVAL => \|err\| return errnoBug(err), // invalid parameters
5712	.NOLCK => return error.SystemResources,
5713	.AGAIN => {
5714	const thread = try sync.cancel_region.awaitIoUring();
5715	thread.enqueue().* = .{
5716	.opcode = .NOP,
5717	.flags = 0,
5718	.ioprio = 0,
5719	.fd = 0,
5720	.off = 0,
5721	.addr = 0,
5722	.len = 0,
5723	.rw_flags = 0,
5724	.user_data = @intFromPtr(sync.cancel_region.fiber),
5725	.buf_index = 0,
5726	.personality = 0,
5727	.splice_fd_in = 0,
5728	.addr3 = 0,
5729	.resv = 0,
5730	};
5731	ev.yield(null, .nothing);
5732	switch (sync.cancel_region.errno()) {
5733	.SUCCESS, .INTR, .CANCELED => {},
5734	else => unreachable,
5735	}
5736	switch (blocking) {
5737	.blocking => continue,
5738	.nonblocking => return error.WouldBlock,
5739	}
5740	},
5741	.OPNOTSUPP => return error.FileLocksUnsupported,
5742	else => \|err\| return unexpectedErrno(err),
5743	}
5744	}
5745	}
5746
5747	fn getsockname(
5748	ev: *Evented,
5749	sync: *CancelRegion.Sync,
5750	socket_fd: fd_t,
5751	addr: *linux.sockaddr,
5752	addr_len: *linux.socklen_t,
5753	) !void {
5754	_ = ev;
5755	while (true) {
5756	try sync.cancel_region.await(.nothing);
5757	switch (linux.errno(linux.getsockname(socket_fd, addr, addr_len))) {
5758	.SUCCESS => return,
5759	.INTR => continue,
5760	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
5761	.FAULT => \|err\| return errnoBug(err),
5762	.INVAL => \|err\| return errnoBug(err), // invalid parameters
5763	.NOTSOCK => \|err\| return errnoBug(err), // always a race condition
5764	.NOBUFS => return error.SystemResources,
5765	else => \|err\| return unexpectedErrno(err),
5766	}
5767	}
5768	}
5769
5770	fn linkat(
5771	ev: *Evented,
5772	cancel_region: *CancelRegion,
5773	old_dir: fd_t,
5774	old_path: [*:0]const u8,
5775	new_dir: fd_t,
5776	new_path: [*:0]const u8,
5777	flags: u32,
5778	) File.HardLinkError!void {
5779	while (true) {
5780	const thread = try cancel_region.awaitIoUring();
5781	thread.enqueue().* = .{
5782	.opcode = .LINKAT,
5783	.flags = 0,
5784	.ioprio = 0,
5785	.fd = old_dir,
5786	.off = @intFromPtr(new_path),
5787	.addr = @intFromPtr(old_path),
5788	.len = @bitCast(new_dir),
5789	.rw_flags = flags,
5790	.user_data = @intFromPtr(cancel_region.fiber),
5791	.buf_index = 0,
5792	.personality = 0,
5793	.splice_fd_in = 0,
5794	.addr3 = 0,
5795	.resv = 0,
5796	};
5797	ev.yield(null, .nothing);
5798	switch (cancel_region.errno()) {
5799	.SUCCESS => return,
5800	.INTR, .CANCELED => continue,
5801	.ACCES => return error.AccessDenied,
5802	.DQUOT => return error.DiskQuota,
5803	.EXIST => return error.PathAlreadyExists,
5804	.IO => return error.HardwareFailure,
5805	.LOOP => return error.SymLinkLoop,
5806	.MLINK => return error.LinkQuotaExceeded,
5807	.NAMETOOLONG => return error.NameTooLong,
5808	.NOENT => return error.FileNotFound,
5809	.NOMEM => return error.SystemResources,
5810	.NOSPC => return error.NoSpaceLeft,
5811	.NOTDIR => return error.NotDir,
5812	.PERM => return error.PermissionDenied,
5813	.ROFS => return error.ReadOnlyFileSystem,
5814	.XDEV => return error.CrossDevice,
5815	.ILSEQ => return error.BadPathName,
5816	.FAULT => \|err\| return errnoBug(err),
5817	.INVAL => \|err\| return errnoBug(err),
5818	else => \|err\| return unexpectedErrno(err),
5819	}
5820	}
5821	}
5822
5823	fn lseek(
5824	ev: *Evented,
5825	sync: *CancelRegion.Sync,
5826	fd: fd_t,
5827	offset: u64,
5828	whence: u32,
5829	) File.SeekError!void {
5830	_ = ev;
5831	while (true) {
5832	try sync.cancel_region.await(.nothing);
5833	var result: u64 = undefined;
5834	switch (linux.errno(switch (@sizeOf(usize)) {
5835	else => comptime unreachable,
5836	4 => linux.llseek(fd, offset, &result, whence),
5837	8 => linux.lseek(fd, @bitCast(offset), whence),
5838	})) {
5839	.SUCCESS => return,
5840	.INTR => continue,
5841	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
5842	.INVAL => return error.Unseekable,
5843	.OVERFLOW => return error.Unseekable,
5844	.SPIPE => return error.Unseekable,
5845	.NXIO => return error.Unseekable,
5846	else => \|err\| return unexpectedErrno(err),
5847	}
5848	}
5849	}
5850
5851	fn openat(
5852	ev: *Evented,
5853	cancel_region: *CancelRegion,
5854	dir: fd_t,
5855	path: [*:0]const u8,
5856	flags: linux.O,
5857	mode: linux.mode_t,
5858	) File.OpenError!fd_t {
5859	var mut_flags = flags;
5860	if (@hasField(linux.O, "LARGEFILE")) mut_flags.LARGEFILE = true;
5861	while (true) {
5862	const thread = try cancel_region.awaitIoUring();
5863	thread.enqueue().* = .{
5864	.opcode = .OPENAT,
5865	.flags = 0,
5866	.ioprio = 0,
5867	.fd = dir,
5868	.off = 0,
5869	.addr = @intFromPtr(path),
5870	.len = mode,
5871	.rw_flags = @bitCast(mut_flags),
5872	.user_data = @intFromPtr(cancel_region.fiber),
5873	.buf_index = 0,
5874	.personality = 0,
5875	.splice_fd_in = 0,
5876	.addr3 = 0,
5877	.resv = 0,
5878	};
5879	ev.yield(null, .nothing);
5880	const completion = cancel_region.completion();
5881	switch (completion.errno()) {
5882	.SUCCESS => return completion.result,
5883	.INTR, .CANCELED => continue,
5884	.FAULT => \|err\| return errnoBug(err),
5885	.INVAL => return error.BadPathName,
5886	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
5887	.ACCES => return error.AccessDenied,
5888	.FBIG => return error.FileTooBig,
5889	.OVERFLOW => return error.FileTooBig,
5890	.ISDIR => return error.IsDir,
5891	.LOOP => return error.SymLinkLoop,
5892	.MFILE => return error.ProcessFdQuotaExceeded,
5893	.NAMETOOLONG => return error.NameTooLong,
5894	.NFILE => return error.SystemFdQuotaExceeded,
5895	.NODEV => return error.NoDevice,
5896	.NOENT => return error.FileNotFound,
5897	.SRCH => return error.FileNotFound, // Linux when opening procfs files.
5898	.NOMEM => return error.SystemResources,
5899	.NOSPC => return error.NoSpaceLeft,
5900	.NOTDIR => return error.NotDir,
5901	.PERM => return error.PermissionDenied,
5902	.EXIST => return error.PathAlreadyExists,
5903	.BUSY => return error.DeviceBusy,
5904	.OPNOTSUPP => return error.FileLocksUnsupported,
5905	.AGAIN => return error.WouldBlock,
5906	.TXTBSY => return error.FileBusy,
5907	.NXIO => return error.NoDevice,
5908	.ILSEQ => return error.BadPathName,
5909	else => \|err\| return unexpectedErrno(err),
5910	}
5911	}
5912	}
5913
5914	fn preadv(
5915	ev: *Evented,
5916	cancel_region: *CancelRegion,
5917	fd: fd_t,
5918	iov: []const iovec,
5919	offset: ?u64,
5920	) File.Reader.Error!usize {
5921	if (iov.len == 0) return 0;
5922	const gather = iov.len > 1 or iov[0].len > 0xfffff000;
5923	while (true) {
5924	const thread = try cancel_region.awaitIoUring();
5925	thread.enqueue().* = .{
5926	.opcode = if (gather) .READV else .READ,
5927	.flags = 0,
5928	.ioprio = 0,
5929	.fd = fd,
5930	.off = offset orelse std.math.maxInt(u64),
5931	.addr = if (gather) @intFromPtr(iov.ptr) else @intFromPtr(iov[0].base),
5932	.len = @intCast(if (gather) iov.len else iov[0].len),
5933	.rw_flags = 0,
5934	.user_data = @intFromPtr(cancel_region.fiber),
5935	.buf_index = 0,
5936	.personality = 0,
5937	.splice_fd_in = 0,
5938	.addr3 = 0,
5939	.resv = 0,
5940	};
5941	ev.yield(null, .nothing);
5942	const completion = cancel_region.completion();
5943	switch (completion.errno()) {
5944	.SUCCESS => return @as(u32, @bitCast(completion.result)),
5945	.INTR, .CANCELED => continue,
5946	.INVAL => \|err\| return errnoBug(err),
5947	.FAULT => \|err\| return errnoBug(err),
5948	.AGAIN => return error.WouldBlock,
5949	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed
5950	.IO => return error.InputOutput,
5951	.ISDIR => return error.IsDir,
5952	.NOBUFS => return error.SystemResources,
5953	.NOMEM => return error.SystemResources,
5954	.NOTCONN => return error.SocketUnconnected,
5955	.CONNRESET => return error.ConnectionResetByPeer,
5956	else => \|err\| return unexpectedErrno(err),
5957	}
5958	}
5959	}
5960
5961	fn pwritev(
5962	ev: *Evented,
5963	cancel_region: *CancelRegion,
5964	fd: fd_t,
5965	iov: []const iovec_const,
5966	offset: ?u64,
5967	) File.Writer.Error!usize {
5968	if (iov.len == 0) return 0;
5969	const scatter = iov.len > 1 or iov[0].len > 0xfffff000;
5970	while (true) {
5971	const thread = try cancel_region.awaitIoUring();
5972	thread.enqueue().* = .{
5973	.opcode = if (scatter) .WRITEV else .WRITE,
5974	.flags = 0,
5975	.ioprio = 0,
5976	.fd = fd,
5977	.off = offset orelse std.math.maxInt(u64),
5978	.addr = if (scatter) @intFromPtr(iov.ptr) else @intFromPtr(iov[0].base),
5979	.len = @intCast(if (scatter) iov.len else iov[0].len),
5980	.rw_flags = 0,
5981	.user_data = @intFromPtr(cancel_region.fiber),
5982	.buf_index = 0,
5983	.personality = 0,
5984	.splice_fd_in = 0,
5985	.addr3 = 0,
5986	.resv = 0,
5987	};
5988	ev.yield(null, .nothing);
5989	const completion = cancel_region.completion();
5990	switch (completion.errno()) {
5991	.SUCCESS => return @as(u32, @bitCast(completion.result)),
5992	.INTR, .CANCELED => continue,
5993	.INVAL => \|err\| return errnoBug(err),
5994	.FAULT => \|err\| return errnoBug(err),
5995	.AGAIN => return error.WouldBlock,
5996	.BADF => return error.NotOpenForWriting, // Can be a race condition.
5997	.DESTADDRREQ => \|err\| return errnoBug(err), // `connect` was never called.
5998	.DQUOT => return error.DiskQuota,
5999	.FBIG => return error.FileTooBig,
6000	.IO => return error.InputOutput,
6001	.NOSPC => return error.NoSpaceLeft,
6002	.PERM => return error.PermissionDenied,
6003	.PIPE => return error.BrokenPipe,
6004	.CONNRESET => \|err\| return errnoBug(err), // Not a socket handle.
6005	.BUSY => return error.DeviceBusy,
6006	else => \|err\| return unexpectedErrno(err),
6007	}
6008	}
6009	}
6010
6011	fn readAll(
6012	ev: *Evented,
6013	cancel_region: *CancelRegion,
6014	fd: fd_t,
6015	buffer: []u8,
6016	) (File.Reader.Error \|\| error{EndOfStream})!void {
6017	var index: usize = 0;
6018	while (buffer.len - index != 0) {
6019	const len = try ev.preadv(cancel_region, fd, &.{
6020	.{ .base = buffer[index..].ptr, .len = buffer.len - index },
6021	}, null);
6022	if (len == 0) return error.EndOfStream;
6023	index += len;
6024	}
6025	}
6026
6027	fn realPath(
6028	ev: *Evented,
6029	sync: *CancelRegion.Sync,
6030	fd: fd_t,
6031	out_buffer: []u8,
6032	) File.RealPathError!usize {
6033	_ = ev;
6034	var procfs_buf: [std.fmt.count("/proc/self/fd/{d}\x00", .{std.math.minInt(fd_t)})]u8 = undefined;
6035	const proc_path = std.fmt.bufPrintSentinel(&procfs_buf, "/proc/self/fd/{d}", .{fd}, 0) catch
6036	unreachable;
6037	while (true) {
6038	try sync.cancel_region.await(.nothing);
6039	const rc = linux.readlink(proc_path, out_buffer.ptr, out_buffer.len);
6040	switch (linux.errno(rc)) {
6041	.SUCCESS => return rc,
6042	.INTR => continue,
6043	.ACCES => return error.AccessDenied,
6044	.FAULT => \|err\| return errnoBug(err),
6045	.IO => return error.FileSystem,
6046	.LOOP => return error.SymLinkLoop,
6047	.NAMETOOLONG => return error.NameTooLong,
6048	.NOENT => return error.FileNotFound,
6049	.NOMEM => return error.SystemResources,
6050	.NOTDIR => return error.NotDir,
6051	.ILSEQ => \|err\| return errnoBug(err),
6052	else => \|err\| return unexpectedErrno(err),
6053	}
6054	}
6055	}
6056
6057	fn renameat(
6058	ev: *Evented,
6059	cancel_region: *CancelRegion,
6060	old_dir: fd_t,
6061	old_path: [*:0]const u8,
6062	new_dir: fd_t,
6063	new_path: [*:0]const u8,
6064	flags: linux.RENAME,
6065	) Dir.RenameError!void {
6066	while (true) {
6067	const thread = try cancel_region.awaitIoUring();
6068	thread.enqueue().* = .{
6069	.opcode = .RENAMEAT,
6070	.flags = 0,
6071	.ioprio = 0,
6072	.fd = old_dir,
6073	.off = @intFromPtr(new_path),
6074	.addr = @intFromPtr(old_path),
6075	.len = @bitCast(new_dir),
6076	.rw_flags = @bitCast(flags),
6077	.user_data = @intFromPtr(cancel_region.fiber),
6078	.buf_index = 0,
6079	.personality = 0,
6080	.splice_fd_in = 0,
6081	.addr3 = 0,
6082	.resv = 0,
6083	};
6084	ev.yield(null, .nothing);
6085	switch (cancel_region.errno()) {
6086	.SUCCESS => return,
6087	.INTR, .CANCELED => continue,
6088	.ACCES => return error.AccessDenied,
6089	.PERM => return error.PermissionDenied,
6090	.BUSY => return error.FileBusy,
6091	.DQUOT => return error.DiskQuota,
6092	.ISDIR => return error.IsDir,
6093	.IO => return error.HardwareFailure,
6094	.LOOP => return error.SymLinkLoop,
6095	.MLINK => return error.LinkQuotaExceeded,
6096	.NAMETOOLONG => return error.NameTooLong,
6097	.NOENT => return error.FileNotFound,
6098	.NOTDIR => return error.NotDir,
6099	.NOMEM => return error.SystemResources,
6100	.NOSPC => return error.NoSpaceLeft,
6101	.EXIST => return error.DirNotEmpty,
6102	.NOTEMPTY => return error.DirNotEmpty,
6103	.ROFS => return error.ReadOnlyFileSystem,
6104	.XDEV => return error.CrossDevice,
6105	.ILSEQ => return error.BadPathName,
6106	.FAULT => \|err\| return errnoBug(err),
6107	.INVAL => \|err\| return errnoBug(err),
6108	else => \|err\| return unexpectedErrno(err),
6109	}
6110	}
6111	}
6112
6113	fn setsockopt(
6114	ev: *Evented,
6115	cancel_region: *CancelRegion,
6116	fd: fd_t,
6117	level: i32,
6118	opt_name: u32,
6119	option: u32,
6120	) !void {
6121	const o: []const u8 = @ptrCast(&option);
6122	while (true) {
6123	const off: extern struct {
6124	cmd_op: linux.IO_URING_SOCKET_OP,
6125	pad: u32,
6126	} align(@alignOf(u64)) = .{
6127	.cmd_op = .SETSOCKOPT,
6128	.pad = 0,
6129	};
6130	const addr: extern struct { level: i32, opt_name: u32 } align(@alignOf(u64)) = .{
6131	.level = level,
6132	.opt_name = opt_name,
6133	};
6134	const thread = try cancel_region.awaitIoUring();
6135	thread.enqueue().* = .{
6136	.opcode = .URING_CMD,
6137	.flags = 0,
6138	.ioprio = 0,
6139	.fd = fd,
6140	.off = @as(const u64, @ptrCast(&off)).,
6141	.addr = @as(const u64, @ptrCast(&addr)).,
6142	.len = 0,
6143	.rw_flags = 0,
6144	.user_data = @intFromPtr(cancel_region.fiber),
6145	.buf_index = 0,
6146	.personality = 0,
6147	.splice_fd_in = @intCast(o.len),
6148	.addr3 = @intFromPtr(o.ptr),
6149	.resv = 0,
6150	};
6151	ev.yield(null, .nothing);
6152	switch (cancel_region.errno()) {
6153	.SUCCESS => return,
6154	.INTR, .CANCELED => continue,
6155	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
6156	.NOTSOCK => \|err\| return errnoBug(err),
6157	.INVAL => \|err\| return errnoBug(err),
6158	.FAULT => \|err\| return errnoBug(err),
6159	else => \|err\| return unexpectedErrno(err),
6160	}
6161	}
6162	}
6163
6164	fn socket(
6165	ev: *Evented,
6166	cancel_region: *CancelRegion,
6167	family: linux.sa_family_t,
6168	options: net.IpAddress.BindOptions,
6169	) error{
6170	AddressFamilyUnsupported,
6171	ProtocolUnsupportedBySystem,
6172	ProcessFdQuotaExceeded,
6173	SystemFdQuotaExceeded,
6174	SystemResources,
6175	ProtocolUnsupportedByAddressFamily,
6176	SocketModeUnsupported,
6177	OptionUnsupported,
6178	Unexpected,
6179	Canceled,
6180	}!fd_t {
6181	const mode = posixSocketMode(options.mode);
6182	const protocol = posixProtocol(options.protocol);
6183	const socket_fd = while (true) {
6184	const thread = try cancel_region.awaitIoUring();
6185	thread.enqueue().* = .{
6186	.opcode = .SOCKET,
6187	.flags = 0,
6188	.ioprio = 0,
6189	.fd = family,
6190	.off = mode \| linux.SOCK.CLOEXEC,
6191	.addr = 0,
6192	.len = protocol,
6193	.rw_flags = 0,
6194	.user_data = @intFromPtr(cancel_region.fiber),
6195	.buf_index = 0,
6196	.personality = 0,
6197	.splice_fd_in = 0,
6198	.addr3 = 0,
6199	.resv = 0,
6200	};
6201	ev.yield(null, .nothing);
6202	const completion = cancel_region.completion();
6203	switch (completion.errno()) {
6204	.SUCCESS => break completion.result,
6205	.INTR, .CANCELED => continue,
6206	.AFNOSUPPORT => return error.AddressFamilyUnsupported,
6207	.INVAL => return error.ProtocolUnsupportedBySystem,
6208	.MFILE => return error.ProcessFdQuotaExceeded,
6209	.NFILE => return error.SystemFdQuotaExceeded,
6210	.NOBUFS => return error.SystemResources,
6211	.NOMEM => return error.SystemResources,
6212	.PROTONOSUPPORT => return error.ProtocolUnsupportedByAddressFamily,
6213	.PROTOTYPE => return error.SocketModeUnsupported,
6214	else => \|err\| return unexpectedErrno(err),
6215	}
6216	};
6217	errdefer ev.close(cancel_region, socket_fd);
6218
6219	if (options.ip6_only) {
6220	if (linux.IPV6 == void) return error.OptionUnsupported;
6221	try ev.setsockopt(cancel_region, socket_fd, linux.IPPROTO.IPV6, linux.IPV6.V6ONLY, 0);
6222	}
6223
6224	return socket_fd;
6225	}
6226
6227	fn stat(ev: Evented, cancel_region: CancelRegion, fd: fd_t) Dir.StatError!Dir.Stat {
6228	return ev.statx(cancel_region, fd, "", linux.AT.EMPTY_PATH) catch \|err\| switch (err) {
6229	error.BadPathName, error.NameTooLong => unreachable, // path is empty
6230	error.AccessDenied => return errnoBug(.ACCES),
6231	error.SymLinkLoop => return errnoBug(.LOOP),
6232	error.FileNotFound => return errnoBug(.NOENT),
6233	error.NotDir => return errnoBug(.NOTDIR),
6234	else => \|e\| return e,
6235	};
6236	}
6237
6238	fn statx(
6239	ev: *Evented,
6240	cancel_region: *CancelRegion,
6241	dir: fd_t,
6242	path: [*:0]const u8,
6243	flags: u32,
6244	) (Dir.StatError \|\| Dir.PathNameError \|\| error{ FileNotFound, NotDir, SymLinkLoop })!Dir.Stat {
6245	while (true) {
6246	var statx_buf = std.mem.zeroes(linux.Statx);
6247	const thread = try cancel_region.awaitIoUring();
6248	thread.enqueue().* = .{
6249	.opcode = .STATX,
6250	.flags = 0,
6251	.ioprio = 0,
6252	.fd = dir,
6253	.off = @intFromPtr(&statx_buf),
6254	.addr = @intFromPtr(path),
6255	.len = @bitCast(linux_statx_request),
6256	.rw_flags = flags,
6257	.user_data = @intFromPtr(cancel_region.fiber),
6258	.buf_index = 0,
6259	.personality = 0,
6260	.splice_fd_in = 0,
6261	.addr3 = 0,
6262	.resv = 0,
6263	};
6264	ev.yield(null, .nothing);
6265	switch (cancel_region.errno()) {
6266	.SUCCESS => return statFromLinux(&statx_buf),
6267	.INTR, .CANCELED => continue,
6268	.ACCES => return error.AccessDenied,
6269	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
6270	.FAULT => \|err\| return errnoBug(err),
6271	.INVAL => \|err\| return errnoBug(err),
6272	.LOOP => return error.SymLinkLoop,
6273	.NAMETOOLONG => \|err\| return errnoBug(err),
6274	.NOENT => return error.FileNotFound,
6275	.NOTDIR => return error.NotDir,
6276	.NOMEM => return error.SystemResources,
6277	else => \|err\| return unexpectedErrno(err),
6278	}
6279	}
6280	}
6281
6282	fn urandomReadAll(
6283	ev: *Evented,
6284	cancel_region: *CancelRegion,
6285	buffer: []u8,
6286	) (File.OpenError \|\| File.Reader.Error \|\| error{EndOfStream})!void {
6287	return ev.readAll(cancel_region, try ev.random_fd.open(ev, cancel_region, "/dev/urandom", .{
6288	.ACCMODE = .RDONLY,
6289	.CLOEXEC = true,
6290	}), buffer);
6291	}
6292
6293	fn utimensat(
6294	ev: *Evented,
6295	sync: *CancelRegion.Sync,
6296	dir: fd_t,
6297	path: [*:0]const u8,
6298	times: ?*const [2]linux.timespec,
6299	flags: u32,
6300	) File.SetTimestampsError!void {
6301	_ = ev;
6302	while (true) {
6303	try sync.cancel_region.await(.nothing);
6304	switch (linux.errno(linux.utimensat(dir, path, times, flags))) {
6305	.SUCCESS => return,
6306	.INTR => continue,
6307	.BADF => \|err\| return errnoBug(err), // always a race condition
6308	.FAULT => \|err\| return errnoBug(err),
6309	.INVAL => \|err\| return errnoBug(err),
6310	.ACCES => return error.AccessDenied,
6311	.PERM => return error.PermissionDenied,
6312	.ROFS => return error.ReadOnlyFileSystem,
6313	else => \|err\| return unexpectedErrno(err),
6314	}
6315	}
6316	}
6317
6318	fn writeAll(
6319	ev: *Evented,
6320	cancel_region: *CancelRegion,
6321	fd: fd_t,
6322	buffer: []const u8,
6323	) (File.Writer.Error \|\| error{EndOfStream})!void {
6324	var index: usize = 0;
6325	while (buffer.len - index != 0) {
6326	const len = try ev.pwritev(cancel_region, fd, &.{
6327	.{ .base = buffer[index..].ptr, .len = buffer.len - index },
6328	}, null);
6329	if (len == 0) return error.EndOfStream;
6330	index += len;
6331	}
6332	}
6333
6334	test {
6335	_ = Fiber.CancelProtection;
6336	}

lib/std/Io/Kqueue.zig+14-204

...	@@ -31,8 +31,8 @@ const changes_buffer_len = 64;	...	@@ -31,8 +31,8 @@ const changes_buffer_len = 64;
31		31
32	const Thread = struct {	32	const Thread = struct {
33	thread: std.Thread,	33	thread: std.Thread,
34	idle_context: Context,	34	idle_context: Io.fiber.Context,
35	current_context: *Context,	35	current_context: *Io.fiber.Context,
36	ready_queue: ?*Fiber,	36	ready_queue: ?*Fiber,
37	kq_fd: posix.fd_t,	37	kq_fd: posix.fd_t,
38	idle_search_index: u32,	38	idle_search_index: u32,
...	@@ -74,7 +74,7 @@ const Thread = struct {	...	@@ -74,7 +74,7 @@ const Thread = struct {
74		74
75	const Fiber = struct {	75	const Fiber = struct {
76	required_align: void align(4),	76	required_align: void align(4),
77	context: Context,	77	context: Io.fiber.Context,
78	awaiter: ?*Fiber,	78	awaiter: ?*Fiber,
79	queue_next: ?*Fiber,	79	queue_next: ?*Fiber,
80	cancel_thread: ?*Thread,	80	cancel_thread: ?*Thread,
...	@@ -291,12 +291,12 @@ fn yield(k: Kqueue, maybe_ready_fiber: ?Fiber, pending_task: SwitchMessage.Pen	...	@@ -291,12 +291,12 @@ fn yield(k: Kqueue, maybe_ready_fiber: ?Fiber, pending_task: SwitchMessage.Pen
291	&thread.idle_context;	291	&thread.idle_context;
292	const message: SwitchMessage = .{	292	const message: SwitchMessage = .{
293	.contexts = .{	293	.contexts = .{
294	.prev = thread.current_context,	294	.old = thread.current_context,
295	.ready = ready_context,	295	.new = ready_context,
296	},	296	},
297	.pending_task = pending_task,	297	.pending_task = pending_task,
298	};	298	};
299	std.log.debug("switching from {} to {}", .{ message.contexts.prev, message.contexts.ready });	299	std.log.debug("switching from {} to {}", .{ message.contexts.old, message.contexts.new });
300	contextSwitch(&message).handle(k);	300	contextSwitch(&message).handle(k);
301	}	301	}
302		302
...	@@ -393,7 +393,7 @@ fn schedule(k: Kqueue, thread: Thread, ready_queue: Fiber.Queue) void {	...	@@ -393,7 +393,7 @@ fn schedule(k: Kqueue, thread: Thread, ready_queue: Fiber.Queue) void {
393	)) \|old_head\| ready_queue.tail.queue_next = old_head;	393	)) \|old_head\| ready_queue.tail.queue_next = old_head;
394	}	394	}
395		395
396	fn mainIdle(k: Kqueue, message: const SwitchMessage) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Context)))) noreturn {	396	fn mainIdle(k: Kqueue, message: const SwitchMessage) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Io.fiber.Context)))) noreturn {
397	message.handle(k);	397	message.handle(k);
398	k.idle(&k.threads.allocated[0]);	398	k.idle(&k.threads.allocated[0]);
399	k.yield(@ptrCast(&k.main_fiber_buffer), .nothing);	399	k.yield(@ptrCast(&k.main_fiber_buffer), .nothing);
...	@@ -483,10 +483,7 @@ fn idle(k: Kqueue, thread: Thread) void {	...	@@ -483,10 +483,7 @@ fn idle(k: Kqueue, thread: Thread) void {
483	}	483	}
484		484
485	const SwitchMessage = struct {	485	const SwitchMessage = struct {
486	contexts: extern struct {	486	contexts: Io.fiber.Switch,
487	prev: *Context,
488	ready: *Context,
489	},
490	pending_task: PendingTask,	487	pending_task: PendingTask,
491		488
492	const PendingTask = union(enum) {	489	const PendingTask = union(enum) {
...	@@ -500,11 +497,11 @@ const SwitchMessage = struct {	...	@@ -500,11 +497,11 @@ const SwitchMessage = struct {
500		497
501	fn handle(message: const SwitchMessage, k: Kqueue) void {	498	fn handle(message: const SwitchMessage, k: Kqueue) void {
502	const thread: *Thread = .current();	499	const thread: *Thread = .current();
503	thread.current_context = message.contexts.ready;	500	thread.current_context = message.contexts.new;
504	switch (message.pending_task) {	501	switch (message.pending_task) {
505	.nothing => {},	502	.nothing => {},
506	.reschedule => if (message.contexts.prev != &thread.idle_context) {	503	.reschedule => if (message.contexts.old != &thread.idle_context) {
507	const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));	504	const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
508	assert(prev_fiber.queue_next == null);	505	assert(prev_fiber.queue_next == null);
509	k.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });	506	k.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });
510	},	507	},
...	@@ -512,13 +509,13 @@ const SwitchMessage = struct {	...	@@ -512,13 +509,13 @@ const SwitchMessage = struct {
512	k.recycle(fiber);	509	k.recycle(fiber);
513	},	510	},
514	.register_awaiter => \|awaiter\| {	511	.register_awaiter => \|awaiter\| {
515	const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));	512	const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
516	assert(prev_fiber.queue_next == null);	513	assert(prev_fiber.queue_next == null);
517	if (@atomicRmw(?*Fiber, awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished)	514	if (@atomicRmw(?*Fiber, awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished)
518	k.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });	515	k.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });
519	},	516	},
520	.register_select => \|futures\| {	517	.register_select => \|futures\| {
521	const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));	518	const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
522	assert(prev_fiber.queue_next == null);	519	assert(prev_fiber.queue_next == null);
523	for (futures) \|any_future\| {	520	for (futures) \|any_future\| {
524	const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));	521	const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));
...	@@ -550,195 +547,8 @@ const SwitchMessage = struct {	...	@@ -550,195 +547,8 @@ const SwitchMessage = struct {
550	}	547	}
551	};	548	};
552		549
553	const Context = switch (builtin.cpu.arch) {
554	.aarch64 => extern struct {
555	sp: u64,
556	fp: u64,
557	pc: u64,
558	},
559	.x86_64 => extern struct {
560	rsp: u64,
561	rbp: u64,
562	rip: u64,
563	},
564	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
565	};
566
567	inline fn contextSwitch(message: const SwitchMessage) const SwitchMessage {	550	inline fn contextSwitch(message: const SwitchMessage) const SwitchMessage {
568	return @fieldParentPtr("contexts", switch (builtin.cpu.arch) {	551	return @fieldParentPtr("contexts", Io.fiber.contextSwitch(&message.contexts));
569	.aarch64 => asm volatile (
570	\\ ldp x0, x2, [x1]
571	\\ ldr x3, [x2, #16]
572	\\ mov x4, sp
573	\\ stp x4, fp, [x0]
574	\\ adr x5, 0f
575	\\ ldp x4, fp, [x2]
576	\\ str x5, [x0, #16]
577	\\ mov sp, x4
578	\\ br x3
579	\\0:
580	: [received_message] "={x1}" (-> *const @FieldType(SwitchMessage, "contexts")),
581	: [message_to_send] "{x1}" (&message.contexts),
582	: .{
583	.x0 = true,
584	.x1 = true,
585	.x2 = true,
586	.x3 = true,
587	.x4 = true,
588	.x5 = true,
589	.x6 = true,
590	.x7 = true,
591	.x8 = true,
592	.x9 = true,
593	.x10 = true,
594	.x11 = true,
595	.x12 = true,
596	.x13 = true,
597	.x14 = true,
598	.x15 = true,
599	.x16 = true,
600	.x17 = true,
601	.x19 = true,
602	.x20 = true,
603	.x21 = true,
604	.x22 = true,
605	.x23 = true,
606	.x24 = true,
607	.x25 = true,
608	.x26 = true,
609	.x27 = true,
610	.x28 = true,
611	.x30 = true,
612	.z0 = true,
613	.z1 = true,
614	.z2 = true,
615	.z3 = true,
616	.z4 = true,
617	.z5 = true,
618	.z6 = true,
619	.z7 = true,
620	.z8 = true,
621	.z9 = true,
622	.z10 = true,
623	.z11 = true,
624	.z12 = true,
625	.z13 = true,
626	.z14 = true,
627	.z15 = true,
628	.z16 = true,
629	.z17 = true,
630	.z18 = true,
631	.z19 = true,
632	.z20 = true,
633	.z21 = true,
634	.z22 = true,
635	.z23 = true,
636	.z24 = true,
637	.z25 = true,
638	.z26 = true,
639	.z27 = true,
640	.z28 = true,
641	.z29 = true,
642	.z30 = true,
643	.z31 = true,
644	.p0 = true,
645	.p1 = true,
646	.p2 = true,
647	.p3 = true,
648	.p4 = true,
649	.p5 = true,
650	.p6 = true,
651	.p7 = true,
652	.p8 = true,
653	.p9 = true,
654	.p10 = true,
655	.p11 = true,
656	.p12 = true,
657	.p13 = true,
658	.p14 = true,
659	.p15 = true,
660	.fpcr = true,
661	.fpsr = true,
662	.ffr = true,
663	.memory = true,
664	}),
665	.x86_64 => asm volatile (
666	\\ movq 0(%%rsi), %%rax
667	\\ movq 8(%%rsi), %%rcx
668	\\ leaq 0f(%%rip), %%rdx
669	\\ movq %%rsp, 0(%%rax)
670	\\ movq %%rbp, 8(%%rax)
671	\\ movq %%rdx, 16(%%rax)
672	\\ movq 0(%%rcx), %%rsp
673	\\ movq 8(%%rcx), %%rbp
674	\\ jmpq *16(%%rcx)
675	\\0:
676	: [received_message] "={rsi}" (-> *const @FieldType(SwitchMessage, "contexts")),
677	: [message_to_send] "{rsi}" (&message.contexts),
678	: .{
679	.rax = true,
680	.rcx = true,
681	.rdx = true,
682	.rbx = true,
683	.rsi = true,
684	.rdi = true,
685	.r8 = true,
686	.r9 = true,
687	.r10 = true,
688	.r11 = true,
689	.r12 = true,
690	.r13 = true,
691	.r14 = true,
692	.r15 = true,
693	.mm0 = true,
694	.mm1 = true,
695	.mm2 = true,
696	.mm3 = true,
697	.mm4 = true,
698	.mm5 = true,
699	.mm6 = true,
700	.mm7 = true,
701	.zmm0 = true,
702	.zmm1 = true,
703	.zmm2 = true,
704	.zmm3 = true,
705	.zmm4 = true,
706	.zmm5 = true,
707	.zmm6 = true,
708	.zmm7 = true,
709	.zmm8 = true,
710	.zmm9 = true,
711	.zmm10 = true,
712	.zmm11 = true,
713	.zmm12 = true,
714	.zmm13 = true,
715	.zmm14 = true,
716	.zmm15 = true,
717	.zmm16 = true,
718	.zmm17 = true,
719	.zmm18 = true,
720	.zmm19 = true,
721	.zmm20 = true,
722	.zmm21 = true,
723	.zmm22 = true,
724	.zmm23 = true,
725	.zmm24 = true,
726	.zmm25 = true,
727	.zmm26 = true,
728	.zmm27 = true,
729	.zmm28 = true,
730	.zmm29 = true,
731	.zmm30 = true,
732	.zmm31 = true,
733	.fpsr = true,
734	.fpcr = true,
735	.mxcsr = true,
736	.rflags = true,
737	.dirflag = true,
738	.memory = true,
739	}),
740	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
741	});
742	}	552	}
743		553
744	fn mainIdleEntry() callconv(.naked) void {	554	fn mainIdleEntry() callconv(.naked) void {

lib/std/Io/Reader.zig+1-1

...	@@ -375,7 +375,7 @@ pub fn appendRemainingAligned(	...	@@ -375,7 +375,7 @@ pub fn appendRemainingAligned(
375	defer list.* = a.toArrayListAligned(alignment);	375	defer list.* = a.toArrayListAligned(alignment);
376		376
377	var remaining = limit;	377	var remaining = limit;
378	while (remaining.nonzero()) {	378	while (remaining != .nothing) {
379	const n = stream(r, &a.writer, remaining) catch \|err\| switch (err) {	379	const n = stream(r, &a.writer, remaining) catch \|err\| switch (err) {
380	error.EndOfStream => return,	380	error.EndOfStream => return,
381	error.WriteFailed => return error.OutOfMemory,	381	error.WriteFailed => return error.OutOfMemory,

lib/std/Io/Threaded.zig+154-93

...	@@ -72,6 +72,9 @@ stderr_mutex_locker: std.Thread.Id = Thread.invalid_id,	...	@@ -72,6 +72,9 @@ stderr_mutex_locker: std.Thread.Id = Thread.invalid_id,
72	stderr_mutex_lock_count: usize = 0,	72	stderr_mutex_lock_count: usize = 0,
73		73
74	argv0: Argv0,	74	argv0: Argv0,
		75	/// Protected by `mutex`. Determines whether `environ` has been
		76	/// memoized based on `process_environ`.
		77	environ_initialized: bool,
75	environ: Environ,	78	environ: Environ,
76		79
77	null_file: NullFile = .{},	80	null_file: NullFile = .{},
...	@@ -125,9 +128,6 @@ pub const Argv0 = switch (native_os) {	...	@@ -125,9 +128,6 @@ pub const Argv0 = switch (native_os) {
125	pub const Environ = struct {	128	pub const Environ = struct {
126	/// Unmodified data directly from the OS.	129	/// Unmodified data directly from the OS.
127	process_environ: process.Environ,	130	process_environ: process.Environ,
128	/// Protected by `mutex`. Determines whether the other fields have been
129	/// memoized based on `process_environ`.
130	initialized: bool = false,
131	/// Protected by `mutex`. Memoized based on `process_environ`. Tracks whether the	131	/// Protected by `mutex`. Memoized based on `process_environ`. Tracks whether the
132	/// environment variables are present, ignoring their value.	132	/// environment variables are present, ignoring their value.
133	exist: Exist = .{},	133	exist: Exist = .{},
...	@@ -161,9 +161,6 @@ pub const Environ = struct {	...	@@ -161,9 +161,6 @@ pub const Environ = struct {
161	};	161	};
162		162
163	pub fn scan(environ: *Environ, allocator: std.mem.Allocator) void {	163	pub fn scan(environ: *Environ, allocator: std.mem.Allocator) void {
164	if (environ.initialized) return;
165	environ.initialized = true;
166
167	if (is_windows) {	164	if (is_windows) {
168	// This value expires with any call that modifies the environment,	165	// This value expires with any call that modifies the environment,
169	// which is outside of this Io implementation's control, so references	166	// which is outside of this Io implementation's control, so references
...	@@ -1589,6 +1586,7 @@ pub fn init(	...	@@ -1589,6 +1586,7 @@ pub fn init(
1589	.old_sig_pipe = undefined,	1586	.old_sig_pipe = undefined,
1590	.have_signal_handler = init_single_threaded.have_signal_handler,	1587	.have_signal_handler = init_single_threaded.have_signal_handler,
1591	.argv0 = options.argv0,	1588	.argv0 = options.argv0,
		1589	.environ_initialized = options.environ.block.isEmpty(),
1592	.environ = .{ .process_environ = options.environ },	1590	.environ = .{ .process_environ = options.environ },
1593	.worker_threads = init_single_threaded.worker_threads,	1591	.worker_threads = init_single_threaded.worker_threads,
1594	.disable_memory_mapping = options.disable_memory_mapping,	1592	.disable_memory_mapping = options.disable_memory_mapping,
...	@@ -1606,6 +1604,7 @@ pub fn init(	...	@@ -1606,6 +1604,7 @@ pub fn init(
1606	.old_sig_pipe = undefined,	1604	.old_sig_pipe = undefined,
1607	.have_signal_handler = false,	1605	.have_signal_handler = false,
1608	.argv0 = options.argv0,	1606	.argv0 = options.argv0,
		1607	.environ_initialized = options.environ.block.isEmpty(),
1609	.environ = .{ .process_environ = options.environ },	1608	.environ = .{ .process_environ = options.environ },
1610	.worker_threads = .init(null),	1609	.worker_threads = .init(null),
1611	.disable_memory_mapping = options.disable_memory_mapping,	1610	.disable_memory_mapping = options.disable_memory_mapping,
...	@@ -1643,9 +1642,8 @@ pub const init_single_threaded: Threaded = .{	...	@@ -1643,9 +1642,8 @@ pub const init_single_threaded: Threaded = .{
1643	.old_sig_pipe = undefined,	1642	.old_sig_pipe = undefined,
1644	.have_signal_handler = false,	1643	.have_signal_handler = false,
1645	.argv0 = .empty,	1644	.argv0 = .empty,
1646	.environ = .{ .process_environ = .{	1645	.environ_initialized = true,
1647	.block = if (process.Environ.Block == process.Environ.GlobalBlock) .global else .empty,	1646	.environ = .empty,
1648	} },
1649	.worker_threads = .init(null),	1647	.worker_threads = .init(null),
1650	.disable_memory_mapping = false,	1648	.disable_memory_mapping = false,
1651	};	1649	};
...	@@ -1768,6 +1766,8 @@ pub fn io(t: *Threaded) Io {	...	@@ -1768,6 +1766,8 @@ pub fn io(t: *Threaded) Io {
1768	return .{	1766	return .{
1769	.userdata = t,	1767	.userdata = t,
1770	.vtable = &.{	1768	.vtable = &.{
		1769	.crashHandler = crashHandler,
		1770
1771	.async = async,	1771	.async = async,
1772	.concurrent = concurrent,	1772	.concurrent = concurrent,
1773	.await = await,	1773	.await = await,
...	@@ -1932,6 +1932,8 @@ pub fn ioBasic(t: *Threaded) Io {	...	@@ -1932,6 +1932,8 @@ pub fn ioBasic(t: *Threaded) Io {
1932	return .{	1932	return .{
1933	.userdata = t,	1933	.userdata = t,
1934	.vtable = &.{	1934	.vtable = &.{
		1935	.crashHandler = crashHandler,
		1936
1935	.async = async,	1937	.async = async,
1936	.concurrent = concurrent,	1938	.concurrent = concurrent,
1937	.await = await,	1939	.await = await,
...	@@ -2157,6 +2159,14 @@ const use_libc_getrandom = std.c.versionCheck(if (builtin.abi.isAndroid()) .{	...	@@ -2157,6 +2159,14 @@ const use_libc_getrandom = std.c.versionCheck(if (builtin.abi.isAndroid()) .{
2157		2159
2158	const use_dev_urandom = @TypeOf(posix.system.getrandom) == void and native_os == .linux;	2160	const use_dev_urandom = @TypeOf(posix.system.getrandom) == void and native_os == .linux;
2159		2161
		2162	fn crashHandler(userdata: ?*anyopaque) void {
		2163	const t: *Threaded = @ptrCast(@alignCast(userdata));
		2164	_ = t;
		2165	const thread = Thread.current orelse return;
		2166	thread.status.store(.{ .cancelation = .canceled, .awaitable = .null }, .monotonic);
		2167	thread.cancel_protection = .blocked;
		2168	}
		2169
2160	fn async(	2170	fn async(
2161	userdata: ?*anyopaque,	2171	userdata: ?*anyopaque,
2162	result: []u8,	2172	result: []u8,
...	@@ -2838,19 +2848,19 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout	...	@@ -2838,19 +2848,19 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2838	var poll_buffer: [poll_buffer_len]posix.pollfd = undefined;	2848	var poll_buffer: [poll_buffer_len]posix.pollfd = undefined;
2839	var poll_storage: struct {	2849	var poll_storage: struct {
2840	gpa: std.mem.Allocator,	2850	gpa: std.mem.Allocator,
2841	b: *Io.Batch,	2851	batch: *Io.Batch,
2842	slice: []posix.pollfd,	2852	slice: []posix.pollfd,
2843	len: u32,	2853	len: u32,
2844		2854
2845	fn add(storage: *@This(), file: Io.File, events: @FieldType(posix.pollfd, "events")) Io.ConcurrentError!void {	2855	fn add(storage: *@This(), file: Io.File, events: @FieldType(posix.pollfd, "events")) Io.ConcurrentError!void {
2846	const len = storage.len;	2856	const len = storage.len;
2847	if (len == poll_buffer_len) {	2857	if (len == poll_buffer_len) {
2848	const slice: []posix.pollfd = if (storage.b.context) \|context\|	2858	const slice: []posix.pollfd = if (storage.batch.userdata) \|batch_userdata\|
2849	@as([*]posix.pollfd, @ptrCast(@alignCast(context)))[0..storage.b.storage.len]	2859	@as([*]posix.pollfd, @ptrCast(@alignCast(batch_userdata)))[0..storage.batch.storage.len]
2850	else allocation: {	2860	else allocation: {
2851	const allocation = storage.gpa.alloc(posix.pollfd, storage.b.storage.len) catch	2861	const allocation = storage.gpa.alloc(posix.pollfd, storage.batch.storage.len) catch
2852	return error.ConcurrencyUnavailable;	2862	return error.ConcurrencyUnavailable;
2853	storage.b.context = allocation.ptr;	2863	storage.batch.userdata = allocation.ptr;
2854	break :allocation allocation;	2864	break :allocation allocation;
2855	};	2865	};
2856	@memcpy(slice[0..poll_buffer_len], storage.slice);	2866	@memcpy(slice[0..poll_buffer_len], storage.slice);
...	@@ -2863,7 +2873,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout	...	@@ -2863,7 +2873,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2863	};	2873	};
2864	storage.len = len + 1;	2874	storage.len = len + 1;
2865	}	2875	}
2866	} = .{ .gpa = t.allocator, .b = b, .slice = &poll_buffer, .len = 0 };	2876	} = .{ .gpa = t.allocator, .batch = b, .slice = &poll_buffer, .len = 0 };
2867	{	2877	{
2868	var index = b.submitted.head;	2878	var index = b.submitted.head;
2869	while (index != .none) {	2879	while (index != .none) {
...	@@ -2962,21 +2972,21 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout	...	@@ -2962,21 +2972,21 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2962	}	2972	}
2963	}	2973	}
2964		2974
2965	const WindowsBatchPendingOperationContext = extern struct {	2975	const WindowsBatchOperationUserdata = extern struct {
2966	file: windows.HANDLE,	2976	file: windows.HANDLE,
2967	iosb: windows.IO_STATUS_BLOCK,	2977	iosb: windows.IO_STATUS_BLOCK,
2968		2978
2969	const Erased = Io.Operation.Storage.Pending.Context;	2979	const Erased = Io.Operation.Storage.Pending.Userdata;
2970		2980
2971	comptime {	2981	comptime {
2972	assert(@sizeOf(Erased) <= @sizeOf(WindowsBatchPendingOperationContext));	2982	assert(@sizeOf(WindowsBatchOperationUserdata) <= @sizeOf(Erased));
2973	}	2983	}
2974		2984
2975	fn toErased(context: WindowsBatchPendingOperationContext) Erased {	2985	fn toErased(userdata: WindowsBatchOperationUserdata) Erased {
2976	return @ptrCast(context);	2986	return @ptrCast(userdata);
2977	}	2987	}
2978		2988
2979	fn fromErased(erased: Erased) WindowsBatchPendingOperationContext {	2989	fn fromErased(erased: Erased) WindowsBatchOperationUserdata {
2980	return @ptrCast(erased);	2990	return @ptrCast(erased);
2981	}	2991	}
2982	};	2992	};
...	@@ -2989,15 +2999,16 @@ fn batchCancel(userdata: ?anyopaque, b: Io.Batch) void {	...	@@ -2989,15 +2999,16 @@ fn batchCancel(userdata: ?anyopaque, b: Io.Batch) void {
2989	var index = b.pending.head;	2999	var index = b.pending.head;
2990	while (index != .none) {	3000	while (index != .none) {
2991	const pending = &b.storage[index.toIndex()].pending;	3001	const pending = &b.storage[index.toIndex()].pending;
2992	const context: *WindowsBatchPendingOperationContext = .fromErased(&pending.context);	3002	const operation_userdata: *WindowsBatchOperationUserdata = .fromErased(&pending.userdata);
2993	var cancel_iosb: windows.IO_STATUS_BLOCK = undefined;	3003	var cancel_iosb: windows.IO_STATUS_BLOCK = undefined;
2994	_ = windows.ntdll.NtCancelIoFileEx(context.file, &context.iosb, &cancel_iosb);	3004	_ = windows.ntdll.NtCancelIoFileEx(operation_userdata.file, &operation_userdata.iosb, &cancel_iosb);
2995	index = pending.node.next;	3005	index = pending.node.next;
2996	}	3006	}
2997	while (b.pending.head != .none) waitForApcOrAlert();	3007	while (b.pending.head != .none) waitForApcOrAlert();
2998	} else if (b.context) \|context\| {	3008	} else if (b.userdata) \|batch_userdata\| {
2999	t.allocator.free(@as([*]posix.pollfd, @ptrCast(@alignCast(context)))[0..b.storage.len]);	3009	const poll_storage: [*]posix.pollfd = @ptrCast(@alignCast(batch_userdata));
3000	b.context = null;	3010	t.allocator.free(poll_storage[0..b.storage.len]);
		3011	b.userdata = null;
3001	}	3012	}
3002	}	3013	}
3003		3014
...	@@ -3007,9 +3018,9 @@ fn batchApc(	...	@@ -3007,9 +3018,9 @@ fn batchApc(
3007	_: windows.ULONG,	3018	_: windows.ULONG,
3008	) callconv(.winapi) void {	3019	) callconv(.winapi) void {
3009	const b: *Io.Batch = @ptrCast(@alignCast(apc_context));	3020	const b: *Io.Batch = @ptrCast(@alignCast(apc_context));
3010	const context: *WindowsBatchPendingOperationContext = @fieldParentPtr("iosb", iosb);	3021	const operation_userdata: *WindowsBatchOperationUserdata = @fieldParentPtr("iosb", iosb);
3011	const erased_context = context.toErased();	3022	const erased_userdata = operation_userdata.toErased();
3012	const pending: *Io.Operation.Storage.Pending = @fieldParentPtr("context", erased_context);	3023	const pending: *Io.Operation.Storage.Pending = @fieldParentPtr("userdata", erased_userdata);
3013	switch (pending.node.prev) {	3024	switch (pending.node.prev) {
3014	.none => b.pending.head = pending.node.next,	3025	.none => b.pending.head = pending.node.next,
3015	else => \|prev_index\| b.storage[prev_index.toIndex()].pending.node.next = pending.node.next,	3026	else => \|prev_index\| b.storage[prev_index.toIndex()].pending.node.next = pending.node.next,
...	@@ -3019,24 +3030,23 @@ fn batchApc(	...	@@ -3019,24 +3030,23 @@ fn batchApc(
3019	else => \|next_index\| b.storage[next_index.toIndex()].pending.node.prev = pending.node.prev,	3030	else => \|next_index\| b.storage[next_index.toIndex()].pending.node.prev = pending.node.prev,
3020	}	3031	}
3021	const storage: *Io.Operation.Storage = @fieldParentPtr("pending", pending);	3032	const storage: *Io.Operation.Storage = @fieldParentPtr("pending", pending);
3022	const index = storage - b.storage.ptr;	3033	const index: Io.Operation.OptionalIndex = .fromIndex(storage - b.storage.ptr);
3023	switch (iosb.u.Status) {	3034	switch (iosb.u.Status) {
3024	.CANCELLED => {	3035	.CANCELLED => {
3025	const tail_index = b.unused.tail;	3036	const tail_index = b.unused.tail;
3026	switch (tail_index) {	3037	switch (tail_index) {
3027	.none => b.unused.head = .fromIndex(index),	3038	.none => b.unused.head = index,
3028	else => b.storage[tail_index.toIndex()].unused.next = .fromIndex(index),	3039	else => b.storage[tail_index.toIndex()].unused.next = index,
3029	}	3040	}
3030	storage.* = .{ .unused = .{ .prev = tail_index, .next = .none } };	3041	storage.* = .{ .unused = .{ .prev = tail_index, .next = .none } };
3031	b.unused.tail = .fromIndex(index);	3042	b.unused.tail = index;
3032	},	3043	},
3033	else => {	3044	else => {
3034	switch (b.completed.tail) {	3045	switch (b.completed.tail) {
3035	.none => b.completed.head = .fromIndex(index),	3046	.none => b.completed.head = index,
3036	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next =	3047	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next = index,
3037	.fromIndex(index),
3038	}	3048	}
3039	b.completed.tail = .fromIndex(index);	3049	b.completed.tail = index;
3040	const result: Io.Operation.Result = switch (pending.tag) {	3050	const result: Io.Operation.Result = switch (pending.tag) {
3041	.file_read_streaming => .{ .file_read_streaming = ntReadFileResult(iosb) },	3051	.file_read_streaming => .{ .file_read_streaming = ntReadFileResult(iosb) },
3042	.file_write_streaming => .{ .file_write_streaming = ntWriteFileResult(iosb) },	3052	.file_write_streaming => .{ .file_write_streaming = ntWriteFileResult(iosb) },
...	@@ -3057,38 +3067,38 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr	...	@@ -3057,38 +3067,38 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3057	storage.* = .{ .pending = .{	3067	storage.* = .{ .pending = .{
3058	.node = .{ .prev = b.pending.tail, .next = .none },	3068	.node = .{ .prev = b.pending.tail, .next = .none },
3059	.tag = submission.operation,	3069	.tag = submission.operation,
3060	.context = undefined,	3070	.userdata = undefined,
3061	} };	3071	} };
3062	switch (b.pending.tail) {	3072	switch (b.pending.tail) {
3063	.none => b.pending.head = index,	3073	.none => b.pending.head = index,
3064	else => \|tail_index\| b.storage[tail_index.toIndex()].pending.node.next = index,	3074	else => \|tail_index\| b.storage[tail_index.toIndex()].pending.node.next = index,
3065	}	3075	}
3066	b.pending.tail = index;	3076	b.pending.tail = index;
3067	const context: *WindowsBatchPendingOperationContext = .fromErased(&storage.pending.context);	3077	const operation_userdata: *WindowsBatchOperationUserdata = .fromErased(&storage.pending.userdata);
3068	errdefer {	3078	errdefer {
3069	context.iosb = .{ .u = .{ .Status = .CANCELLED }, .Information = undefined };	3079	operation_userdata.iosb = .{ .u = .{ .Status = .CANCELLED }, .Information = undefined };
3070	batchApc(b, &context.iosb, 0);	3080	batchApc(b, &operation_userdata.iosb, 0);
3071	}	3081	}
3072	switch (submission.operation) {	3082	switch (submission.operation) {
3073	.file_read_streaming => \|o\| o: {	3083	.file_read_streaming => \|o\| o: {
3074	var data_index: usize = 0;	3084	var data_index: usize = 0;
3075	while (o.data.len - data_index != 0 and o.data[data_index].len == 0) data_index += 1;	3085	while (o.data.len - data_index != 0 and o.data[data_index].len == 0) data_index += 1;
3076	if (o.data.len - data_index == 0) {	3086	if (o.data.len - data_index == 0) {
3077	context.iosb = .{ .u = .{ .Status = .SUCCESS }, .Information = 0 };	3087	operation_userdata.iosb = .{ .u = .{ .Status = .SUCCESS }, .Information = 0 };
3078	batchApc(b, &context.iosb, 0);	3088	batchApc(b, &operation_userdata.iosb, 0);
3079	break :o;	3089	break :o;
3080	}	3090	}
3081	const buffer = o.data[data_index];	3091	const buffer = o.data[data_index];
3082	const short_buffer_len = std.math.lossyCast(u32, buffer.len);	3092	const short_buffer_len = std.math.lossyCast(u32, buffer.len);
3083		3093
3084	if (o.file.flags.nonblocking) {	3094	if (o.file.flags.nonblocking) {
3085	context.file = o.file.handle;	3095	operation_userdata.file = o.file.handle;
3086	switch (windows.ntdll.NtReadFile(	3096	switch (windows.ntdll.NtReadFile(
3087	o.file.handle,	3097	o.file.handle,
3088	null, // event	3098	null, // event
3089	&batchApc,	3099	&batchApc,
3090	b,	3100	b,
3091	&context.iosb,	3101	&operation_userdata.iosb,
3092	buffer.ptr,	3102	buffer.ptr,
3093	short_buffer_len,	3103	short_buffer_len,
3094	null, // byte offset	3104	null, // byte offset
...	@@ -3097,8 +3107,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr	...	@@ -3097,8 +3107,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3097	.PENDING, .SUCCESS => {},	3107	.PENDING, .SUCCESS => {},
3098	.CANCELLED => unreachable,	3108	.CANCELLED => unreachable,
3099	else => \|status\| {	3109	else => \|status\| {
3100	context.iosb.u.Status = status;	3110	operation_userdata.iosb.u.Status = status;
3101	batchApc(b, &context.iosb, 0);	3111	batchApc(b, &operation_userdata.iosb, 0);
3102	},	3112	},
3103	}	3113	}
3104	} else {	3114	} else {
...	@@ -3110,7 +3120,7 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr	...	@@ -3110,7 +3120,7 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3110	null, // event	3120	null, // event
3111	null, // APC routine	3121	null, // APC routine
3112	null, // APC context	3122	null, // APC context
3113	&context.iosb,	3123	&operation_userdata.iosb,
3114	buffer.ptr,	3124	buffer.ptr,
3115	short_buffer_len,	3125	short_buffer_len,
3116	null, // byte offset	3126	null, // byte offset
...	@@ -3123,9 +3133,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr	...	@@ -3123,9 +3133,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3123	},	3133	},
3124	else => \|status\| {	3134	else => \|status\| {
3125	syscall.finish();	3135	syscall.finish();
3126		3136	operation_userdata.iosb.u.Status = status;
3127	context.iosb.u.Status = status;	3137	batchApc(b, &operation_userdata.iosb, 0);
3128	batchApc(b, &context.iosb, 0);
3129	break;	3138	break;
3130	},	3139	},
3131	};	3140	};
...	@@ -3134,18 +3143,18 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr	...	@@ -3134,18 +3143,18 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3134	.file_write_streaming => \|o\| o: {	3143	.file_write_streaming => \|o\| o: {
3135	const buffer = windowsWriteBuffer(o.header, o.data, o.splat);	3144	const buffer = windowsWriteBuffer(o.header, o.data, o.splat);
3136	if (buffer.len == 0) {	3145	if (buffer.len == 0) {
3137	context.iosb = .{ .u = .{ .Status = .SUCCESS }, .Information = 0 };	3146	operation_userdata.iosb = .{ .u = .{ .Status = .SUCCESS }, .Information = 0 };
3138	batchApc(b, &context.iosb, 0);	3147	batchApc(b, &operation_userdata.iosb, 0);
3139	break :o;	3148	break :o;
3140	}	3149	}
3141	if (o.file.flags.nonblocking) {	3150	if (o.file.flags.nonblocking) {
3142	context.file = o.file.handle;	3151	operation_userdata.file = o.file.handle;
3143	switch (windows.ntdll.NtWriteFile(	3152	switch (windows.ntdll.NtWriteFile(
3144	o.file.handle,	3153	o.file.handle,
3145	null, // event	3154	null, // event
3146	&batchApc,	3155	&batchApc,
3147	b,	3156	b,
3148	&context.iosb,	3157	&operation_userdata.iosb,
3149	buffer.ptr,	3158	buffer.ptr,
3150	@intCast(buffer.len),	3159	@intCast(buffer.len),
3151	null, // byte offset	3160	null, // byte offset
...	@@ -3154,8 +3163,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr	...	@@ -3154,8 +3163,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3154	.PENDING, .SUCCESS => {},	3163	.PENDING, .SUCCESS => {},
3155	.CANCELLED => unreachable,	3164	.CANCELLED => unreachable,
3156	else => \|status\| {	3165	else => \|status\| {
3157	context.iosb.u.Status = status;	3166	operation_userdata.iosb.u.Status = status;
3158	batchApc(b, &context.iosb, 0);	3167	batchApc(b, &operation_userdata.iosb, 0);
3159	},	3168	},
3160	}	3169	}
3161	} else {	3170	} else {
...	@@ -3167,7 +3176,7 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr	...	@@ -3167,7 +3176,7 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3167	null, // event	3176	null, // event
3168	null, // APC routine	3177	null, // APC routine
3169	null, // APC context	3178	null, // APC context
3170	&context.iosb,	3179	&operation_userdata.iosb,
3171	buffer.ptr,	3180	buffer.ptr,
3172	@intCast(buffer.len),	3181	@intCast(buffer.len),
3173	null, // byte offset	3182	null, // byte offset
...	@@ -3180,9 +3189,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr	...	@@ -3180,9 +3189,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3180	},	3189	},
3181	else => \|status\| {	3190	else => \|status\| {
3182	syscall.finish();	3191	syscall.finish();
3183		3192	operation_userdata.iosb.u.Status = status;
3184	context.iosb.u.Status = status;	3193	batchApc(b, &operation_userdata.iosb, 0);
3185	batchApc(b, &context.iosb, 0);
3186	break;	3194	break;
3187	},	3195	},
3188	};	3196	};
...	@@ -3194,13 +3202,13 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr	...	@@ -3194,13 +3202,13 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3194	else => &windows.ntdll.NtDeviceIoControlFile,	3202	else => &windows.ntdll.NtDeviceIoControlFile,
3195	};	3203	};
3196	if (o.file.flags.nonblocking) {	3204	if (o.file.flags.nonblocking) {
3197	context.file = o.file.handle;	3205	operation_userdata.file = o.file.handle;
3198	switch (NtControlFile(	3206	switch (NtControlFile(
3199	o.file.handle,	3207	o.file.handle,
3200	null, // event	3208	null, // event
3201	&batchApc,	3209	&batchApc,
3202	b,	3210	b,
3203	&context.iosb,	3211	&operation_userdata.iosb,
3204	o.code,	3212	o.code,
3205	if (o.in.len > 0) o.in.ptr else null,	3213	if (o.in.len > 0) o.in.ptr else null,
3206	@intCast(o.in.len),	3214	@intCast(o.in.len),
...	@@ -3210,8 +3218,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr	...	@@ -3210,8 +3218,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3210	.PENDING, .SUCCESS => {},	3218	.PENDING, .SUCCESS => {},
3211	.CANCELLED => unreachable,	3219	.CANCELLED => unreachable,
3212	else => \|status\| {	3220	else => \|status\| {
3213	context.iosb.u.Status = status;	3221	operation_userdata.iosb.u.Status = status;
3214	batchApc(b, &context.iosb, 0);	3222	batchApc(b, &operation_userdata.iosb, 0);
3215	},	3223	},
3216	}	3224	}
3217	} else {	3225	} else {
...	@@ -3223,7 +3231,7 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr	...	@@ -3223,7 +3231,7 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3223	null, // event	3231	null, // event
3224	null, // APC routine	3232	null, // APC routine
3225	null, // APC context	3233	null, // APC context
3226	&context.iosb,	3234	&operation_userdata.iosb,
3227	o.code,	3235	o.code,
3228	if (o.in.len > 0) o.in.ptr else null,	3236	if (o.in.len > 0) o.in.ptr else null,
3229	@intCast(o.in.len),	3237	@intCast(o.in.len),
...	@@ -3237,9 +3245,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr	...	@@ -3237,9 +3245,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3237	},	3245	},
3238	else => \|status\| {	3246	else => \|status\| {
3239	syscall.finish();	3247	syscall.finish();
3240		3248	operation_userdata.iosb.u.Status = status;
3241	context.iosb.u.Status = status;	3249	batchApc(b, &operation_userdata.iosb, 0);
3242	batchApc(b, &context.iosb, 0);
3243	break;	3250	break;
3244	},	3251	},
3245	};	3252	};
...	@@ -3816,7 +3823,13 @@ fn filePathKind(t: *Threaded, dir: Dir, sub_path: []const u8) !File.Kind {	...	@@ -3816,7 +3823,13 @@ fn filePathKind(t: *Threaded, dir: Dir, sub_path: []const u8) !File.Kind {
3816	const syscall: Syscall = try .start();	3823	const syscall: Syscall = try .start();
3817	while (true) {	3824	while (true) {
3818	var statx = std.mem.zeroes(linux.Statx);	3825	var statx = std.mem.zeroes(linux.Statx);
3819	switch (linux.errno(linux.statx(dir.handle, sub_path_posix, 0, .{ .TYPE = true }, &statx))) {	3826	switch (linux.errno(linux.statx(
		3827	dir.handle,
		3828	sub_path_posix,
		3829	linux.AT.NO_AUTOMOUNT \| linux.AT.SYMLINK_NOFOLLOW,
		3830	.{ .TYPE = true },
		3831	&statx,
		3832	))) {
3820	.SUCCESS => {	3833	.SUCCESS => {
3821	syscall.finish();	3834	syscall.finish();
3822	if (!statx.mask.TYPE) return error.Unexpected;	3835	if (!statx.mask.TYPE) return error.Unexpected;
...	@@ -3832,7 +3845,7 @@ fn filePathKind(t: *Threaded, dir: Dir, sub_path: []const u8) !File.Kind {	...	@@ -3832,7 +3845,7 @@ fn filePathKind(t: *Threaded, dir: Dir, sub_path: []const u8) !File.Kind {
3832	}	3845	}
3833	}	3846	}
3834		3847
3835	const stat = try dirStatFile(t, dir, sub_path, .{});	3848	const stat = try dirStatFile(t, dir, sub_path, .{ .follow_symlinks = false });
3836	return stat.kind;	3849	return stat.kind;
3837	}	3850	}
3838		3851
...	@@ -13573,13 +13586,13 @@ fn netWriteWindows(	...	@@ -13573,13 +13586,13 @@ fn netWriteWindows(
13573	addWsaBuf(&iovecs, &len, header);	13586	addWsaBuf(&iovecs, &len, header);
13574	for (data[0 .. data.len - 1]) \|bytes\| addWsaBuf(&iovecs, &len, bytes);	13587	for (data[0 .. data.len - 1]) \|bytes\| addWsaBuf(&iovecs, &len, bytes);
13575	const pattern = data[data.len - 1];	13588	const pattern = data[data.len - 1];
		13589	var backup_buffer: [64]u8 = undefined;
13576	if (iovecs.len - len != 0) switch (splat) {	13590	if (iovecs.len - len != 0) switch (splat) {
13577	0 => {},	13591	0 => {},
13578	1 => addWsaBuf(&iovecs, &len, pattern),	13592	1 => addWsaBuf(&iovecs, &len, pattern),
13579	else => switch (pattern.len) {	13593	else => switch (pattern.len) {
13580	0 => {},	13594	0 => {},
13581	1 => {	13595	1 => {
13582	var backup_buffer: [64]u8 = undefined;
13583	const splat_buffer = &backup_buffer;	13596	const splat_buffer = &backup_buffer;
13584	const memset_len = @min(splat_buffer.len, splat);	13597	const memset_len = @min(splat_buffer.len, splat);
13585	const buf = splat_buffer[0..memset_len];	13598	const buf = splat_buffer[0..memset_len];
...	@@ -14519,7 +14532,7 @@ pub fn statFromLinux(stx: *const std.os.linux.Statx) Io.UnexpectedError!File.Sta	...	@@ -14519,7 +14532,7 @@ pub fn statFromLinux(stx: *const std.os.linux.Statx) Io.UnexpectedError!File.Sta
14519	};	14532	};
14520	}	14533	}
14521		14534
14522	fn statxKind(stx_mode: u16) File.Kind {	14535	pub fn statxKind(stx_mode: u16) File.Kind {
14523	return switch (stx_mode & std.os.linux.S.IFMT) {	14536	return switch (stx_mode & std.os.linux.S.IFMT) {
14524	std.os.linux.S.IFDIR => .directory,	14537	std.os.linux.S.IFDIR => .directory,
14525	std.os.linux.S.IFCHR => .character_device,	14538	std.os.linux.S.IFCHR => .character_device,
...	@@ -14532,7 +14545,7 @@ fn statxKind(stx_mode: u16) File.Kind {	...	@@ -14532,7 +14545,7 @@ fn statxKind(stx_mode: u16) File.Kind {
14532	};	14545	};
14533	}	14546	}
14534		14547
14535	fn statFromPosix(st: *const posix.Stat) File.Stat {	14548	pub fn statFromPosix(st: *const posix.Stat) File.Stat {
14536	const atime = st.atime();	14549	const atime = st.atime();
14537	const mtime = st.mtime();	14550	const mtime = st.mtime();
14538	const ctime = st.ctime();	14551	const ctime = st.ctime();
...	@@ -15116,7 +15129,9 @@ const WindowsEnvironStrings = struct {	...	@@ -15116,7 +15129,9 @@ const WindowsEnvironStrings = struct {
15116	fn scanEnviron(t: *Threaded) void {	15129	fn scanEnviron(t: *Threaded) void {
15117	mutexLock(&t.mutex);	15130	mutexLock(&t.mutex);
15118	defer mutexUnlock(&t.mutex);	15131	defer mutexUnlock(&t.mutex);
		15132	if (t.environ_initialized) return;
15119	t.environ.scan(t.allocator);	15133	t.environ.scan(t.allocator);
		15134	t.environ_initialized = true;
15120	}	15135	}
15121		15136
15122	fn processReplace(userdata: ?*anyopaque, options: process.ReplaceOptions) process.ReplaceError {	15137	fn processReplace(userdata: ?*anyopaque, options: process.ReplaceOptions) process.ReplaceError {
...	@@ -15256,7 +15271,7 @@ fn spawnPosix(t: *Threaded, options: process.SpawnOptions) process.SpawnError!Sp	...	@@ -15256,7 +15271,7 @@ fn spawnPosix(t: *Threaded, options: process.SpawnOptions) process.SpawnError!Sp
15256		15271
15257	// This pipe communicates to the parent errors in the child between `fork` and `execvpe`.	15272	// This pipe communicates to the parent errors in the child between `fork` and `execvpe`.
15258	// It is closed by the child (via CLOEXEC) without writing if `execvpe` succeeds.	15273	// It is closed by the child (via CLOEXEC) without writing if `execvpe` succeeds.
15259	const err_pipe: [2]posix.fd_t = try pipe2(.{ .CLOEXEC = true });	15274	const err_pipe = try pipe2(.{ .CLOEXEC = true });
15260	errdefer destroyPipe(err_pipe);	15275	errdefer destroyPipe(err_pipe);
15261		15276
15262	t.scanEnviron(); // for PATH	15277	t.scanEnviron(); // for PATH
...	@@ -15327,7 +15342,7 @@ fn spawnPosix(t: *Threaded, options: process.SpawnOptions) process.SpawnError!Sp	...	@@ -15327,7 +15342,7 @@ fn spawnPosix(t: *Threaded, options: process.SpawnOptions) process.SpawnError!Sp
15327	}	15342	}
15328		15343
15329	if (options.start_suspended) {	15344	if (options.start_suspended) {
15330	switch (posix.errno(posix.system.kill(posix.system.getpid(), .STOP))) {	15345	switch (posix.errno(posix.system.kill(0, .STOP))) {
15331	.SUCCESS => {},	15346	.SUCCESS => {},
15332	.PERM => forkBail(ep1, error.PermissionDenied),	15347	.PERM => forkBail(ep1, error.PermissionDenied),
15333	else => forkBail(ep1, error.Unexpected),	15348	else => forkBail(ep1, error.Unexpected),
...	@@ -15539,15 +15554,15 @@ fn childCleanupWindows(child: *process.Child) void {	...	@@ -15539,15 +15554,15 @@ fn childCleanupWindows(child: *process.Child) void {
15539	windows.CloseHandle(child.thread_handle);	15554	windows.CloseHandle(child.thread_handle);
15540	child.thread_handle = undefined;	15555	child.thread_handle = undefined;
15541		15556
15542	if (child.stdin) \|*stdin\| {	15557	if (child.stdin) \|stdin\| {
15543	windows.CloseHandle(stdin.handle);	15558	windows.CloseHandle(stdin.handle);
15544	child.stdin = null;	15559	child.stdin = null;
15545	}	15560	}
15546	if (child.stdout) \|*stdout\| {	15561	if (child.stdout) \|stdout\| {
15547	windows.CloseHandle(stdout.handle);	15562	windows.CloseHandle(stdout.handle);
15548	child.stdout = null;	15563	child.stdout = null;
15549	}	15564	}
15550	if (child.stderr) \|*stderr\| {	15565	if (child.stderr) \|stderr\| {
15551	windows.CloseHandle(stderr.handle);	15566	windows.CloseHandle(stderr.handle);
15552	child.stderr = null;	15567	child.stderr = null;
15553	}	15568	}
...	@@ -15621,7 +15636,7 @@ fn childWaitPosix(child: *process.Child) process.Child.WaitError!process.Child.T	...	@@ -15621,7 +15636,7 @@ fn childWaitPosix(child: *process.Child) process.Child.WaitError!process.Child.T
15621	};	15636	};
15622	}	15637	}
15623		15638
15624	fn statusToTerm(status: u32) process.Child.Term {	15639	pub fn statusToTerm(status: u32) process.Child.Term {
15625	return if (posix.W.IFEXITED(status))	15640	return if (posix.W.IFEXITED(status))
15626	.{ .exited = posix.W.EXITSTATUS(status) }	15641	.{ .exited = posix.W.EXITSTATUS(status) }
15627	else if (posix.W.IFSIGNALED(status))	15642	else if (posix.W.IFSIGNALED(status))
...	@@ -15677,15 +15692,15 @@ fn childKillPosix(child: *process.Child) !void {	...	@@ -15677,15 +15692,15 @@ fn childKillPosix(child: *process.Child) !void {
15677	}	15692	}
15678		15693
15679	fn childCleanupPosix(child: *process.Child) void {	15694	fn childCleanupPosix(child: *process.Child) void {
15680	if (child.stdin) \|*stdin\| {	15695	if (child.stdin) \|stdin\| {
15681	closeFd(stdin.handle);	15696	closeFd(stdin.handle);
15682	child.stdin = null;	15697	child.stdin = null;
15683	}	15698	}
15684	if (child.stdout) \|*stdout\| {	15699	if (child.stdout) \|stdout\| {
15685	closeFd(stdout.handle);	15700	closeFd(stdout.handle);
15686	child.stdout = null;	15701	child.stdout = null;
15687	}	15702	}
15688	if (child.stderr) \|*stderr\| {	15703	if (child.stderr) \|stderr\| {
15689	closeFd(stderr.handle);	15704	closeFd(stderr.handle);
15690	child.stderr = null;	15705	child.stderr = null;
15691	}	15706	}
...	@@ -15818,21 +15833,57 @@ fn processSpawnWindows(userdata: ?*anyopaque, options: process.SpawnOptions) pro	...	@@ -15818,21 +15833,57 @@ fn processSpawnWindows(userdata: ?*anyopaque, options: process.SpawnOptions) pro
15818	.dwFlags = windows.STARTF_USESTDHANDLES,	15833	.dwFlags = windows.STARTF_USESTDHANDLES,
15819	.hStdInput = switch (options.stdin) {	15834	.hStdInput = switch (options.stdin) {
15820	.inherit => peb.ProcessParameters.hStdInput,	15835	.inherit => peb.ProcessParameters.hStdInput,
15821	.file => \|file\| file.handle,	15836	.file => \|file\| try OpenFile(&.{}, .{
		15837	.access_mask = .{
		15838	.STANDARD = .{ .SYNCHRONIZE = true },
		15839	.GENERIC = .{ .READ = true },
		15840	},
		15841	.dir = file.handle,
		15842	.sa = &.{
		15843	.nLength = @sizeOf(windows.SECURITY_ATTRIBUTES),
		15844	.lpSecurityDescriptor = null,
		15845	.bInheritHandle = windows.TRUE,
		15846	},
		15847	.creation = .OPEN,
		15848	}),
15822	.ignore => nul_handle,	15849	.ignore => nul_handle,
15823	.pipe => stdin_pipe[1],	15850	.pipe => stdin_pipe[1],
15824	.close => null,	15851	.close => null,
15825	},	15852	},
15826	.hStdOutput = switch (options.stdout) {	15853	.hStdOutput = switch (options.stdout) {
15827	.inherit => peb.ProcessParameters.hStdOutput,	15854	.inherit => peb.ProcessParameters.hStdOutput,
15828	.file => \|file\| file.handle,	15855	.file => \|file\| try OpenFile(&.{}, .{
		15856	.access_mask = .{
		15857	.STANDARD = .{ .SYNCHRONIZE = true },
		15858	.GENERIC = .{ .WRITE = true },
		15859	},
		15860	.dir = file.handle,
		15861	.sa = &.{
		15862	.nLength = @sizeOf(windows.SECURITY_ATTRIBUTES),
		15863	.lpSecurityDescriptor = null,
		15864	.bInheritHandle = windows.TRUE,
		15865	},
		15866	.creation = .OPEN,
		15867	}),
15829	.ignore => nul_handle,	15868	.ignore => nul_handle,
15830	.pipe => stdout_pipe[1],	15869	.pipe => stdout_pipe[1],
15831	.close => null,	15870	.close => null,
15832	},	15871	},
15833	.hStdError = switch (options.stderr) {	15872	.hStdError = switch (options.stderr) {
15834	.inherit => peb.ProcessParameters.hStdError,	15873	.inherit => peb.ProcessParameters.hStdError,
15835	.file => \|file\| file.handle,	15874	.file => \|file\| try OpenFile(&.{}, .{
		15875	.access_mask = .{
		15876	.STANDARD = .{ .SYNCHRONIZE = true },
		15877	.GENERIC = .{ .WRITE = true },
		15878	},
		15879	.dir = file.handle,
		15880	.sa = &.{
		15881	.nLength = @sizeOf(windows.SECURITY_ATTRIBUTES),
		15882	.lpSecurityDescriptor = null,
		15883	.bInheritHandle = windows.TRUE,
		15884	},
		15885	.creation = .OPEN,
		15886	}),
15836	.ignore => nul_handle,	15887	.ignore => nul_handle,
15837	.pipe => stderr_pipe[1],	15888	.pipe => stderr_pipe[1],
15838	.close => null,	15889	.close => null,
...	@@ -16030,6 +16081,10 @@ fn processSpawnWindows(userdata: ?*anyopaque, options: process.SpawnOptions) pro	...	@@ -16030,6 +16081,10 @@ fn processSpawnWindows(userdata: ?*anyopaque, options: process.SpawnOptions) pro
16030	.id = piProcInfo.hProcess,	16081	.id = piProcInfo.hProcess,
16031	.thread_handle = piProcInfo.hThread,	16082	.thread_handle = piProcInfo.hThread,
16032	.stdin = stdin: switch (options.stdin) {	16083	.stdin = stdin: switch (options.stdin) {
		16084	.file => {
		16085	windows.CloseHandle(siStartInfo.hStdInput.?);
		16086	break :stdin null;
		16087	},
16033	.pipe => {	16088	.pipe => {
16034	windows.CloseHandle(stdin_pipe[1]);	16089	windows.CloseHandle(stdin_pipe[1]);
16035	break :stdin .{ .handle = stdin_pipe[0], .flags = .{ .nonblocking = false } };	16090	break :stdin .{ .handle = stdin_pipe[0], .flags = .{ .nonblocking = false } };
...	@@ -16037,6 +16092,10 @@ fn processSpawnWindows(userdata: ?*anyopaque, options: process.SpawnOptions) pro	...	@@ -16037,6 +16092,10 @@ fn processSpawnWindows(userdata: ?*anyopaque, options: process.SpawnOptions) pro
16037	else => null,	16092	else => null,
16038	},	16093	},
16039	.stdout = stdout: switch (options.stdout) {	16094	.stdout = stdout: switch (options.stdout) {
		16095	.file => {
		16096	windows.CloseHandle(siStartInfo.hStdOutput.?);
		16097	break :stdout null;
		16098	},
16040	.pipe => {	16099	.pipe => {
16041	windows.CloseHandle(stdout_pipe[1]);	16100	windows.CloseHandle(stdout_pipe[1]);
16042	break :stdout .{ .handle = stdout_pipe[0], .flags = .{ .nonblocking = true } };	16101	break :stdout .{ .handle = stdout_pipe[0], .flags = .{ .nonblocking = true } };
...	@@ -16044,6 +16103,10 @@ fn processSpawnWindows(userdata: ?*anyopaque, options: process.SpawnOptions) pro	...	@@ -16044,6 +16103,10 @@ fn processSpawnWindows(userdata: ?*anyopaque, options: process.SpawnOptions) pro
16044	else => null,	16103	else => null,
16045	},	16104	},
16046	.stderr = stderr: switch (options.stderr) {	16105	.stderr = stderr: switch (options.stderr) {
		16106	.file => {
		16107	windows.CloseHandle(siStartInfo.hStdError.?);
		16108	break :stderr null;
		16109	},
16047	.pipe => {	16110	.pipe => {
16048	windows.CloseHandle(stderr_pipe[1]);	16111	windows.CloseHandle(stderr_pipe[1]);
16049	break :stderr .{ .handle = stderr_pipe[0], .flags = .{ .nonblocking = true } };	16112	break :stderr .{ .handle = stderr_pipe[0], .flags = .{ .nonblocking = true } };
...	@@ -16054,6 +16117,8 @@ fn processSpawnWindows(userdata: ?*anyopaque, options: process.SpawnOptions) pro	...	@@ -16054,6 +16117,8 @@ fn processSpawnWindows(userdata: ?*anyopaque, options: process.SpawnOptions) pro
16054	};	16117	};
16055	}	16118	}
16056		16119
		16120	fn inheritFile() windows.HANDLE {}
		16121
16057	fn getCngDevice(t: *Threaded) Io.RandomSecureError!windows.HANDLE {	16122	fn getCngDevice(t: *Threaded) Io.RandomSecureError!windows.HANDLE {
16058	{	16123	{
16059	mutexLock(&t.mutex);	16124	mutexLock(&t.mutex);
...	@@ -17758,11 +17823,7 @@ const parking_futex = struct {	...	@@ -17758,11 +17823,7 @@ const parking_futex = struct {
17758	waiter.node.next = waking_head;	17823	waiter.node.next = waking_head;
17759	waking_head = &waiter.node;	17824	waking_head = &waiter.node;
17760	num_removed += 1;	17825	num_removed += 1;
17761	// Signal to `waiter` that they're about to be unparked, in case we're racing with their
17762	// timeout. See corresponding logic in `wake`.
17763	waiter.address = 0;
17764	}	17826	}
17765
17766	_ = bucket.num_waiters.fetchSub(num_removed, .monotonic);	17827	_ = bucket.num_waiters.fetchSub(num_removed, .monotonic);
17767	}	17828	}
17768		17829
...	@@ -19056,7 +19117,7 @@ const OpenError = error{	...	@@ -19056,7 +19117,7 @@ const OpenError = error{
19056	const OpenFileOptions = struct {	19117	const OpenFileOptions = struct {
19057	access_mask: windows.ACCESS_MASK,	19118	access_mask: windows.ACCESS_MASK,
19058	dir: ?windows.HANDLE = null,	19119	dir: ?windows.HANDLE = null,
19059	sa: ?*windows.SECURITY_ATTRIBUTES = null,	19120	sa: ?*const windows.SECURITY_ATTRIBUTES = null,
19060	share_access: windows.FILE.SHARE = .VALID_FLAGS,	19121	share_access: windows.FILE.SHARE = .VALID_FLAGS,
19061	creation: windows.FILE.CREATE_DISPOSITION,	19122	creation: windows.FILE.CREATE_DISPOSITION,
19062	filter: Filter = .non_directory_only,	19123	filter: Filter = .non_directory_only,
...	@@ -19076,10 +19137,10 @@ const OpenFileOptions = struct {	...	@@ -19076,10 +19137,10 @@ const OpenFileOptions = struct {
19076		19137
19077	/// TODO: inline this logic everywhere and delete this function	19138	/// TODO: inline this logic everywhere and delete this function
19078	fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!windows.HANDLE {	19139	fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!windows.HANDLE {
19079	if (std.mem.eql(u16, sub_path_w, &[_]u16{'.'}) and options.filter == .non_directory_only) {	19140	if (std.mem.eql(u16, sub_path_w, &.{'.'}) and options.filter == .non_directory_only) {
19080	return error.IsDir;	19141	return error.IsDir;
19081	}	19142	}
19082	if (std.mem.eql(u16, sub_path_w, &[_]u16{ '.', '.' }) and options.filter == .non_directory_only) {	19143	if (std.mem.eql(u16, sub_path_w, &.{ '.', '.' }) and options.filter == .non_directory_only) {
19083	return error.IsDir;	19144	return error.IsDir;
19084	}	19145	}
19085		19146

lib/std/Io/Uring.zig created+6173

...	@@ -0,0 +1,6173 @@
1	const addressFromPosix = Io.Threaded.addressFromPosix;
2	const addressToPosix = Io.Threaded.addressToPosix;
3	const Alignment = std.mem.Alignment;
4	const Allocator = std.mem.Allocator;
5	const Argv0 = Io.Threaded.Argv0;
6	const assert = std.debug.assert;
7	const builtin = @import("builtin");
8	const ChdirError = Io.Threaded.ChdirError;
9	const clockToPosix = Io.Threaded.clockToPosix;
10	const Csprng = Io.Threaded.Csprng;
11	const default_PATH = Io.Threaded.default_PATH;
12	const Dir = Io.Dir;
13	const Environ = Io.Threaded.Environ;
14	const errnoBug = Io.Threaded.errnoBug;
15	const Evented = @This();
16	const fallbackSeed = Io.Threaded.fallbackSeed;
17	const fd_t = linux.fd_t;
18	const File = Io.File;
19	const Io = std.Io;
20	const IoUring = linux.IoUring;
21	const iovec = std.posix.iovec;
22	const iovec_const = std.posix.iovec_const;
23	const linux = std.os.linux;
24	const linux_statx_request = Io.Threaded.linux_statx_request;
25	const LOCK = std.posix.LOCK;
26	const log = std.log.scoped(.@"io-uring");
27	const max_iovecs_len = Io.Threaded.max_iovecs_len;
28	const nanosecondsFromPosix = Io.Threaded.nanosecondsFromPosix;
29	const net = Io.net;
30	const PATH_MAX = linux.PATH_MAX;
31	const pathToPosix = Io.Threaded.pathToPosix;
32	const pid_t = linux.pid_t;
33	const PosixAddress = Io.Threaded.PosixAddress;
34	const posixAddressFamily = Io.Threaded.posixAddressFamily;
35	const posixProtocol = Io.Threaded.posixProtocol;
36	const posixSocketMode = Io.Threaded.posixSocketMode;
37	const process = std.process;
38	const recoverableOsBugDetected = Io.Threaded.recoverableOsBugDetected;
39	const setTimestampToPosix = Io.Threaded.setTimestampToPosix;
40	const splat_buffer_size = Io.Threaded.splat_buffer_size;
41	const statFromLinux = Io.Threaded.statFromLinux;
42	const statxKind = Io.Threaded.statxKind;
43	const std = @import("../std.zig");
44	const timestampFromPosix = Io.Threaded.timestampFromPosix;
45	const unexpectedErrno = std.posix.unexpectedErrno;
46	const winsize = std.posix.winsize;
47
48	const tracy = if (@hasDecl(@import("root"), "tracy")) @import("root").tracy else struct {
49	const enable = false;
50	inline fn fiberEnter(fiber: [*:0]const u8) void {
51	_ = fiber;
52	}
53	inline fn fiberLeave() void {}
54	};
55
56	/// Empirically saw >128KB being used by the self-hosted backend to panic.
57	/// Empirically saw glibc complain about 256KB.
58	const idle_stack_size = 512 * 1024;
59
60	const max_idle_search = 1;
61	const max_steal_ready_search = 2;
62	const max_steal_free_search = 4;
63
64	backing_allocator_needs_mutex: bool,
65	backing_allocator_mutex: Io.Mutex,
66	/// Does not need to be thread-safe if not used elsewhere.
67	backing_allocator: Allocator,
68	main_fiber_buffer: [
69	std.mem.alignForward(usize, @sizeOf(Fiber), @alignOf(Completion)) + @sizeOf(Completion)
70	]u8 align(@max(@alignOf(Fiber), @alignOf(Completion))),
71	log2_ring_entries: u4,
72	threads: Thread.List,
73	sync_limit: ?Io.Semaphore,
74
75	stderr_writer_initialized: bool = false,
76	stderr_mutex: Io.Mutex,
77	stderr_writer: File.Writer = .{
78	.io = undefined,
79	.interface = Io.File.Writer.initInterface(&.{}),
80	.file = .stderr(),
81	.mode = .streaming,
82	},
83	stderr_mode: Io.Terminal.Mode = .no_color,
84
85	environ_mutex: Io.Mutex,
86	environ_initialized: bool,
87	environ: Environ,
88
89	null_fd: CachedFd,
90	random_fd: CachedFd,
91
92	csprng_mutex: Io.Mutex,
93	csprng: Csprng,
94
95	const Thread = struct {
96	required_align: void align(4),
97	thread: std.Thread,
98	idle_context: Io.fiber.Context,
99	current_context: *Io.fiber.Context,
100	ready_queue: ?*Fiber,
101	free_queue: ?*Fiber,
102	io_uring: IoUring,
103	idle_search_index: u32,
104	steal_ready_search_index: u32,
105	steal_free_search_index: u32,
106	name_arena: if (tracy.enable) std.heap.ArenaAllocator.State else struct {},
107	csprng: Csprng,
108
109	threadlocal var self: ?*Thread = null;
110
111	noinline fn current() *Thread {
112	return self.?;
113	}
114
115	fn deinit(thread: *Thread, gpa: Allocator) void {
116	var next_fiber = thread.free_queue;
117	while (next_fiber) \|free_fiber\| {
118	next_fiber = free_fiber.status.free_next;
119	gpa.free(free_fiber.allocatedSlice());
120	}
121	thread.io_uring.deinit();
122	}
123
124	fn currentFiber(thread: Thread) Fiber {
125	assert(thread.current_context != &thread.idle_context);
126	return @fieldParentPtr("context", thread.current_context);
127	}
128
129	fn enqueue(thread: Thread) linux.io_uring_sqe {
130	while (true) return thread.io_uring.get_sqe() catch {
131	thread.submit();
132	continue;
133	};
134	}
135
136	fn submit(thread: *Thread) void {
137	_ = thread.io_uring.submit() catch \|err\| switch (err) {
138	error.SignalInterrupt => {},
139	else => \|e\| @panic(@errorName(e)),
140	};
141	}
142
143	const List = struct {
144	allocated: []Thread,
145	reserved: u32,
146	active: u32,
147	};
148	};
149
150	const Fiber = struct {
151	required_align: void align(4),
152	context: Io.fiber.Context,
153	await_count: i32,
154	link: union {
155	awaiter: ?*Fiber,
156	group: struct { prev: ?Fiber, next: ?Fiber },
157	},
158	status: union(enum) {
159	queue_next: ?*Fiber,
160	awaiting_group: Group,
161	free_next: ?*Fiber,
162	},
163	cancel_status: CancelStatus,
164	cancel_protection: CancelProtection,
165	name: if (tracy.enable) [*:0]const u8 else void,
166
167	var next_name: u64 = 0;
168
169	const CancelStatus = packed struct(u32) {
170	requested: bool,
171	awaiting: Awaiting,
172
173	const unrequested: CancelStatus = .{ .requested = false, .awaiting = .nothing };
174
175	const Awaiting = enum(u31) {
176	nothing = std.math.maxInt(u31),
177	group = std.math.maxInt(u31) - 1,
178	select = std.math.maxInt(u31) - 2,
179	/// An io_uring fd.
180	_,
181
182	fn subWrap(lhs: Awaiting, rhs: Awaiting) Awaiting {
183	return @enumFromInt(@intFromEnum(lhs) -% @intFromEnum(rhs));
184	}
185
186	fn fromIoUringFd(fd: fd_t) Awaiting {
187	const awaiting: Awaiting = @enumFromInt(fd);
188	switch (awaiting) {
189	.nothing, .group, .select => unreachable,
190	_ => return awaiting,
191	}
192	}
193
194	fn toIoUringFd(awaiting: Awaiting) fd_t {
195	switch (awaiting) {
196	.nothing, .group, .select => unreachable,
197	_ => return @intFromEnum(awaiting),
198	}
199	}
200	};
201
202	fn changeAwaiting(
203	cancel_status: *CancelStatus,
204	old_awaiting: Awaiting,
205	new_awaiting: Awaiting,
206	) bool {
207	const old_cancel_status = @atomicRmw(CancelStatus, cancel_status, .Add, .{
208	.requested = false,
209	.awaiting = new_awaiting.subWrap(old_awaiting),
210	}, .monotonic);
211	assert(old_cancel_status.awaiting == old_awaiting);
212	return old_cancel_status.requested;
213	}
214	};
215
216	const CancelProtection = packed struct {
217	user: Io.CancelProtection,
218	acknowledged: bool,
219
220	const unblocked: CancelProtection = .{ .user = .unblocked, .acknowledged = false };
221
222	fn check(cancel_protection: CancelProtection) Io.CancelProtection {
223	return @enumFromInt(@intFromBool(cancel_protection != unblocked));
224	}
225
226	fn acknowledge(cancel_protection: *CancelProtection) void {
227	assert(!cancel_protection.acknowledged);
228	cancel_protection.acknowledged = true;
229	}
230
231	fn recancel(cancel_protection: *CancelProtection) void {
232	assert(cancel_protection.acknowledged);
233	cancel_protection.acknowledged = false;
234	}
235
236	test check {
237	try std.testing.expectEqual(Io.CancelProtection.unblocked, check(.unblocked));
238	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
239	.user = .unblocked,
240	.acknowledged = true,
241	}));
242	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
243	.user = .blocked,
244	.acknowledged = false,
245	}));
246	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
247	.user = .blocked,
248	.acknowledged = true,
249	}));
250	}
251	};
252
253	const finished: ?*Fiber = @ptrFromInt(@alignOf(Fiber));
254
255	const max_result_align: Alignment = .@"16";
256	const max_result_size = max_result_align.forward(512);
257	/// This includes any stack realignments that need to happen, and also the
258	/// initial frame return address slot and argument frame, depending on target.
259	const min_stack_size = 60 * 1024 * 1024;
260	const max_context_align: Alignment = .@"16";
261	const max_context_size = max_context_align.forward(1024);
262	const max_closure_size: usize = @sizeOf(AsyncClosure);
263	const max_closure_align: Alignment = .of(AsyncClosure);
264	const allocation_size = std.mem.alignForward(
265	usize,
266	max_closure_align.max(max_context_align).forward(
267	max_result_align.forward(@sizeOf(Fiber)) + max_result_size + min_stack_size,
268	) + max_closure_size + max_context_size,
269	std.heap.page_size_max,
270	);
271	comptime {
272	assert(max_result_align.compare(.gte, .of(Completion)));
273	assert(max_result_size >= @sizeOf(Completion));
274	}
275
276	fn create(ev: Evented) error{OutOfMemory}!Fiber {
277	const thread: *Thread = .current();
278	if (@atomicRmw(?*Fiber, &thread.free_queue, .Xchg, finished, .acquire)) \|free_fiber\| {
279	assert(free_fiber != finished);
280	@atomicStore(?*Fiber, &thread.free_queue, free_fiber.status.free_next, .release);
281	return free_fiber;
282	}
283	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
284	for (0..@min(max_steal_free_search, active_threads)) \|_\| {
285	defer thread.steal_free_search_index += 1;
286	if (thread.steal_free_search_index == active_threads) thread.steal_free_search_index = 0;
287	const steal_free_search_thread =
288	&ev.threads.allocated[0..active_threads][thread.steal_free_search_index];
289	if (steal_free_search_thread == thread) continue;
290	const free_fiber =
291	@atomicLoad(?*Fiber, &steal_free_search_thread.free_queue, .monotonic) orelse continue;
292	if (free_fiber == finished) continue;
293	if (@cmpxchgWeak(
294	?*Fiber,
295	&steal_free_search_thread.free_queue,
296	free_fiber,
297	null,
298	.acquire,
299	.monotonic,
300	)) \|_\| continue;
301	@atomicStore(?*Fiber, &thread.free_queue, free_fiber.status.free_next, .release);
302	return free_fiber;
303	}
304	@atomicStore(?*Fiber, &thread.free_queue, null, .monotonic);
305	return @ptrCast(try ev.allocator().alignedAlloc(u8, .of(Fiber), allocation_size));
306	}
307
308	fn destroy(fiber: *Fiber) void {
309	const thread: *Thread = .current();
310	assert(fiber.status.queue_next == null);
311	fiber.status = .{ .free_next = @atomicLoad(?*Fiber, &thread.free_queue, .acquire) };
312	while (true) fiber.status.free_next = @cmpxchgWeak(
313	?*Fiber,
314	&thread.free_queue,
315	fiber.status.free_next,
316	fiber,
317	.acq_rel,
318	.acquire,
319	) orelse break;
320	}
321
322	fn allocatedSlice(f: *Fiber) []align(@alignOf(Fiber)) u8 {
323	return @as([*]align(@alignOf(Fiber)) u8, @ptrCast(f))[0..allocation_size];
324	}
325
326	fn allocatedEnd(f: Fiber) []u8 {
327	const allocated_slice = f.allocatedSlice();
328	return allocated_slice[allocated_slice.len..].ptr;
329	}
330
331	fn resultPointer(f: Fiber, comptime Result: type) Result {
332	return @ptrCast(@alignCast(f.resultBytes(.of(Result))));
333	}
334
335	fn resultBytes(f: Fiber, alignment: Alignment) []u8 {
336	return @ptrFromInt(alignment.forward(@intFromPtr(f) + @sizeOf(Fiber)));
337	}
338
339	const Queue = struct { head: Fiber, tail: Fiber };
340
341	/// Like a `*Fiber`, but 2 bits smaller than a pointer (because the LSBs are always 0 due to
342	/// alignment) so that those two bits can be used in a `packed struct`.
343	const PackedPtr = enum(@Int(.unsigned, @bitSizeOf(usize) - 2)) {
344	null = 0,
345	all_ones = std.math.maxInt(@Int(.unsigned, @bitSizeOf(usize) - 2)),
346	_,
347
348	const Split = packed struct(usize) { low: u2, high: PackedPtr };
349	fn pack(ptr: ?*Fiber) PackedPtr {
350	const split: Split = @bitCast(@intFromPtr(ptr));
351	assert(split.low == 0);
352	return split.high;
353	}
354	fn unpack(ptr: PackedPtr) ?*Fiber {
355	const split: Split = .{ .low = 0, .high = ptr };
356	return @ptrFromInt(@as(usize, @bitCast(split)));
357	}
358	};
359
360	fn requestCancel(fiber: Fiber, ev: Evented) void {
361	const cancel_status = @atomicRmw(
362	Fiber.CancelStatus,
363	&fiber.cancel_status,
364	.Or,
365	.{ .requested = true, .awaiting = @enumFromInt(0) },
366	.acquire,
367	);
368	assert(!cancel_status.requested);
369	switch (cancel_status.awaiting) {
370	.nothing => {},
371	.group => {
372	// The awaiter received a cancelation request while awaiting a group,
373	// so propagate the cancelation to the group.
374	if (fiber.status.awaiting_group.cancel(ev, null)) {
375	fiber.status = .{ .queue_next = null };
376	_ = ev.schedule(.current(), .{ .head = fiber, .tail = fiber });
377	}
378	},
379	.select => if (@atomicRmw(i32, &fiber.await_count, .Add, 1, .monotonic) == -1) {
380	_ = ev.schedule(.current(), .{ .head = fiber, .tail = fiber });
381	},
382	_ => \|awaiting\| {
383	const awaiting_io_uring_fd = awaiting.toIoUringFd();
384	const thread: *Thread = .current();
385	thread.enqueue().* = if (thread.io_uring.fd == awaiting_io_uring_fd) .{
386	.opcode = .ASYNC_CANCEL,
387	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
388	.ioprio = 0,
389	.fd = 0,
390	.off = 0,
391	.addr = @intFromPtr(fiber),
392	.len = 0,
393	.rw_flags = 0,
394	.user_data = @intFromEnum(Completion.Userdata.wakeup),
395	.buf_index = 0,
396	.personality = 0,
397	.splice_fd_in = 0,
398	.addr3 = 0,
399	.resv = 0,
400	} else .{
401	.opcode = .MSG_RING,
402	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
403	.ioprio = 0,
404	.fd = awaiting_io_uring_fd,
405	.off = @intFromPtr(fiber) \| 0b01,
406	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
407	.len = 0,
408	.rw_flags = 0,
409	.user_data = @intFromEnum(Completion.Userdata.cleanup),
410	.buf_index = 0,
411	.personality = 0,
412	.splice_fd_in = 0,
413	.addr3 = 0,
414	.resv = 0,
415	};
416	},
417	}
418	}
419	};
420
421	const CancelRegion = struct {
422	fiber: *Fiber,
423	status: Fiber.CancelStatus,
424	fn init() CancelRegion {
425	const fiber = Thread.current().currentFiber();
426	return .{
427	.fiber = fiber,
428	.status = .{
429	.requested = fiber.cancel_protection.check() == .unblocked,
430	.awaiting = .nothing,
431	},
432	};
433	}
434	fn initBlocked() CancelRegion {
435	return .{
436	.fiber = Thread.current().currentFiber(),
437	.status = .{ .requested = false, .awaiting = .nothing },
438	};
439	}
440	fn deinit(cancel_region: *CancelRegion) void {
441	if (cancel_region.status.requested) {
442	@branchHint(.likely);
443	_ = cancel_region.fiber.cancel_status.changeAwaiting(
444	cancel_region.status.awaiting,
445	.nothing,
446	);
447	}
448	cancel_region.* = undefined;
449	}
450	fn await(cancel_region: *CancelRegion, awaiting: Fiber.CancelStatus.Awaiting) Io.Cancelable!void {
451	if (!cancel_region.status.requested) {
452	@branchHint(.unlikely);
453	return;
454	}
455	const status: Fiber.CancelStatus = .{ .requested = true, .awaiting = awaiting };
456	if (cancel_region.fiber.cancel_status.changeAwaiting(
457	cancel_region.status.awaiting,
458	status.awaiting,
459	)) {
460	@branchHint(.unlikely);
461	cancel_region.fiber.cancel_protection.acknowledge();
462	cancel_region.status = .unrequested;
463	return error.Canceled;
464	}
465	cancel_region.status = status;
466	}
467	fn awaitIoUring(cancel_region: CancelRegion) Io.Cancelable!Thread {
468	const thread: *Thread = .current();
469	try cancel_region.await(.fromIoUringFd(thread.io_uring.fd));
470	return thread;
471	}
472	fn completion(cancel_region: *const CancelRegion) Completion {
473	return cancel_region.fiber.resultPointer(Completion).*;
474	}
475	fn errno(cancel_region: *const CancelRegion) linux.E {
476	return cancel_region.completion().errno();
477	}
478
479	const Sync = struct {
480	cancel_region: CancelRegion,
481	fn init(ev: *Evented) Io.Cancelable!Sync {
482	if (ev.sync_limit) \|*sync_limit\| try sync_limit.wait(ev.io());
483	return .{ .cancel_region = .init() };
484	}
485	fn initBlocked(ev: *Evented) Sync {
486	if (ev.sync_limit) \|*sync_limit\| sync_limit.waitUncancelable(ev.io());
487	return .{ .cancel_region = .initBlocked() };
488	}
489	fn deinit(sync: Sync, ev: Evented) void {
490	sync.cancel_region.deinit();
491	if (ev.sync_limit) \|*sync_limit\| sync_limit.post(ev.io());
492	}
493
494	const Maybe = union(enum) {
495	cancel_region: CancelRegion,
496	sync: Sync,
497
498	fn deinit(maybe: Maybe, ev: Evented) void {
499	switch (maybe.*) {
500	.cancel_region => \|*cancel_region\| cancel_region.deinit(),
501	.sync => \|*sync\| sync.deinit(ev),
502	}
503	}
504
505	fn enterSync(maybe: Maybe, ev: Evented) Io.Cancelable!*Sync {
506	switch (maybe.*) {
507	.cancel_region => \|cancel_region\| {
508	if (ev.sync_limit) \|*sync_limit\| try sync_limit.wait(ev.io());
509	maybe.* = .{ .sync = .{ .cancel_region = cancel_region } };
510	},
511	.sync => {},
512	}
513	return &maybe.sync;
514	}
515
516	fn leaveSync(maybe: Maybe, ev: Evented) void {
517	switch (maybe.*) {
518	.cancel_region => {},
519	.sync => \|sync\| {
520	if (ev.sync_limit) \|*sync_limit\| sync_limit.post(ev.io());
521	maybe.* = .{ .cancel_region = sync.cancel_region };
522	},
523	}
524	}
525
526	fn cancelRegion(maybe: Maybe) CancelRegion {
527	return switch (maybe.*) {
528	.cancel_region => \|*cancel_region\| cancel_region,
529	.sync => \|*sync\| &sync.cancel_region,
530	};
531	}
532	};
533	};
534	};
535
536	const CachedFd = struct {
537	once: Once,
538
539	const Once = enum(fd_t) {
540	uninitialized = -1,
541	initializing = -2,
542	/// fd
543	_,
544
545	fn fromFd(fd: fd_t) Once {
546	return @enumFromInt(@as(u31, @intCast(fd)));
547	}
548
549	fn toFd(once: Once) fd_t {
550	return @as(u31, @intCast(@intFromEnum(once)));
551	}
552	};
553
554	const init: CachedFd = .{ .once = .uninitialized };
555
556	fn close(cached_fd: *CachedFd) void {
557	switch (cached_fd.once) {
558	.uninitialized => {},
559	.initializing => unreachable,
560	_ => \|fd\| {
561	assert(@intFromEnum(fd) >= 0);
562	_ = linux.close(@intFromEnum(fd));
563	cached_fd.* = .init;
564	},
565	}
566	}
567
568	fn open(
569	cached_fd: *CachedFd,
570	ev: *Evented,
571	cancel_region: *CancelRegion,
572	path: [*:0]const u8,
573	flags: linux.O,
574	) File.OpenError!fd_t {
575	var once = @atomicLoad(Once, &cached_fd.once, .monotonic);
576	while (true) {
577	switch (once) {
578	.uninitialized => {},
579	.initializing => try futexWait(
580	ev,
581	@ptrCast(&cached_fd.once),
582	@bitCast(@intFromEnum(once)),
583	.none,
584	),
585	_ => \|fd\| {
586	@branchHint(.likely);
587	return fd.toFd();
588	},
589	}
590	once = @cmpxchgWeak(
591	Once,
592	&cached_fd.once,
593	.uninitialized,
594	.initializing,
595	.monotonic,
596	.monotonic,
597	) orelse {
598	errdefer {
599	@atomicStore(Once, &cached_fd.once, .uninitialized, .monotonic);
600	futexWake(ev, @ptrCast(&cached_fd.once), 1);
601	}
602	const fd = try ev.openat(cancel_region, linux.AT.FDCWD, path, flags, 0);
603	@atomicStore(Once, &cached_fd.once, .fromFd(fd), .monotonic);
604	futexWake(ev, @ptrCast(&cached_fd.once), std.math.maxInt(u32));
605	return fd;
606	};
607	}
608	}
609	};
610
611	pub fn allocator(ev: *Evented) std.mem.Allocator {
612	return if (ev.backing_allocator_needs_mutex) .{
613	.ptr = ev,
614	.vtable = &.{
615	.alloc = alloc,
616	.resize = resize,
617	.remap = remap,
618	.free = free,
619	},
620	} else ev.backing_allocator;
621	}
622
623	fn alloc(userdata: anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[]u8 {
624	const ev: *Evented = @ptrCast(@alignCast(userdata));
625	const ev_io = ev.io();
626	ev.backing_allocator_mutex.lockUncancelable(ev_io);
627	defer ev.backing_allocator_mutex.unlock(ev_io);
628	return ev.backing_allocator.rawAlloc(len, alignment, ret_addr);
629	}
630
631	fn resize(
632	userdata: *anyopaque,
633	memory: []u8,
634	alignment: std.mem.Alignment,
635	new_len: usize,
636	ret_addr: usize,
637	) bool {
638	const ev: *Evented = @ptrCast(@alignCast(userdata));
639	const ev_io = ev.io();
640	ev.backing_allocator_mutex.lockUncancelable(ev_io);
641	defer ev.backing_allocator_mutex.unlock(ev_io);
642	return ev.backing_allocator.rawResize(memory, alignment, new_len, ret_addr);
643	}
644
645	fn remap(
646	userdata: *anyopaque,
647	memory: []u8,
648	alignment: Alignment,
649	new_len: usize,
650	ret_addr: usize,
651	) ?[*]u8 {
652	const ev: *Evented = @ptrCast(@alignCast(userdata));
653	const ev_io = ev.io();
654	ev.backing_allocator_mutex.lockUncancelable(ev_io);
655	defer ev.backing_allocator_mutex.unlock(ev_io);
656	return ev.backing_allocator.rawRemap(memory, alignment, new_len, ret_addr);
657	}
658
659	fn free(userdata: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void {
660	const ev: *Evented = @ptrCast(@alignCast(userdata));
661	const ev_io = ev.io();
662	ev.backing_allocator_mutex.lockUncancelable(ev_io);
663	defer ev.backing_allocator_mutex.unlock(ev_io);
664	return ev.backing_allocator.rawFree(memory, alignment, ret_addr);
665	}
666
667	pub fn io(ev: *Evented) Io {
668	return .{
669	.userdata = ev,
670	.vtable = &.{
671	.crashHandler = crashHandler,
672
673	.async = async,
674	.concurrent = concurrent,
675	.await = await,
676	.cancel = cancel,
677
678	.groupAsync = groupAsync,
679	.groupConcurrent = groupConcurrent,
680	.groupAwait = groupAwait,
681	.groupCancel = groupCancel,
682
683	.recancel = recancel,
684	.swapCancelProtection = swapCancelProtection,
685	.checkCancel = checkCancel,
686
687	.select = select,
688
689	.futexWait = futexWait,
690	.futexWaitUncancelable = futexWaitUncancelable,
691	.futexWake = futexWake,
692
693	.operate = operate,
694	.batchAwaitAsync = batchAwaitAsync,
695	.batchAwaitConcurrent = batchAwaitConcurrent,
696	.batchCancel = batchCancel,
697
698	.dirCreateDir = dirCreateDir,
699	.dirCreateDirPath = dirCreateDirPath,
700	.dirCreateDirPathOpen = dirCreateDirPathOpen,
701	.dirOpenDir = dirOpenDir,
702	.dirStat = dirStat,
703	.dirStatFile = dirStatFile,
704	.dirAccess = dirAccess,
705	.dirCreateFile = dirCreateFile,
706	.dirCreateFileAtomic = dirCreateFileAtomic,
707	.dirOpenFile = dirOpenFile,
708	.dirClose = dirClose,
709	.dirRead = dirRead,
710	.dirRealPath = dirRealPath,
711	.dirRealPathFile = dirRealPathFile,
712	.dirDeleteFile = dirDeleteFile,
713	.dirDeleteDir = dirDeleteDir,
714	.dirRename = dirRename,
715	.dirRenamePreserve = dirRenamePreserve,
716	.dirSymLink = dirSymLink,
717	.dirReadLink = dirReadLink,
718	.dirSetOwner = dirSetOwner,
719	.dirSetFileOwner = dirSetFileOwner,
720	.dirSetPermissions = dirSetPermissions,
721	.dirSetFilePermissions = dirSetFilePermissions,
722	.dirSetTimestamps = dirSetTimestamps,
723	.dirHardLink = dirHardLink,
724
725	.fileStat = fileStat,
726	.fileLength = fileLength,
727	.fileClose = fileClose,
728	.fileWritePositional = fileWritePositional,
729	.fileWriteFileStreaming = fileWriteFileStreaming,
730	.fileWriteFilePositional = fileWriteFilePositional,
731	.fileReadPositional = fileReadPositional,
732	.fileSeekBy = fileSeekBy,
733	.fileSeekTo = fileSeekTo,
734	.fileSync = fileSync,
735	.fileIsTty = fileIsTty,
736	.fileEnableAnsiEscapeCodes = fileEnableAnsiEscapeCodes,
737	.fileSupportsAnsiEscapeCodes = fileIsTty,
738	.fileSetLength = fileSetLength,
739	.fileSetOwner = fileSetOwner,
740	.fileSetPermissions = fileSetPermissions,
741	.fileSetTimestamps = fileSetTimestamps,
742	.fileLock = fileLock,
743	.fileTryLock = fileTryLock,
744	.fileUnlock = fileUnlock,
745	.fileDowngradeLock = fileDowngradeLock,
746	.fileRealPath = fileRealPath,
747	.fileHardLink = fileHardLink,
748
749	.fileMemoryMapCreate = fileMemoryMapCreate,
750	.fileMemoryMapDestroy = fileMemoryMapDestroy,
751	.fileMemoryMapSetLength = fileMemoryMapSetLength,
752	.fileMemoryMapRead = fileMemoryMapRead,
753	.fileMemoryMapWrite = fileMemoryMapWrite,
754
755	.processExecutableOpen = processExecutableOpen,
756	.processExecutablePath = processExecutablePath,
757	.lockStderr = lockStderr,
758	.tryLockStderr = tryLockStderr,
759	.unlockStderr = unlockStderr,
760	.processCurrentPath = processCurrentPath,
761	.processSetCurrentDir = processSetCurrentDir,
762	.processReplace = processReplace,
763	.processReplacePath = processReplacePath,
764	.processSpawn = processSpawn,
765	.processSpawnPath = processSpawnPath,
766	.childWait = childWait,
767	.childKill = childKill,
768
769	.progressParentFile = progressParentFile,
770
771	.now = now,
772	.clockResolution = clockResolution,
773	.sleep = sleep,
774
775	.random = random,
776	.randomSecure = randomSecure,
777
778	.netListenIp = netListenIpUnavailable,
779	.netAccept = netAcceptUnavailable,
780	.netBindIp = netBindIp,
781	.netConnectIp = netConnectIpUnavailable,
782	.netListenUnix = netListenUnixUnavailable,
783	.netConnectUnix = netConnectUnixUnavailable,
784	.netSocketCreatePair = netSocketCreatePairUnavailable,
785	.netSend = netSendUnavailable,
786	.netReceive = netReceive,
787	.netRead = netReadUnavailable,
788	.netWrite = netWriteUnavailable,
789	.netWriteFile = netWriteFileUnavailable,
790	.netClose = netClose,
791	.netShutdown = netShutdown,
792	.netInterfaceNameResolve = netInterfaceNameResolveUnavailable,
793	.netInterfaceName = netInterfaceNameUnavailable,
794	.netLookup = netLookupUnavailable,
795	},
796	};
797	}
798
799	pub const InitOptions = struct {
800	backing_allocator_needs_mutex: bool = true,
801
802	/// Maximum thread pool size (excluding the main thread).
803	/// Defaults to one less than the number of logical CPU cores.
804	thread_limit: ?usize = null,
805	/// Maximum number of threads that may perform synchronous syscalls.
806	sync_limit: Io.Limit = .unlimited,
807
808	log2_ring_entries: u4 = 3,
809
810	/// Affects the following operations:
811	/// * `processExecutablePath` on OpenBSD and Haiku.
812	argv0: Argv0 = .empty,
813	/// Affects the following operations:
814	/// * `fileIsTty`
815	/// * `processSpawn`, `processSpawnPath`, `processReplace`, `processReplacePath`
816	environ: process.Environ = .empty,
817	};
818
819	pub fn init(ev: *Evented, backing_allocator: Allocator, options: InitOptions) !void {
820	const threads_size = @sizeOf(Thread) * if (options.thread_limit) \|thread_limit\|
821	1 + thread_limit
822	else
823	@max(std.Thread.getCpuCount() catch 1, 1);
824	const idle_stack_end_offset =
825	std.mem.alignForward(usize, threads_size + idle_stack_size, std.heap.pageSize());
826	const allocated_slice = try backing_allocator.alignedAlloc(u8, .of(Thread), idle_stack_end_offset);
827	errdefer backing_allocator.free(allocated_slice);
828	ev.* = .{
829	.backing_allocator_needs_mutex = options.backing_allocator_needs_mutex,
830	.backing_allocator_mutex = .init,
831	.backing_allocator = backing_allocator,
832	.main_fiber_buffer = undefined,
833	.log2_ring_entries = options.log2_ring_entries,
834	.threads = .{
835	.allocated = @ptrCast(allocated_slice[0..threads_size]),
836	.reserved = 1,
837	.active = 1,
838	},
839	.sync_limit = if (options.sync_limit.toInt()) \|sync_limit\| .{ .permits = sync_limit } else null,
840
841	.stderr_writer_initialized = false,
842	.stderr_mutex = .init,
843	.stderr_writer = .{
844	.io = ev.io(),
845	.interface = Io.File.Writer.initInterface(&.{}),
846	.file = .stderr(),
847	.mode = .streaming,
848	},
849	.stderr_mode = .no_color,
850
851	.environ_mutex = .init,
852	.environ_initialized = options.environ.block.isEmpty(),
853	.environ = .{ .process_environ = options.environ },
854
855	.null_fd = .init,
856	.random_fd = .init,
857
858	.csprng_mutex = .init,
859	.csprng = .uninitialized,
860	};
861	const main_fiber: *Fiber = @ptrCast(&ev.main_fiber_buffer);
862	main_fiber.* = .{
863	.required_align = {},
864	.context = undefined,
865	.await_count = 0,
866	.link = .{ .awaiter = null },
867	.status = .{ .queue_next = null },
868	.cancel_status = .unrequested,
869	.cancel_protection = .unblocked,
870	.name = if (tracy.enable) "main task",
871	};
872	const main_thread = &ev.threads.allocated[0];
873	Thread.self = main_thread;
874	main_thread.* = .{
875	.required_align = {},
876	.thread = undefined,
877	.idle_context = switch (builtin.cpu.arch) {
878	.aarch64 => .{
879	.sp = @intFromPtr(allocated_slice[idle_stack_end_offset..].ptr),
880	.fp = @intFromPtr(ev),
881	.pc = @intFromPtr(&mainIdleEntry),
882	},
883	.x86_64 => .{
884	.rsp = @intFromPtr(allocated_slice[idle_stack_end_offset..].ptr),
885	.rbp = @intFromPtr(ev),
886	.rip = @intFromPtr(&mainIdleEntry),
887	},
888	else => @compileError("unimplemented architecture"),
889	},
890	.current_context = &main_fiber.context,
891	.ready_queue = null,
892	.free_queue = null,
893	.io_uring = try .init(
894	@as(u16, 1) << ev.log2_ring_entries,
895	linux.IORING_SETUP_COOP_TASKRUN \| linux.IORING_SETUP_SINGLE_ISSUER,
896	),
897	.idle_search_index = 1,
898	.steal_ready_search_index = 1,
899	.steal_free_search_index = 1,
900	.name_arena = .{},
901	.csprng = .uninitialized,
902	};
903	errdefer main_thread.io_uring.deinit();
904	if (tracy.enable) tracy.fiberEnter(main_fiber.name);
905	}
906
907	pub fn deinit(ev: *Evented) void {
908	const main_fiber: *Fiber = @ptrCast(&ev.main_fiber_buffer);
909	assert(Thread.current().currentFiber() == main_fiber);
910	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
911	for (ev.threads.allocated[0..active_threads]) \|*thread\| {
912	const ready_fiber = @atomicLoad(?*Fiber, &thread.ready_queue, .monotonic);
913	assert(ready_fiber == null or ready_fiber == Fiber.finished); // pending async
914	}
915	ev.yield(null, .exit);
916	ev.null_fd.close();
917	ev.random_fd.close();
918	const allocated_ptr: [*]align(@alignOf(Thread)) u8 = @ptrCast(@alignCast(ev.threads.allocated.ptr));
919	const idle_stack_end_offset = std.mem.alignForward(
920	usize,
921	ev.threads.allocated.len * @sizeOf(Thread) + idle_stack_size,
922	std.heap.page_size_max,
923	);
924	for (ev.threads.allocated[1..active_threads]) \|*thread\| thread.thread.join();
925	for (ev.threads.allocated[0..active_threads]) \|*thread\| thread.deinit(ev.backing_allocator);
926	assert(active_threads == ev.threads.active); // spawned threads while there was no pending async?
927	ev.backing_allocator.free(allocated_ptr[0..idle_stack_end_offset]);
928	ev.* = undefined;
929	}
930
931	fn findReadyFiber(ev: Evented, thread: Thread) ?*Fiber {
932	if (@atomicRmw(?*Fiber, &thread.ready_queue, .Xchg, Fiber.finished, .acquire)) \|ready_fiber\| {
933	assert(ready_fiber != Fiber.finished);
934	@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.status.queue_next, .release);
935	ready_fiber.status.queue_next = null;
936	return ready_fiber;
937	}
938	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
939	for (0..@min(max_steal_ready_search, active_threads)) \|_\| {
940	defer thread.steal_ready_search_index += 1;
941	if (thread.steal_ready_search_index == active_threads) thread.steal_ready_search_index = 0;
942	const steal_ready_search_thread =
943	&ev.threads.allocated[0..active_threads][thread.steal_ready_search_index];
944	if (steal_ready_search_thread == thread) continue;
945	const ready_fiber =
946	@atomicLoad(?*Fiber, &steal_ready_search_thread.ready_queue, .monotonic) orelse continue;
947	if (ready_fiber == Fiber.finished) continue;
948	if (@cmpxchgWeak(
949	?*Fiber,
950	&steal_ready_search_thread.ready_queue,
951	ready_fiber,
952	null,
953	.acquire,
954	.monotonic,
955	)) \|_\| continue;
956	@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.status.queue_next, .release);
957	ready_fiber.status.queue_next = null;
958	return ready_fiber;
959	}
960	// couldn't find anything to do, so we are now open for business
961	@atomicStore(?*Fiber, &thread.ready_queue, null, .monotonic);
962	return null;
963	}
964
965	fn yield(ev: Evented, maybe_ready_fiber: ?Fiber, pending_task: SwitchMessage.PendingTask) void {
966	const thread: *Thread = .current();
967	const ready_context = if (maybe_ready_fiber orelse ev.findReadyFiber(thread)) \|ready_fiber\|
968	&ready_fiber.context
969	else
970	&thread.idle_context;
971	const message: SwitchMessage = .{
972	.contexts = .{
973	.old = thread.current_context,
974	.new = ready_context,
975	},
976	.pending_task = pending_task,
977	};
978	contextSwitch(&message).handle(ev);
979	}
980
981	fn schedule(ev: Evented, thread: Thread, ready_queue: Fiber.Queue) bool {
982	// shared fields of previous `Thread` must be initialized before later ones are marked as active
983	const new_thread_index = @atomicLoad(u32, &ev.threads.active, .acquire);
984	for (0..@min(max_idle_search, new_thread_index)) \|_\| {
985	defer thread.idle_search_index += 1;
986	if (thread.idle_search_index == new_thread_index) thread.idle_search_index = 0;
987	const idle_search_thread = &ev.threads.allocated[0..new_thread_index][thread.idle_search_index];
988	if (idle_search_thread == thread) continue;
989	if (@cmpxchgWeak(
990	?*Fiber,
991	&idle_search_thread.ready_queue,
992	null,
993	ready_queue.head,
994	.release,
995	.monotonic,
996	)) \|_\| continue;
997	thread.enqueue().* = .{
998	.opcode = .MSG_RING,
999	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1000	.ioprio = 0,
1001	.fd = idle_search_thread.io_uring.fd,
1002	.off = @intFromEnum(Completion.Userdata.wakeup),
1003	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
1004	.len = 0,
1005	.rw_flags = 0,
1006	.user_data = @intFromEnum(Completion.Userdata.wakeup),
1007	.buf_index = 0,
1008	.personality = 0,
1009	.splice_fd_in = 0,
1010	.addr3 = 0,
1011	.resv = 0,
1012	};
1013	return true;
1014	}
1015	spawn_thread: {
1016	// previous failed reservations must have completed before retrying
1017	if (new_thread_index == ev.threads.allocated.len or @cmpxchgWeak(
1018	u32,
1019	&ev.threads.reserved,
1020	new_thread_index,
1021	new_thread_index + 1,
1022	.acquire,
1023	.monotonic,
1024	) != null) break :spawn_thread;
1025	const new_thread = &ev.threads.allocated[new_thread_index];
1026	const next_thread_index = new_thread_index + 1;
1027	var params = std.mem.zeroInit(linux.io_uring_params, .{
1028	.flags = linux.IORING_SETUP_ATTACH_WQ \|
1029	linux.IORING_SETUP_R_DISABLED \|
1030	linux.IORING_SETUP_COOP_TASKRUN \|
1031	linux.IORING_SETUP_SINGLE_ISSUER,
1032	.wq_fd = @as(u32, @intCast(ev.threads.allocated[0].io_uring.fd)),
1033	});
1034	new_thread.* = .{
1035	.required_align = {},
1036	.thread = undefined,
1037	.idle_context = undefined,
1038	.current_context = &new_thread.idle_context,
1039	.ready_queue = ready_queue.head,
1040	.free_queue = null,
1041	.io_uring = IoUring.init_params(@as(u16, 1) << ev.log2_ring_entries, &params) catch \|err\| {
1042	@atomicStore(u32, &ev.threads.reserved, new_thread_index, .release);
1043	// no more access to `thread` after giving up reservation
1044	log.warn("unable to create worker thread due to io_uring init failure: {s}", .{
1045	@errorName(err),
1046	});
1047	break :spawn_thread;
1048	},
1049	.idle_search_index = 0,
1050	.steal_ready_search_index = 0,
1051	.steal_free_search_index = 0,
1052	.name_arena = .{},
1053	.csprng = .uninitialized,
1054	};
1055	new_thread.thread = std.Thread.spawn(.{
1056	.stack_size = idle_stack_size,
1057	.allocator = ev.allocator(),
1058	}, threadEntry, .{ ev, new_thread_index }) catch \|err\| {
1059	new_thread.io_uring.deinit();
1060	@atomicStore(u32, &ev.threads.reserved, new_thread_index, .release);
1061	// no more access to `thread` after giving up reservation
1062	log.warn("unable to create worker thread due spawn failure: {s}", .{@errorName(err)});
1063	break :spawn_thread;
1064	};
1065	// shared fields of `Thread` must be initialized before being marked active
1066	@atomicStore(u32, &ev.threads.active, next_thread_index, .release);
1067	return false;
1068	}
1069	// nobody wanted it, so just queue it on ourselves
1070	while (true) ready_queue.tail.status.queue_next = @cmpxchgWeak(
1071	?*Fiber,
1072	&thread.ready_queue,
1073	ready_queue.tail.status.queue_next,
1074	ready_queue.head,
1075	.acq_rel,
1076	.acquire,
1077	) orelse break;
1078	return false;
1079	}
1080
1081	fn threadEntry(ev: *Evented, index: u32) void {
1082	const thread: *Thread = &ev.threads.allocated[index];
1083	Thread.self = thread;
1084	switch (linux.errno(linux.io_uring_register(thread.io_uring.fd, .REGISTER_ENABLE_RINGS, null, 0))) {
1085	.SUCCESS => ev.idle(thread),
1086	else => \|err\| @panic(@tagName(err)),
1087	}
1088	}
1089
1090	const Completion = struct {
1091	result: i32,
1092	flags: u32,
1093
1094	const Userdata = enum(usize) {
1095	unused,
1096	wakeup,
1097	futex_wake,
1098	close,
1099	cleanup,
1100	exit,
1101	/// If bit 0 is 1, a pointer to the `context` field of `Io.Batch.Storage.Pending`.
1102	/// If bits 0 and 1 are 0, a `*Fiber`.
1103	_,
1104	};
1105
1106	fn errno(completion: Completion) linux.E {
1107	return linux.errno(@bitCast(@as(isize, completion.result)));
1108	}
1109	};
1110
1111	fn mainIdleEntry() callconv(.naked) void {
1112	switch (builtin.cpu.arch) {
1113	.aarch64 => asm volatile (
1114	\\ mov x0, fp
1115	\\ mov fp, #0
1116	\\ b %[mainIdle]
1117	:
1118	: [mainIdle] "X" (&mainIdle),
1119	),
1120	.x86_64 => asm volatile (
1121	\\ movq %%rbp, %%rdi
1122	\\ xor %%ebp, %%ebp
1123	\\ jmp %[mainIdle:P]
1124	:
1125	: [mainIdle] "X" (&mainIdle),
1126	),
1127	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1128	}
1129	}
1130
1131	fn mainIdle(
1132	ev: *Evented,
1133	message: *const SwitchMessage,
1134	) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Io.fiber.Context)))) noreturn {
1135	message.handle(ev);
1136	ev.idle(&ev.threads.allocated[0]);
1137	ev.yield(@ptrCast(&ev.main_fiber_buffer), .nothing);
1138	unreachable; // switched to dead fiber
1139	}
1140
1141	fn idle(ev: Evented, thread: Thread) void {
1142	var maybe_ready_fiber: ?*Fiber = null;
1143	while (true) {
1144	while (maybe_ready_fiber orelse ev.findReadyFiber(thread)) \|ready_fiber\| {
1145	ev.yield(ready_fiber, .nothing);
1146	maybe_ready_fiber = null;
1147	}
1148	_ = thread.io_uring.submit_and_wait(1) catch \|err\| switch (err) {
1149	error.SignalInterrupt => {},
1150	else => \|e\| @panic(@errorName(e)),
1151	};
1152	var maybe_ready_queue: ?Fiber.Queue = null;
1153	while (true) {
1154	var cqes_buffer: [1 << 8]linux.io_uring_cqe = undefined;
1155	const cqes = cqes_buffer[0 .. thread.io_uring.copy_cqes(&cqes_buffer, 0) catch \|err\| switch (err) {
1156	error.SignalInterrupt => 0,
1157	else => \|e\| @panic(@errorName(e)),
1158	}];
1159	if (cqes.len == 0) break;
1160	for (cqes) \|cqe\| if (cqe.flags & linux.IORING_CQE_F_SKIP == 0) switch (@as(
1161	Completion.Userdata,
1162	@enumFromInt(cqe.user_data),
1163	)) {
1164	.unused => unreachable, // bad submission queued?
1165	.wakeup => {},
1166	.futex_wake => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
1167	.SUCCESS => recoverableOsBugDetected(), // success is skipped
1168	.INVAL => {}, // invalid futex_wait() on ptr done elsewhere
1169	.INTR, .CANCELED => recoverableOsBugDetected(), // `Completion.Userdata.futex_wake` is not cancelable
1170	.FAULT => {}, // pointer became invalid while doing the wake
1171	else => recoverableOsBugDetected(), // deadlock due to operating system bug
1172	},
1173	.close => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
1174	.BADF => recoverableOsBugDetected(), // Always a race condition.
1175	.INTR => {}, // This is still a success. See https://github.com/ziglang/zig/issues/2425
1176	else => {},
1177	},
1178	.cleanup => @panic("failed to notify other threads that we are exiting"),
1179	.exit => {
1180	assert(maybe_ready_fiber == null and maybe_ready_queue == null); // pending async
1181	return;
1182	},
1183	_ => if (@as(?*Fiber, ready_fiber: switch (@as(u2, @truncate(cqe.user_data))) {
1184	0b00 => {
1185	const ready_fiber: *Fiber = @ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
1186	ready_fiber.resultPointer(Completion).* = .{
1187	.result = cqe.res,
1188	.flags = cqe.flags,
1189	};
1190	break :ready_fiber ready_fiber;
1191	},
1192	0b01 => {
1193	thread.enqueue().* = .{
1194	.opcode = .ASYNC_CANCEL,
1195	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1196	.ioprio = 0,
1197	.fd = 0,
1198	.off = 0,
1199	.addr = cqe.user_data & ~@as(usize, 0b11),
1200	.len = 0,
1201	.rw_flags = 0,
1202	.user_data = @intFromEnum(Completion.Userdata.wakeup),
1203	.buf_index = 0,
1204	.personality = 0,
1205	.splice_fd_in = 0,
1206	.addr3 = 0,
1207	.resv = 0,
1208	};
1209	break :ready_fiber null;
1210	},
1211	0b10 => {
1212	const batch_userdata: *Io.Operation.Storage.Pending.Userdata =
1213	@ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
1214	const batch: *Io.Batch = @ptrFromInt(batch_userdata[0]);
1215	var next: usize = 0b00;
1216	batch_userdata[0..3].* = .{ next, @as(u32, @bitCast(cqe.res)), cqe.flags };
1217	while (true) {
1218	next = @cmpxchgWeak(
1219	usize,
1220	@as(*usize, @ptrCast(&batch.userdata)),
1221	next,
1222	cqe.user_data,
1223	.release,
1224	.acquire,
1225	) orelse break;
1226	batch_userdata[0] = next;
1227	}
1228	break :ready_fiber switch (@as(u2, @truncate(next))) {
1229	0b00, 0b01 => @ptrFromInt(next & ~@as(usize, 0b11)),
1230	0b10, 0b11 => null,
1231	};
1232	},
1233	0b11 => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
1234	.SUCCESS => unreachable, // no event count specified
1235	.TIME => {
1236	const context: *usize = @ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
1237	const fiber = @atomicRmw(usize, context, .Add, 0b01, .acquire);
1238	break :ready_fiber switch (@as(u2, @truncate(fiber))) {
1239	else => unreachable, // timeout completed multiple times
1240	0b00 => @ptrFromInt(fiber & ~@as(usize, 0b11)),
1241	0b10 => null,
1242	};
1243	},
1244	.CANCELED => null, // user data may have been invalidated
1245	else => \|err\| unexpectedErrno(err) catch null,
1246	},
1247	})) \|ready_fiber\| {
1248	assert(ready_fiber.status.queue_next == null);
1249	if (maybe_ready_fiber == null) {
1250	maybe_ready_fiber = ready_fiber;
1251	} else if (maybe_ready_queue) \|*ready_queue\| {
1252	ready_queue.tail.status.queue_next = ready_fiber;
1253	ready_queue.tail = ready_fiber;
1254	} else maybe_ready_queue = .{ .head = ready_fiber, .tail = ready_fiber };
1255	},
1256	};
1257	}
1258	if (maybe_ready_queue) \|ready_queue\| _ = ev.schedule(thread, ready_queue);
1259	}
1260	}
1261
1262	const SwitchMessage = struct {
1263	contexts: Io.fiber.Switch,
1264	pending_task: PendingTask,
1265
1266	const PendingTask = union(enum) {
1267	nothing,
1268	reschedule,
1269	await: u31,
1270	group_await: Group,
1271	group_cancel: Group,
1272	batch_await: *Io.Batch,
1273	destroy,
1274	exit,
1275	};
1276
1277	fn handle(message: const SwitchMessage, ev: Evented) void {
1278	const thread: *Thread = .current();
1279	thread.current_context = message.contexts.new;
1280	if (tracy.enable) {
1281	if (message.contexts.new != &thread.idle_context) {
1282	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.new));
1283	tracy.fiberEnter(fiber.name);
1284	} else tracy.fiberLeave();
1285	}
1286	switch (message.pending_task) {
1287	.nothing => {},
1288	.reschedule => if (message.contexts.old != &thread.idle_context) {
1289	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
1290	assert(fiber.status.queue_next == null);
1291	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1292	},
1293	.await => \|count\| {
1294	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
1295	if (@atomicRmw(i32, &fiber.await_count, .Sub, count, .monotonic) > 0)
1296	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1297	},
1298	.group_await => \|group\| {
1299	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
1300	if (group.await(ev, fiber))
1301	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1302	},
1303	.group_cancel => \|group\| {
1304	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
1305	if (group.cancel(ev, fiber))
1306	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1307	},
1308	.batch_await => \|batch\| {
1309	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
1310	if (@cmpxchgStrong(
1311	?*anyopaque,
1312	&batch.userdata,
1313	null,
1314	fiber,
1315	.release,
1316	.monotonic,
1317	)) \|head\| {
1318	assert(@as(u2, @truncate(@intFromPtr(head))) != 0b00);
1319	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1320	}
1321	},
1322	.destroy => {
1323	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
1324	fiber.destroy();
1325	},
1326	.exit => for (
1327	ev.threads.allocated[0..@atomicLoad(u32, &ev.threads.active, .acquire)],
1328	) \|*each_thread\| {
1329	thread.enqueue().* = .{
1330	.opcode = .MSG_RING,
1331	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1332	.ioprio = 0,
1333	.fd = each_thread.io_uring.fd,
1334	.off = @intFromEnum(Completion.Userdata.exit),
1335	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
1336	.len = 0,
1337	.rw_flags = 0,
1338	.user_data = @intFromEnum(Completion.Userdata.cleanup),
1339	.buf_index = 0,
1340	.personality = 0,
1341	.splice_fd_in = 0,
1342	.addr3 = 0,
1343	.resv = 0,
1344	};
1345	},
1346	}
1347	}
1348	};
1349
1350	inline fn contextSwitch(message: const SwitchMessage) const SwitchMessage {
1351	return @fieldParentPtr("contexts", Io.fiber.contextSwitch(&message.contexts));
1352	}
1353
1354	fn crashHandler(userdata: ?*anyopaque) void {
1355	const ev: *Evented = @ptrCast(@alignCast(userdata));
1356	_ = ev;
1357	const thread = Thread.self orelse std.process.abort();
1358	if (thread.current_context == &thread.idle_context) std.process.abort();
1359	const fiber = thread.currentFiber();
1360	@atomicStore(
1361	Fiber.CancelStatus,
1362	&fiber.cancel_status,
1363	.{ .requested = true, .awaiting = .nothing },
1364	.monotonic,
1365	);
1366	fiber.cancel_protection = .{ .user = .blocked, .acknowledged = true };
1367	}
1368
1369	const AsyncClosure = struct {
1370	ev: *Evented,
1371	fiber: *Fiber,
1372	start: const fn (context: const anyopaque, result: *anyopaque) void,
1373	result_align: Alignment,
1374
1375	fn fromFiber(fiber: Fiber) AsyncClosure {
1376	return @ptrFromInt(Fiber.max_context_align.max(.of(AsyncClosure)).backward(
1377	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
1378	) - @sizeOf(AsyncClosure));
1379	}
1380
1381	fn contextPointer(closure: AsyncClosure) []align(Fiber.max_context_align.toByteUnits()) u8 {
1382	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(AsyncClosure));
1383	}
1384
1385	fn entry() callconv(.naked) void {
1386	switch (builtin.cpu.arch) {
1387	.aarch64 => asm volatile (
1388	\\ mov x0, sp
1389	\\ b %[call]
1390	:
1391	: [call] "X" (&call),
1392	),
1393	.x86_64 => asm volatile (
1394	\\ leaq 8(%%rsp), %%rdi
1395	\\ jmp %[call:P]
1396	:
1397	: [call] "X" (&call),
1398	),
1399	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1400	}
1401	}
1402
1403	fn call(
1404	closure: *AsyncClosure,
1405	message: *const SwitchMessage,
1406	) callconv(.withStackAlign(.c, @alignOf(AsyncClosure))) noreturn {
1407	message.handle(closure.ev);
1408	const fiber = closure.fiber;
1409	closure.start(closure.contextPointer(), fiber.resultBytes(closure.result_align));
1410	closure.ev.yield(
1411	if (@atomicRmw(?*Fiber, &fiber.link.awaiter, .Xchg, Fiber.finished, .acq_rel)) \|awaiter\|
1412	if (@atomicRmw(i32, &awaiter.await_count, .Add, 1, .monotonic) == -1) awaiter else null
1413	else
1414	null,
1415	.nothing,
1416	);
1417	unreachable; // switched to dead fiber
1418	}
1419	};
1420
1421	fn async(
1422	userdata: ?*anyopaque,
1423	result: []u8,
1424	result_alignment: Alignment,
1425	context: []const u8,
1426	context_alignment: Alignment,
1427	start: const fn (context: const anyopaque, result: *anyopaque) void,
1428	) ?*std.Io.AnyFuture {
1429	const ev: *Evented = @ptrCast(@alignCast(userdata));
1430	return concurrent(ev, result.len, result_alignment, context, context_alignment, start) catch {
1431	start(context.ptr, result.ptr);
1432	return null;
1433	};
1434	}
1435
1436	fn concurrent(
1437	userdata: ?*anyopaque,
1438	result_len: usize,
1439	result_alignment: Alignment,
1440	context: []const u8,
1441	context_alignment: Alignment,
1442	start: const fn (context: const anyopaque, result: *anyopaque) void,
1443	) Io.ConcurrentError!*std.Io.AnyFuture {
1444	assert(result_alignment.compare(.lte, Fiber.max_result_align)); // TODO
1445	assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
1446	assert(result_len <= Fiber.max_result_size); // TODO
1447	assert(context.len <= Fiber.max_context_size); // TODO
1448
1449	const ev: *Evented = @ptrCast(@alignCast(userdata));
1450	const fiber = Fiber.create(ev) catch \|err\| switch (err) {
1451	error.OutOfMemory => return error.ConcurrencyUnavailable,
1452	};
1453
1454	const closure: *AsyncClosure = .fromFiber(fiber);
1455	fiber.* = .{
1456	.required_align = {},
1457	.context = switch (builtin.cpu.arch) {
1458	.aarch64 => .{
1459	.sp = @intFromPtr(closure),
1460	.fp = 0,
1461	.pc = @intFromPtr(&AsyncClosure.entry),
1462	},
1463	.x86_64 => .{
1464	.rsp = @intFromPtr(closure) - 8,
1465	.rbp = 0,
1466	.rip = @intFromPtr(&AsyncClosure.entry),
1467	},
1468	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1469	},
1470	.await_count = 0,
1471	.link = .{ .awaiter = null },
1472	.status = .{ .queue_next = null },
1473	.cancel_status = .unrequested,
1474	.cancel_protection = .unblocked,
1475	.name = if (tracy.enable) name: {
1476	const thread: *Thread = .current();
1477	var name_arena = thread.name_arena.promote(std.heap.page_allocator);
1478	defer thread.name_arena = name_arena.state;
1479	break :name std.fmt.allocPrintSentinel(
1480	name_arena.allocator(),
1481	"task {d}",
1482	.{@atomicRmw(u64, &Fiber.next_name, .Add, 1, .monotonic)},
1483	0,
1484	) catch return error.ConcurrencyUnavailable;
1485	},
1486	};
1487	closure.* = .{
1488	.ev = ev,
1489	.fiber = fiber,
1490	.start = start,
1491	.result_align = result_alignment,
1492	};
1493	@memcpy(closure.contextPointer(), context);
1494
1495	const thread: *Thread = .current();
1496	if (ev.schedule(thread, .{ .head = fiber, .tail = fiber })) thread.submit();
1497	return @ptrCast(fiber);
1498	}
1499
1500	fn await(
1501	userdata: ?*anyopaque,
1502	future: *std.Io.AnyFuture,
1503	result: []u8,
1504	result_alignment: Alignment,
1505	) void {
1506	const ev: *Evented = @ptrCast(@alignCast(userdata));
1507	const fiber = Thread.current().currentFiber();
1508	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
1509	if (@atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, fiber, .acq_rel)) \|awaiter\| {
1510	assert(awaiter == Fiber.finished);
1511	} else while (true) {
1512	ev.yield(null, .{ .await = 1 });
1513	const awaiter = @atomicLoad(?*Fiber, &future_fiber.link.awaiter, .acquire);
1514	if (awaiter == Fiber.finished) break;
1515	assert(awaiter == fiber); // spurious wakeup
1516	}
1517	@memcpy(result, future_fiber.resultBytes(result_alignment));
1518	future_fiber.destroy();
1519	}
1520
1521	fn cancel(
1522	userdata: ?*anyopaque,
1523	future: *std.Io.AnyFuture,
1524	result: []u8,
1525	result_alignment: Alignment,
1526	) void {
1527	const ev: *Evented = @ptrCast(@alignCast(userdata));
1528	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
1529	future_fiber.requestCancel(ev);
1530	await(ev, future, result, result_alignment);
1531	}
1532
1533	const Group = struct {
1534	ptr: *Io.Group,
1535
1536	const List = packed struct(usize) {
1537	cancel_requested: bool,
1538	awaiter_delayed: bool,
1539	fibers: Fiber.PackedPtr,
1540	};
1541	fn listPtr(group: Group) *List {
1542	return @ptrCast(&group.ptr.token);
1543	}
1544
1545	const Mutex = packed struct(u32) {
1546	locked: bool,
1547	contended: bool,
1548	shared2: u30,
1549	};
1550	fn mutexPtr(group: Group) *Mutex {
1551	return switch (comptime builtin.cpu.arch.endian()) {
1552	.little => @ptrCast(&group.ptr.state),
1553	.big => @ptrCast(@alignCast(
1554	@as([*]u8, @ptrCast(&group.ptr.state)) + @sizeOf(usize) - @sizeOf(u32),
1555	)),
1556	};
1557	}
1558
1559	const Awaiter = packed struct(usize) {
1560	locked: bool,
1561	contended: bool,
1562	awaiter: Fiber.PackedPtr,
1563	};
1564	fn awaiterPtr(group: Group) *Awaiter {
1565	return @ptrCast(&group.ptr.state);
1566	}
1567
1568	fn lock(group: Group, ev: *Evented) void {
1569	const mutex = group.mutexPtr();
1570	{
1571	const old_state = @atomicRmw(
1572	Mutex,
1573	mutex,
1574	.Or,
1575	.{ .locked = true, .contended = false, .shared2 = 0 },
1576	.acquire,
1577	);
1578	if (!old_state.locked) {
1579	@branchHint(.likely);
1580	return;
1581	}
1582	if (old_state.contended) {
1583	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
1584	}
1585	}
1586	while (true) {
1587	var old_state = @atomicRmw(
1588	Mutex,
1589	mutex,
1590	.Or,
1591	.{ .locked = true, .contended = true, .shared2 = 0 },
1592	.acquire,
1593	);
1594	if (!old_state.locked) {
1595	@branchHint(.likely);
1596	return;
1597	}
1598	old_state.contended = true;
1599	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
1600	}
1601	}
1602
1603	fn unlock(group: Group, ev: *Evented) void {
1604	const mutex = group.mutexPtr();
1605	const old_state = @atomicRmw(
1606	Mutex,
1607	mutex,
1608	.And,
1609	.{ .locked = false, .contended = false, .shared2 = std.math.maxInt(u30) },
1610	.release,
1611	);
1612	assert(old_state.locked);
1613	if (old_state.contended) futexWake(ev, @ptrCast(mutex), 1);
1614	}
1615
1616	fn addFiber(group: Group, ev: Evented, fiber: Fiber) void {
1617	group.lock(ev);
1618	defer group.unlock(ev);
1619	const list_ptr = group.listPtr();
1620	const list = @atomicLoad(List, list_ptr, .monotonic);
1621	if (list.cancel_requested) fiber.cancel_status = .{ .requested = true, .awaiting = .nothing };
1622	const old_head = list.fibers.unpack();
1623	if (old_head) \|head\| head.link.group.prev = fiber;
1624	fiber.link.group.next = old_head;
1625	@atomicStore(List, list_ptr, .{
1626	.cancel_requested = list.cancel_requested,
1627	.awaiter_delayed = list.awaiter_delayed,
1628	.fibers = .pack(fiber),
1629	}, .monotonic);
1630	}
1631
1632	fn removeFiber(group: Group, ev: Evented, fiber: Fiber) ?*Fiber {
1633	group.lock(ev);
1634	defer group.unlock(ev);
1635	const list_ptr = group.listPtr();
1636	const list = @atomicLoad(List, list_ptr, .monotonic);
1637	if (fiber.link.group.next) \|next\| next.link.group.prev = fiber.link.group.prev;
1638	if (fiber.link.group.prev) \|prev\| {
1639	prev.link.group.next = fiber.link.group.next;
1640	} else if (fiber.link.group.next) \|new_head\| {
1641	@atomicStore(List, list_ptr, .{
1642	.cancel_requested = list.cancel_requested,
1643	.awaiter_delayed = list.awaiter_delayed,
1644	.fibers = .pack(new_head),
1645	}, .monotonic);
1646	} else if (@atomicLoad(Awaiter, group.awaiterPtr(), .monotonic).awaiter.unpack()) \|awaiter\| {
1647	if (!awaiter.cancel_status.changeAwaiting(.group, .nothing) or list.cancel_requested) {
1648	@atomicStore(List, list_ptr, .{
1649	.cancel_requested = false,
1650	.awaiter_delayed = false,
1651	.fibers = .null,
1652	}, .release);
1653	assert(awaiter.status.awaiting_group.ptr == group.ptr);
1654	awaiter.status = .{ .queue_next = null };
1655	return awaiter;
1656	}
1657	// Race with `Fiber.requestCancel`
1658	@atomicStore(List, list_ptr, .{
1659	.cancel_requested = false,
1660	.awaiter_delayed = true,
1661	.fibers = .null,
1662	}, .monotonic);
1663	} else @atomicStore(List, list_ptr, .{
1664	.cancel_requested = false,
1665	.awaiter_delayed = false,
1666	.fibers = .null,
1667	}, .release);
1668	return null;
1669	}
1670
1671	fn await(group: Group, ev: Evented, awaiter: Fiber) bool {
1672	group.lock(ev);
1673	defer group.unlock(ev);
1674	if (@atomicLoad(List, group.listPtr(), .monotonic).fibers.unpack()) \|_\| {
1675	if (group.registerAwaiter(awaiter) and awaiter.cancel_protection.check() == .unblocked) {
1676	// The awaiter already had an unacknowledged cancelation request before
1677	// attempting to await a group, so propagate the cancelation to the group.
1678	assert(!group.cancelLocked(ev, null));
1679	}
1680	return false;
1681	}
1682	return true;
1683	}
1684
1685	fn cancel(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
1686	group.lock(ev);
1687	defer group.unlock(ev);
1688	return group.cancelLocked(ev, maybe_awaiter);
1689	}
1690
1691	/// Assumes the mutex is held.
1692	fn cancelLocked(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
1693	const list_ptr = group.listPtr();
1694	const list = @atomicRmw(
1695	List,
1696	list_ptr,
1697	.Add,
1698	.{ .cancel_requested = true, .awaiter_delayed = false, .fibers = .null },
1699	.monotonic,
1700	);
1701	assert(!list.cancel_requested);
1702	if (list.fibers.unpack()) \|head\| {
1703	var maybe_fiber: ?*Fiber = head;
1704	while (maybe_fiber) \|fiber\| {
1705	fiber.requestCancel(ev);
1706	maybe_fiber = fiber.link.group.next;
1707	}
1708	if (maybe_awaiter) \|awaiter\| _ = group.registerAwaiter(awaiter);
1709	return false;
1710	}
1711	@atomicStore(
1712	List,
1713	list_ptr,
1714	.{ .cancel_requested = false, .awaiter_delayed = false, .fibers = .null },
1715	.release,
1716	);
1717	return if (maybe_awaiter) \|_\| true else list.awaiter_delayed;
1718	}
1719
1720	/// Assumes the mutex is held.
1721	fn registerAwaiter(group: Group, awaiter: *Fiber) bool {
1722	assert(awaiter.status.queue_next == null);
1723	awaiter.status = .{ .awaiting_group = group };
1724	assert(@atomicRmw(
1725	Awaiter,
1726	group.awaiterPtr(),
1727	.Add,
1728	.{ .locked = false, .contended = false, .awaiter = .pack(awaiter) },
1729	.monotonic,
1730	).awaiter == .null);
1731	return awaiter.cancel_status.changeAwaiting(.nothing, .group);
1732	}
1733
1734	const AsyncClosure = struct {
1735	ev: *Evented,
1736	group: Group,
1737	fiber: *Fiber,
1738	start: const fn (context: const anyopaque) Io.Cancelable!void,
1739
1740	fn fromFiber(fiber: Fiber) Group.AsyncClosure {
1741	return @ptrFromInt(Fiber.max_context_align.max(.of(Group.AsyncClosure)).backward(
1742	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
1743	) - @sizeOf(Group.AsyncClosure));
1744	}
1745
1746	fn contextPointer(
1747	closure: *Group.AsyncClosure,
1748	) [*]align(Fiber.max_context_align.toByteUnits()) u8 {
1749	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(Group.AsyncClosure));
1750	}
1751
1752	fn entry() callconv(.naked) void {
1753	switch (builtin.cpu.arch) {
1754	.aarch64 => asm volatile (
1755	\\ mov x0, sp
1756	\\ b %[call]
1757	:
1758	: [call] "X" (&call),
1759	),
1760	.x86_64 => asm volatile (
1761	\\ leaq 8(%%rsp), %%rdi
1762	\\ jmp %[call:P]
1763	:
1764	: [call] "X" (&call),
1765	),
1766	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1767	}
1768	}
1769
1770	fn call(
1771	closure: *Group.AsyncClosure,
1772	message: *const SwitchMessage,
1773	) callconv(.withStackAlign(.c, @alignOf(Group.AsyncClosure))) noreturn {
1774	message.handle(closure.ev);
1775	assert(closure.fiber.status.queue_next == null);
1776	const result = closure.start(closure.contextPointer());
1777	const ev = closure.ev;
1778	const group = closure.group;
1779	const fiber = closure.fiber;
1780	const cancel_acknowledged = fiber.cancel_protection.acknowledged;
1781	if (result) {
1782	assert(!cancel_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
1783	} else \|err\| switch (err) {
1784	error.Canceled => assert(cancel_acknowledged), // group task returned `error.Canceled` but was never canceled
1785	}
1786	ev.yield(group.removeFiber(ev, fiber), .destroy);
1787	unreachable; // switched to dead fiber
1788	}
1789	};
1790	};
1791
1792	fn groupAsync(
1793	userdata: ?*anyopaque,
1794	type_erased: *Io.Group,
1795	context: []const u8,
1796	context_alignment: Alignment,
1797	start: const fn (context: const anyopaque) Io.Cancelable!void,
1798	) void {
1799	const ev: *Evented = @ptrCast(@alignCast(userdata));
1800	return groupConcurrent(ev, type_erased, context, context_alignment, start) catch {
1801	const fiber = Thread.current().currentFiber();
1802	const pre_acknowledged = fiber.cancel_protection.acknowledged;
1803	const result = start(context.ptr);
1804	const post_acknowledged = fiber.cancel_protection.acknowledged;
1805	if (result) {
1806	if (pre_acknowledged) {
1807	assert(post_acknowledged); // group task called `recancel` but was not canceled
1808	} else {
1809	assert(!post_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
1810	}
1811	} else \|err\| switch (err) {
1812	// Don't swallow the cancelation: make it visible to the `Group.async` caller.
1813	error.Canceled => {
1814	assert(!pre_acknowledged); // group task called `recancel` but was not canceled
1815	assert(post_acknowledged); // group task returned `error.Canceled` but was never canceled
1816	fiber.cancel_protection.recancel();
1817	},
1818	}
1819	};
1820	}
1821
1822	fn groupConcurrent(
1823	userdata: ?*anyopaque,
1824	type_erased: *Io.Group,
1825	context: []const u8,
1826	context_alignment: Alignment,
1827	start: const fn (context: const anyopaque) Io.Cancelable!void,
1828	) Io.ConcurrentError!void {
1829	assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
1830	assert(context.len <= Fiber.max_context_size); // TODO
1831
1832	const ev: *Evented = @ptrCast(@alignCast(userdata));
1833	const group: Group = .{ .ptr = type_erased };
1834	const fiber = Fiber.create(ev) catch \|err\| switch (err) {
1835	error.OutOfMemory => return error.ConcurrencyUnavailable,
1836	};
1837
1838	const closure: *Group.AsyncClosure = .fromFiber(fiber);
1839	fiber.* = .{
1840	.required_align = {},
1841	.context = switch (builtin.cpu.arch) {
1842	.aarch64 => .{
1843	.sp = @intFromPtr(closure),
1844	.fp = 0,
1845	.pc = @intFromPtr(&Group.AsyncClosure.entry),
1846	},
1847	.x86_64 => .{
1848	.rsp = @intFromPtr(closure) - 8,
1849	.rbp = 0,
1850	.rip = @intFromPtr(&Group.AsyncClosure.entry),
1851	},
1852	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1853	},
1854	.await_count = 0,
1855	.link = .{ .group = .{ .prev = null, .next = null } },
1856	.status = .{ .queue_next = null },
1857	.cancel_status = .unrequested,
1858	.cancel_protection = .unblocked,
1859	.name = if (tracy.enable) name: {
1860	const thread: *Thread = .current();
1861	var name_arena = thread.name_arena.promote(std.heap.page_allocator);
1862	defer thread.name_arena = name_arena.state;
1863	break :name std.fmt.allocPrintSentinel(
1864	name_arena.allocator(),
1865	"group task {d}",
1866	.{@atomicRmw(u64, &Fiber.next_name, .Add, 1, .monotonic)},
1867	0,
1868	) catch return error.ConcurrencyUnavailable;
1869	},
1870	};
1871	closure.* = .{
1872	.ev = ev,
1873	.group = group,
1874	.fiber = fiber,
1875	.start = start,
1876	};
1877	@memcpy(closure.contextPointer(), context);
1878	group.addFiber(ev, fiber);
1879	const thread: *Thread = .current();
1880	if (ev.schedule(thread, .{ .head = fiber, .tail = fiber })) thread.submit();
1881	}
1882
1883	fn groupAwait(
1884	userdata: ?*anyopaque,
1885	type_erased: *Io.Group,
1886	initial_token: *anyopaque,
1887	) Io.Cancelable!void {
1888	const ev: *Evented = @ptrCast(@alignCast(userdata));
1889	_ = initial_token;
1890	ev.yield(null, .{ .group_await = .{ .ptr = type_erased } });
1891	}
1892
1893	fn groupCancel(userdata: ?anyopaque, type_erased: Io.Group, initial_token: *anyopaque) void {
1894	const ev: *Evented = @ptrCast(@alignCast(userdata));
1895	_ = initial_token;
1896	ev.yield(null, .{ .group_cancel = .{ .ptr = type_erased } });
1897	}
1898
1899	fn recancel(userdata: ?*anyopaque) void {
1900	const ev: *Evented = @ptrCast(@alignCast(userdata));
1901	_ = ev;
1902	Thread.current().currentFiber().cancel_protection.recancel();
1903	}
1904
1905	fn swapCancelProtection(userdata: ?*anyopaque, new: Io.CancelProtection) Io.CancelProtection {
1906	const ev: *Evented = @ptrCast(@alignCast(userdata));
1907	_ = ev;
1908	const cancel_protection = &Thread.current().currentFiber().cancel_protection;
1909	defer cancel_protection.user = new;
1910	return cancel_protection.user;
1911	}
1912
1913	fn checkCancel(userdata: ?*anyopaque) Io.Cancelable!void {
1914	const ev: *Evented = @ptrCast(@alignCast(userdata));
1915	_ = ev;
1916	const fiber = Thread.current().currentFiber();
1917	switch (fiber.cancel_protection.check()) {
1918	.unblocked => {
1919	const cancel_status = @atomicLoad(Fiber.CancelStatus, &fiber.cancel_status, .monotonic);
1920	assert(cancel_status.awaiting == .nothing);
1921	if (cancel_status.requested) {
1922	@branchHint(.unlikely);
1923	fiber.cancel_protection.acknowledge();
1924	return error.Canceled;
1925	}
1926	},
1927	.blocked => {},
1928	}
1929	}
1930
1931	fn select(userdata: ?anyopaque, futures: []const Io.AnyFuture) Io.Cancelable!usize {
1932	const ev: *Evented = @ptrCast(@alignCast(userdata));
1933	var cancel_region: CancelRegion = .init();
1934	defer cancel_region.deinit();
1935	var await_count: u31, var result = for (futures, 0..) \|future, future_index\| {
1936	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
1937	if (@atomicRmw(
1938	?*Fiber,
1939	&future_fiber.link.awaiter,
1940	.Xchg,
1941	cancel_region.fiber,
1942	.acq_rel,
1943	)) \|awaiter\| {
1944	assert(awaiter == Fiber.finished);
1945	break .{ @intCast(future_index), future_index };
1946	}
1947	} else result: {
1948	const await_count: u31 = @intCast(futures.len);
1949	cancel_region.await(.select) catch \|err\| switch (err) {
1950	error.Canceled => \|e\| break :result .{ await_count + 1, e },
1951	};
1952	ev.yield(null, .{ .await = 1 });
1953	cancel_region.await(.nothing) catch \|err\| switch (err) {
1954	error.Canceled => \|e\| break :result .{ await_count, e },
1955	};
1956	break :result .{ await_count - 1, futures.len };
1957	};
1958	for (futures[0 .. result catch futures.len], 0..) \|future, future_index\| {
1959	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
1960	const awaiter = @atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, null, .monotonic);
1961	if (awaiter == Fiber.finished) {
1962	@atomicStore(?*Fiber, &future_fiber.link.awaiter, Fiber.finished, .monotonic);
1963	result = if (result) \|finished_index\| @min(future_index, finished_index) else \|e\| e;
1964	} else {
1965	assert(awaiter == cancel_region.fiber);
1966	await_count -= 1;
1967	}
1968	}
1969	// Equivalent to `ev.yield(null, .{ .await = await_count });`,
1970	// but avoiding a context switch in the common case.
1971	switch (std.math.order(
1972	@atomicRmw(i32, &cancel_region.fiber.await_count, .Sub, await_count, .monotonic),
1973	await_count,
1974	)) {
1975	.lt => ev.yield(null, .{ .await = 0 }),
1976	.eq => {},
1977	.gt => unreachable,
1978	}
1979	return result;
1980	}
1981
1982	fn futexWait(
1983	userdata: ?*anyopaque,
1984	ptr: *const u32,
1985	expected: u32,
1986	timeout: Io.Timeout,
1987	) Io.Cancelable!void {
1988	const ev: *Evented = @ptrCast(@alignCast(userdata));
1989	const timespec: ?linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = timespec: switch (timeout) {
1990	.none => .{
1991	null,
1992	.awake,
1993	linux.IORING_TIMEOUT_ABS,
1994	},
1995	.duration => \|duration\| {
1996	const ns = duration.raw.toNanoseconds();
1997	break :timespec .{
1998	.{
1999	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
2000	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
2001	},
2002	duration.clock,
2003	0,
2004	};
2005	},
2006	.deadline => \|deadline\| {
2007	const ns = deadline.raw.toNanoseconds();
2008	break :timespec .{
2009	.{
2010	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
2011	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
2012	},
2013	deadline.clock,
2014	linux.IORING_TIMEOUT_ABS,
2015	};
2016	},
2017	};
2018	var cancel_region: CancelRegion = .init();
2019	defer cancel_region.deinit();
2020	const thread = try cancel_region.awaitIoUring();
2021	thread.enqueue().* = .{
2022	.opcode = .FUTEX_WAIT,
2023	.flags = if (timespec) \|_\| linux.IOSQE_IO_LINK else 0,
2024	.ioprio = 0,
2025	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
2026	.off = expected,
2027	.addr = @intFromPtr(ptr),
2028	.len = 0,
2029	.rw_flags = 0,
2030	.user_data = @intFromPtr(cancel_region.fiber),
2031	.buf_index = 0,
2032	.personality = 0,
2033	.splice_fd_in = 0,
2034	.addr3 = std.math.maxInt(u32),
2035	.resv = 0,
2036	};
2037	if (timespec) \|timespec_ptr\| thread.enqueue(). = .{
2038	.opcode = .LINK_TIMEOUT,
2039	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
2040	.ioprio = 0,
2041	.fd = 0,
2042	.off = 0,
2043	.addr = @intFromPtr(timespec_ptr),
2044	.len = 1,
2045	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
2046	.real => linux.IORING_TIMEOUT_REALTIME,
2047	else => 0,
2048	.boot => linux.IORING_TIMEOUT_BOOTTIME,
2049	}),
2050	.user_data = @intFromEnum(Completion.Userdata.wakeup),
2051	.buf_index = 0,
2052	.personality = 0,
2053	.splice_fd_in = 0,
2054	.addr3 = 0,
2055	.resv = 0,
2056	};
2057	ev.yield(null, .nothing);
2058	switch (cancel_region.errno()) {
2059	.SUCCESS => {}, // notified by `wake()`
2060	.INTR, .CANCELED => {}, // caller's responsibility to retry
2061	.AGAIN => {}, // ptr.* != expect
2062	.INVAL => {}, // possibly timeout overflow
2063	.TIMEDOUT => unreachable,
2064	.FAULT => recoverableOsBugDetected(), // ptr was invalid
2065	else => recoverableOsBugDetected(),
2066	}
2067	}
2068
2069	fn futexWaitUncancelable(userdata: ?anyopaque, ptr: const u32, expected: u32) void {
2070	const ev: *Evented = @ptrCast(@alignCast(userdata));
2071	var cancel_region: CancelRegion = .initBlocked();
2072	defer cancel_region.deinit();
2073	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
2074	error.Canceled => unreachable, // blocked
2075	};
2076	thread.enqueue().* = .{
2077	.opcode = .FUTEX_WAIT,
2078	.flags = 0,
2079	.ioprio = 0,
2080	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
2081	.off = expected,
2082	.addr = @intFromPtr(ptr),
2083	.len = 0,
2084	.rw_flags = 0,
2085	.user_data = @intFromPtr(cancel_region.fiber),
2086	.buf_index = 0,
2087	.personality = 0,
2088	.splice_fd_in = 0,
2089	.addr3 = std.math.maxInt(u32),
2090	.resv = 0,
2091	};
2092	ev.yield(null, .nothing);
2093	switch (cancel_region.errno()) {
2094	.SUCCESS => {}, // notified by `wake()`
2095	.INTR, .CANCELED => {}, // caller's responsibility to retry
2096	.AGAIN => {}, // ptr.* != expect
2097	.INVAL => {}, // possibly timeout overflow
2098	.FAULT => recoverableOsBugDetected(), // ptr was invalid
2099	else => recoverableOsBugDetected(),
2100	}
2101	}
2102
2103	fn futexWake(userdata: ?anyopaque, ptr: const u32, max_waiters: u32) void {
2104	const ev: *Evented = @ptrCast(@alignCast(userdata));
2105	_ = ev;
2106	const thread: *Thread = .current();
2107	thread.enqueue().* = .{
2108	.opcode = .FUTEX_WAKE,
2109	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
2110	.ioprio = 0,
2111	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
2112	.off = max_waiters,
2113	.addr = @intFromPtr(ptr),
2114	.len = 0,
2115	.rw_flags = 0,
2116	.user_data = @intFromEnum(Completion.Userdata.futex_wake),
2117	.buf_index = 0,
2118	.personality = 0,
2119	.splice_fd_in = 0,
2120	.addr3 = std.math.maxInt(u32),
2121	.resv = 0,
2122	};
2123	thread.submit();
2124	}
2125
2126	fn operate(userdata: ?*anyopaque, operation: Io.Operation) Io.Cancelable!Io.Operation.Result {
2127	const ev: *Evented = @ptrCast(@alignCast(userdata));
2128	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
2129	defer maybe_sync.deinit(ev);
2130	return switch (operation) {
2131	.file_read_streaming => \|o\| .{
2132	.file_read_streaming = ev.fileReadStreaming(
2133	&maybe_sync.cancel_region,
2134	o.file,
2135	o.data,
2136	) catch \|err\| switch (err) {
2137	error.Canceled => \|e\| return e,
2138	else => \|e\| e,
2139	},
2140	},
2141	.file_write_streaming => \|o\| .{
2142	.file_write_streaming = ev.fileWriteStreaming(
2143	&maybe_sync.cancel_region,
2144	o.file,
2145	o.header,
2146	o.data,
2147	o.splat,
2148	) catch \|err\| switch (err) {
2149	error.Canceled => \|e\| return e,
2150	else => \|e\| e,
2151	},
2152	},
2153	.device_io_control => \|o\| .{
2154	.device_io_control = try ev.deviceIoControl(try maybe_sync.enterSync(ev), o),
2155	},
2156	};
2157	}
2158
2159	fn fileReadStreaming(
2160	ev: *Evented,
2161	cancel_region: *CancelRegion,
2162	file: File,
2163	data: []const []u8,
2164	) File.ReadStreamingError!usize {
2165	var iovecs_buffer: [max_iovecs_len]iovec = undefined;
2166	var i: usize = 0;
2167	for (data) \|buf\| {
2168	if (iovecs_buffer.len - i == 0) break;
2169	if (buf.len > 0) {
2170	iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len };
2171	i += 1;
2172	}
2173	}
2174	const dest = iovecs_buffer[0..i];
2175	assert(dest[0].len > 0);
2176
2177	const n = try ev.preadv(cancel_region, file.handle, dest, null);
2178	return if (n == 0) error.EndOfStream else n;
2179	}
2180
2181	fn fileWriteStreaming(
2182	ev: *Evented,
2183	cancel_region: *CancelRegion,
2184	file: File,
2185	header: []const u8,
2186	data: []const []const u8,
2187	splat: usize,
2188	) File.Writer.Error!usize {
2189	var iovecs: [max_iovecs_len]iovec_const = undefined;
2190	var iovlen: iovlen_t = 0;
2191	addBuf(&iovecs, &iovlen, header);
2192	for (data[0 .. data.len - 1]) \|bytes\| addBuf(&iovecs, &iovlen, bytes);
2193	const pattern = data[data.len - 1];
2194	var backup_buffer: [splat_buffer_size]u8 = undefined;
2195	if (iovecs.len - iovlen != 0) switch (splat) {
2196	0 => {},
2197	1 => addBuf(&iovecs, &iovlen, pattern),
2198	else => switch (pattern.len) {
2199	0 => {},
2200	1 => {
2201	const splat_buffer = &backup_buffer;
2202	const memset_len = @min(splat_buffer.len, splat);
2203	const buf = splat_buffer[0..memset_len];
2204	@memset(buf, pattern[0]);
2205	addBuf(&iovecs, &iovlen, buf);
2206	var remaining_splat = splat - buf.len;
2207	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0) {
2208	assert(buf.len == splat_buffer.len);
2209	addBuf(&iovecs, &iovlen, splat_buffer);
2210	remaining_splat -= splat_buffer.len;
2211	}
2212	addBuf(&iovecs, &iovlen, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
2213	},
2214	else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
2215	addBuf(&iovecs, &iovlen, pattern);
2216	},
2217	},
2218	};
2219	return ev.pwritev(cancel_region, file.handle, iovecs[0..iovlen], null);
2220	}
2221
2222	fn deviceIoControl(
2223	ev: *Evented,
2224	sync: *CancelRegion.Sync,
2225	o: Io.Operation.DeviceIoControl,
2226	) Io.Cancelable!i32 {
2227	_ = ev;
2228	while (true) {
2229	try sync.cancel_region.await(.nothing);
2230	const rc = linux.ioctl(o.file.handle, @bitCast(o.code), @intFromPtr(o.arg));
2231	switch (linux.errno(rc)) {
2232	.SUCCESS => return @bitCast(@as(u32, @truncate(rc))),
2233	.INTR => continue,
2234	else => \|err\| return -@as(i32, @intFromEnum(err)),
2235	}
2236	}
2237	}
2238
2239	fn batchAwaitAsync(userdata: ?anyopaque, batch: Io.Batch) Io.Cancelable!void {
2240	const ev: *Evented = @ptrCast(@alignCast(userdata));
2241	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
2242	defer maybe_sync.deinit(ev);
2243	ev.batchDrainSubmitted(&maybe_sync, batch, false) catch \|err\| switch (err) {
2244	error.ConcurrencyUnavailable => unreachable, // passed concurrency=false
2245	error.Canceled => \|e\| return e,
2246	};
2247	maybe_sync.leaveSync(ev);
2248	while (true) {
2249	batchDrainReady(batch) catch \|err\| switch (err) {
2250	error.Timeout => unreachable, // no timeout
2251	};
2252	if (batch.completed.head != .none or batch.pending.head == .none) return;
2253	ev.yield(null, .{ .batch_await = batch });
2254	}
2255	}
2256
2257	fn batchAwaitConcurrent(
2258	userdata: ?*anyopaque,
2259	batch: *Io.Batch,
2260	timeout: Io.Timeout,
2261	) Io.Batch.AwaitConcurrentError!void {
2262	const ev: *Evented = @ptrCast(@alignCast(userdata));
2263	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
2264	defer maybe_sync.deinit(ev);
2265	try ev.batchDrainSubmitted(&maybe_sync, batch, true);
2266	maybe_sync.leaveSync(ev);
2267	const timespec: linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = while (true) {
2268	batchDrainReady(batch) catch \|err\| switch (err) {
2269	error.Timeout => unreachable, // no timeout
2270	};
2271	if (batch.completed.head != .none or batch.pending.head == .none) return;
2272	switch (timeout) {
2273	.none => ev.yield(null, .{ .batch_await = batch }),
2274	.duration => \|duration\| {
2275	const ns = duration.raw.toNanoseconds();
2276	break .{
2277	.{
2278	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
2279	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
2280	},
2281	duration.clock,
2282	0,
2283	};
2284	},
2285	.deadline => \|deadline\| {
2286	const ns = deadline.raw.toNanoseconds();
2287	break .{
2288	.{
2289	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
2290	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
2291	},
2292	deadline.clock,
2293	linux.IORING_TIMEOUT_ABS,
2294	};
2295	},
2296	}
2297	};
2298	{
2299	const thread = try maybe_sync.cancel_region.awaitIoUring();
2300	thread.enqueue().* = .{
2301	.opcode = .TIMEOUT,
2302	.flags = 0,
2303	.ioprio = 0,
2304	.fd = 0,
2305	.off = 0,
2306	.addr = @intFromPtr(&timespec),
2307	.len = 1,
2308	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
2309	.real => linux.IORING_TIMEOUT_REALTIME,
2310	else => 0,
2311	.boot => linux.IORING_TIMEOUT_BOOTTIME,
2312	}),
2313	.user_data = @intFromPtr(&batch.userdata) \| 0b11,
2314	.buf_index = 0,
2315	.personality = 0,
2316	.splice_fd_in = 0,
2317	.addr3 = 0,
2318	.resv = 0,
2319	};
2320	}
2321	while (batch.completed.head == .none and batch.pending.head != .none) {
2322	ev.yield(null, .{ .batch_await = batch });
2323	batchDrainReady(batch) catch \|err\| switch (err) {
2324	error.Timeout => \|e\| return if (batch.completed.head == .none and
2325	batch.pending.head != .none) e,
2326	};
2327	}
2328	const thread = try maybe_sync.cancel_region.awaitIoUring();
2329	thread.enqueue().* = .{
2330	.opcode = .TIMEOUT_REMOVE,
2331	.flags = 0,
2332	.ioprio = 0,
2333	.fd = 0,
2334	.off = 0,
2335	.addr = @intFromPtr(&batch.userdata) \| 0b11,
2336	.len = 0,
2337	.rw_flags = 0,
2338	.user_data = @intFromPtr(maybe_sync.cancel_region.fiber),
2339	.buf_index = 0,
2340	.personality = 0,
2341	.splice_fd_in = 0,
2342	.addr3 = 0,
2343	.resv = 0,
2344	};
2345	ev.yield(null, .nothing);
2346	switch (maybe_sync.cancel_region.errno()) {
2347	.SUCCESS => return,
2348	.BUSY, .NOENT => {},
2349	else => \|err\| unexpectedErrno(err) catch {},
2350	}
2351	while (true) {
2352	batchDrainReady(batch) catch \|err\| switch (err) {
2353	error.Timeout => return,
2354	};
2355	ev.yield(null, .{ .batch_await = batch });
2356	}
2357	}
2358
2359	/// If `concurrency` is false, `error.ConcurrencyUnavailable` is unreachable.
2360	fn batchDrainSubmitted(
2361	ev: *Evented,
2362	maybe_sync: *CancelRegion.Sync.Maybe,
2363	batch: *Io.Batch,
2364	concurrency: bool,
2365	) (Io.ConcurrentError \|\| Io.Cancelable)!void {
2366	var index = batch.submitted.head;
2367	if (index == .none) return;
2368	const thread = try maybe_sync.cancelRegion().awaitIoUring();
2369	errdefer batch.submitted.head = index;
2370	while (index != .none) {
2371	const storage = &batch.storage[index.toIndex()];
2372	const next_index = storage.submission.node.next;
2373	if (@as(?Io.Operation.Result, result: switch (storage.submission.operation) {
2374	.file_read_streaming => \|o\| {
2375	const buffer = for (o.data) \|buffer\| {
2376	if (buffer.len > 0) break buffer;
2377	} else break :result .{ .file_read_streaming = 0 };
2378	const fd = o.file.handle;
2379	storage.* = .{ .pending = .{
2380	.node = .{ .prev = batch.pending.tail, .next = .none },
2381	.tag = .file_read_streaming,
2382	.userdata = undefined,
2383	} };
2384	thread.enqueue().* = .{
2385	.opcode = .READ,
2386	.flags = 0,
2387	.ioprio = 0,
2388	.fd = fd,
2389	.off = std.math.maxInt(u64),
2390	.addr = @intFromPtr(buffer.ptr),
2391	.len = @min(buffer.len, 0xfffff000),
2392	.rw_flags = 0,
2393	.user_data = @intFromPtr(&storage.pending.userdata) \| 0b10,
2394	.buf_index = 0,
2395	.personality = 0,
2396	.splice_fd_in = 0,
2397	.addr3 = 0,
2398	.resv = 0,
2399	};
2400	break :result null;
2401	},
2402	.file_write_streaming => \|o\| {
2403	const buffer = buffer: {
2404	if (o.header.len != 0) break :buffer o.header;
2405	for (o.data[0 .. o.data.len - 1]) \|buffer\| {
2406	if (buffer.len > 0) break :buffer buffer;
2407	}
2408	if (o.splat > 0) break :buffer o.data[o.data.len - 1];
2409	break :result .{ .file_write_streaming = 0 };
2410	};
2411	const fd = o.file.handle;
2412	storage.* = .{ .pending = .{
2413	.node = .{ .prev = batch.pending.tail, .next = .none },
2414	.tag = .file_write_streaming,
2415	.userdata = undefined,
2416	} };
2417	thread.enqueue().* = .{
2418	.opcode = .WRITE,
2419	.flags = 0,
2420	.ioprio = 0,
2421	.fd = fd,
2422	.off = std.math.maxInt(u64),
2423	.addr = @intFromPtr(buffer.ptr),
2424	.len = @min(buffer.len, 0xfffff000),
2425	.rw_flags = 0,
2426	.user_data = @intFromPtr(&storage.pending.userdata) \| 0b10,
2427	.buf_index = 0,
2428	.personality = 0,
2429	.splice_fd_in = 0,
2430	.addr3 = 0,
2431	.resv = 0,
2432	};
2433	break :result null;
2434	},
2435	.device_io_control => \|o\| if (concurrency)
2436	return error.ConcurrencyUnavailable
2437	else
2438	.{ .device_io_control = try ev.deviceIoControl(try maybe_sync.enterSync(ev), o) },
2439	})) \|result\| {
2440	switch (batch.completed.tail) {
2441	.none => batch.completed.head = index,
2442	else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next = index,
2443	}
2444	batch.completed.tail = index;
2445	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2446	} else {
2447	switch (batch.pending.tail) {
2448	.none => batch.pending.head = index,
2449	else => \|tail_index\| batch.storage[tail_index.toIndex()].pending.node.next = index,
2450	}
2451	batch.pending.tail = index;
2452	storage.pending.userdata[0] = @intFromPtr(batch);
2453	}
2454	index = next_index;
2455	}
2456	batch.submitted = .{ .head = .none, .tail = .none };
2457	}
2458
2459	fn batchDrainReady(batch: *Io.Batch) Io.Timeout.Error!void {
2460	while (@atomicRmw(?*anyopaque, &batch.userdata, .Xchg, null, .acquire)) \|head\| {
2461	var next: usize = @intFromPtr(head);
2462	var timeout = false;
2463	while (cond: switch (@as(u2, @truncate(next))) {
2464	0b00 => if (timeout) return error.Timeout else false,
2465	0b01 => {
2466	assert(!timeout);
2467	return error.Timeout;
2468	},
2469	0b10 => true,
2470	0b11 => {
2471	assert(!timeout);
2472	timeout = true;
2473	break :cond true;
2474	},
2475	}) {
2476	var operation_userdata: *Io.Operation.Storage.Pending.Userdata =
2477	@ptrFromInt(next & ~@as(usize, 0b11));
2478	next = operation_userdata[0];
2479	const completion: Completion = .{
2480	.result = @bitCast(@as(u32, @intCast(operation_userdata[1]))),
2481	.flags = @intCast(operation_userdata[2]),
2482	};
2483	const pending: *Io.Operation.Storage.Pending =
2484	@fieldParentPtr("userdata", operation_userdata);
2485	const storage: *Io.Operation.Storage = @fieldParentPtr("pending", pending);
2486	const index: Io.Operation.OptionalIndex = .fromIndex(storage - batch.storage.ptr);
2487	assert(completion.flags & linux.IORING_CQE_F_SKIP == 0);
2488	switch (pending.node.prev) {
2489	.none => batch.pending.head = pending.node.next,
2490	else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.next =
2491	pending.node.next,
2492	}
2493	switch (pending.node.next) {
2494	.none => batch.pending.tail = pending.node.prev,
2495	else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.prev =
2496	pending.node.prev,
2497	}
2498	if (@as(?Io.Operation.Result, result: switch (pending.tag) {
2499	.file_read_streaming => .{
2500	.file_read_streaming = switch (completion.errno()) {
2501	.SUCCESS => @as(u32, @bitCast(completion.result)),
2502	.INTR => 0,
2503	.CANCELED => break :result null,
2504	.INVAL => \|err\| errnoBug(err),
2505	.FAULT => \|err\| errnoBug(err),
2506	.AGAIN => error.WouldBlock,
2507	.BADF => \|err\| errnoBug(err), // File descriptor used after closed
2508	.IO => error.InputOutput,
2509	.ISDIR => error.IsDir,
2510	.NOBUFS => error.SystemResources,
2511	.NOMEM => error.SystemResources,
2512	.NOTCONN => error.SocketUnconnected,
2513	.CONNRESET => error.ConnectionResetByPeer,
2514	else => \|err\| unexpectedErrno(err),
2515	},
2516	},
2517	.file_write_streaming => .{
2518	.file_write_streaming = switch (completion.errno()) {
2519	.SUCCESS => @as(u32, @bitCast(completion.result)),
2520	.INTR => 0,
2521	.CANCELED => break :result null,
2522	.INVAL => \|err\| errnoBug(err),
2523	.FAULT => \|err\| errnoBug(err),
2524	.AGAIN => error.WouldBlock,
2525	.BADF => error.NotOpenForWriting, // Can be a race condition.
2526	.DESTADDRREQ => \|err\| errnoBug(err), // `connect` was never called.
2527	.DQUOT => error.DiskQuota,
2528	.FBIG => error.FileTooBig,
2529	.IO => error.InputOutput,
2530	.NOSPC => error.NoSpaceLeft,
2531	.PERM => error.PermissionDenied,
2532	.PIPE => error.BrokenPipe,
2533	.CONNRESET => \|err\| errnoBug(err), // Not a socket handle.
2534	.BUSY => error.DeviceBusy,
2535	else => \|err\| unexpectedErrno(err),
2536	},
2537	},
2538	.device_io_control => unreachable,
2539	})) \|result\| {
2540	switch (batch.completed.tail) {
2541	.none => batch.completed.head = index,
2542	else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next =
2543	index,
2544	}
2545	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2546	batch.completed.tail = index;
2547	} else {
2548	switch (batch.unused.tail) {
2549	.none => batch.unused.head = index,
2550	else => \|tail_index\| batch.storage[tail_index.toIndex()].unused.next = index,
2551	}
2552	storage.* = .{ .unused = .{ .prev = batch.unused.tail, .next = .none } };
2553	batch.unused.tail = index;
2554	}
2555	}
2556	}
2557	}
2558
2559	fn batchCancel(userdata: ?anyopaque, batch: Io.Batch) void {
2560	const ev: *Evented = @ptrCast(@alignCast(userdata));
2561	_ = ev;
2562	batchDrainReady(batch) catch \|err\| switch (err) {
2563	error.Timeout => unreachable, // no timeout
2564	};
2565	var index = batch.pending.head;
2566	if (index == .none) return;
2567	var cancel_region: CancelRegion = .initBlocked();
2568	defer cancel_region.deinit();
2569	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
2570	error.Canceled => unreachable, // blocked
2571	};
2572	while (index != .none) {
2573	const pending = &batch.storage[index.toIndex()].pending;
2574	thread.enqueue().* = .{
2575	.opcode = .ASYNC_CANCEL,
2576	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
2577	.ioprio = 0,
2578	.fd = 0,
2579	.off = 0,
2580	.addr = @intFromPtr(&pending.userdata) \| 0b10,
2581	.len = 0,
2582	.rw_flags = 0,
2583	.user_data = @intFromEnum(Completion.Userdata.wakeup),
2584	.buf_index = 0,
2585	.personality = 0,
2586	.splice_fd_in = 0,
2587	.addr3 = 0,
2588	.resv = 0,
2589	};
2590	index = pending.node.next;
2591	}
2592	while (batch.pending.head != .none) batchDrainReady(batch) catch \|err\| switch (err) {
2593	error.Timeout => unreachable, // no timeout
2594	};
2595	}
2596
2597	fn dirCreateDir(
2598	userdata: ?*anyopaque,
2599	dir: Dir,
2600	sub_path: []const u8,
2601	permissions: Dir.Permissions,
2602	) Dir.CreateDirError!void {
2603	const ev: *Evented = @ptrCast(@alignCast(userdata));
2604
2605	var path_buffer: [PATH_MAX]u8 = undefined;
2606	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2607
2608	var cancel_region: CancelRegion = .init();
2609	defer cancel_region.deinit();
2610	while (true) {
2611	const thread = try cancel_region.awaitIoUring();
2612	thread.enqueue().* = .{
2613	.opcode = .MKDIRAT,
2614	.flags = 0,
2615	.ioprio = 0,
2616	.fd = dir.handle,
2617	.off = 0,
2618	.addr = @intFromPtr(sub_path_posix.ptr),
2619	.len = permissions.toMode(),
2620	.rw_flags = 0,
2621	.user_data = @intFromPtr(cancel_region.fiber),
2622	.buf_index = 0,
2623	.personality = 0,
2624	.splice_fd_in = 0,
2625	.addr3 = 0,
2626	.resv = 0,
2627	};
2628	ev.yield(null, .nothing);
2629	switch (cancel_region.errno()) {
2630	.SUCCESS => return,
2631	.INTR, .CANCELED => continue,
2632	.ACCES => return error.AccessDenied,
2633	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
2634	.PERM => return error.PermissionDenied,
2635	.DQUOT => return error.DiskQuota,
2636	.EXIST => return error.PathAlreadyExists,
2637	.FAULT => \|err\| return errnoBug(err),
2638	.LOOP => return error.SymLinkLoop,
2639	.MLINK => return error.LinkQuotaExceeded,
2640	.NAMETOOLONG => return error.NameTooLong,
2641	.NOENT => return error.FileNotFound,
2642	.NOMEM => return error.SystemResources,
2643	.NOSPC => return error.NoSpaceLeft,
2644	.NOTDIR => return error.NotDir,
2645	.ROFS => return error.ReadOnlyFileSystem,
2646	.ILSEQ => return error.BadPathName,
2647	else => \|err\| return unexpectedErrno(err),
2648	}
2649	}
2650	}
2651
2652	fn dirCreateDirPath(
2653	userdata: ?*anyopaque,
2654	dir: Dir,
2655	sub_path: []const u8,
2656	permissions: Dir.Permissions,
2657	) Dir.CreateDirPathError!Dir.CreatePathStatus {
2658	const ev: *Evented = @ptrCast(@alignCast(userdata));
2659
2660	var it = Dir.path.componentIterator(sub_path);
2661	var status: Dir.CreatePathStatus = .existed;
2662	var component = it.last() orelse return error.BadPathName;
2663	while (true) {
2664	if (dirCreateDir(ev, dir, component.path, permissions)) \|_\| {
2665	status = .created;
2666	} else \|err\| switch (err) {
2667	error.PathAlreadyExists => {
2668	// stat the file and return an error if it's not a directory
2669	// this is important because otherwise a dangling symlink
2670	// could cause an infinite loop
2671	const kind = try ev.filePathKind(dir, component.path);
2672	if (kind != .directory) return error.NotDir;
2673	},
2674	error.FileNotFound => \|e\| {
2675	component = it.previous() orelse return e;
2676	continue;
2677	},
2678	else => \|e\| return e,
2679	}
2680	component = it.next() orelse return status;
2681	}
2682	}
2683
2684	fn filePathKind(ev: *Evented, dir: Dir, sub_path: []const u8) !File.Kind {
2685	var path_buffer: [PATH_MAX]u8 = undefined;
2686	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2687	var cancel_region: CancelRegion = .init();
2688	defer cancel_region.deinit();
2689	while (true) {
2690	var statx_buf = std.mem.zeroes(linux.Statx);
2691	const thread = try cancel_region.awaitIoUring();
2692	thread.enqueue().* = .{
2693	.opcode = .STATX,
2694	.flags = 0,
2695	.ioprio = 0,
2696	.fd = dir.handle,
2697	.off = @intFromPtr(&statx_buf),
2698	.addr = @intFromPtr(sub_path_posix.ptr),
2699	.len = @bitCast(linux.STATX{ .TYPE = true }),
2700	.rw_flags = linux.AT.NO_AUTOMOUNT \| linux.AT.SYMLINK_NOFOLLOW,
2701	.user_data = @intFromPtr(cancel_region.fiber),
2702	.buf_index = 0,
2703	.personality = 0,
2704	.splice_fd_in = 0,
2705	.addr3 = 0,
2706	.resv = 0,
2707	};
2708	ev.yield(null, .nothing);
2709	switch (cancel_region.errno()) {
2710	.SUCCESS => {
2711	if (!statx_buf.mask.TYPE) return error.Unexpected;
2712	return statxKind(statx_buf.mode);
2713	},
2714	.INTR, .CANCELED => continue,
2715	.ACCES => \|err\| return errnoBug(err),
2716	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
2717	.FAULT => \|err\| return errnoBug(err),
2718	.INVAL => \|err\| return errnoBug(err),
2719	.LOOP => \|err\| return errnoBug(err),
2720	.NAMETOOLONG => \|err\| return errnoBug(err),
2721	.NOENT => \|err\| return errnoBug(err),
2722	.NOMEM => return error.SystemResources,
2723	.NOTDIR => \|err\| return errnoBug(err),
2724	else => \|err\| return unexpectedErrno(err),
2725	}
2726	}
2727	}
2728
2729	fn dirCreateDirPathOpen(
2730	userdata: ?*anyopaque,
2731	dir: Dir,
2732	sub_path: []const u8,
2733	permissions: Dir.Permissions,
2734	options: Dir.OpenOptions,
2735	) Dir.CreateDirPathOpenError!Dir {
2736	const ev: *Evented = @ptrCast(@alignCast(userdata));
2737	return dirOpenDir(ev, dir, sub_path, options) catch \|err\| switch (err) {
2738	error.FileNotFound => {
2739	_ = try dirCreateDirPath(ev, dir, sub_path, permissions);
2740	return dirOpenDir(ev, dir, sub_path, options);
2741	},
2742	else => \|e\| return e,
2743	};
2744	}
2745
2746	fn dirOpenDir(
2747	userdata: ?*anyopaque,
2748	dir: Dir,
2749	sub_path: []const u8,
2750	options: Dir.OpenOptions,
2751	) Dir.OpenError!Dir {
2752	const ev: *Evented = @ptrCast(@alignCast(userdata));
2753
2754	var path_buffer: [PATH_MAX]u8 = undefined;
2755	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2756
2757	var cancel_region: CancelRegion = .init();
2758	defer cancel_region.deinit();
2759	return .{
2760	.handle = ev.openat(&cancel_region, dir.handle, sub_path_posix, .{
2761	.ACCMODE = .RDONLY,
2762	.DIRECTORY = true,
2763	.NOFOLLOW = !options.follow_symlinks,
2764	.CLOEXEC = true,
2765	.PATH = !options.iterate,
2766	}, 0) catch \|err\| switch (err) {
2767	error.IsDir => return errnoBug(.ISDIR),
2768	error.WouldBlock => return errnoBug(.AGAIN),
2769	error.FileTooBig => return errnoBug(.FBIG),
2770	error.NoSpaceLeft => return errnoBug(.NOSPC),
2771	error.DeviceBusy => return errnoBug(.BUSY), // O_EXCL not passed
2772	error.FileBusy => return errnoBug(.TXTBSY),
2773	error.PathAlreadyExists => return errnoBug(.EXIST), // Not creating.
2774	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
2775	error.AntivirusInterference => unreachable, // Windows-only
2776	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
2777	else => \|e\| return e,
2778	},
2779	};
2780	}
2781
2782	fn dirStat(userdata: ?*anyopaque, dir: Dir) Dir.StatError!Dir.Stat {
2783	const ev: *Evented = @ptrCast(@alignCast(userdata));
2784	var cancel_region: CancelRegion = .init();
2785	defer cancel_region.deinit();
2786	return ev.stat(&cancel_region, dir.handle);
2787	}
2788
2789	fn dirStatFile(
2790	userdata: ?*anyopaque,
2791	dir: Dir,
2792	sub_path: []const u8,
2793	options: Dir.StatFileOptions,
2794	) Dir.StatFileError!File.Stat {
2795	const ev: *Evented = @ptrCast(@alignCast(userdata));
2796	var path_buffer: [PATH_MAX]u8 = undefined;
2797	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2798	var cancel_region: CancelRegion = .init();
2799	defer cancel_region.deinit();
2800	return ev.statx(&cancel_region, dir.handle, sub_path_posix, linux.AT.NO_AUTOMOUNT \|
2801	@as(u32, if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW));
2802	}
2803
2804	fn dirAccess(
2805	userdata: ?*anyopaque,
2806	dir: Dir,
2807	sub_path: []const u8,
2808	options: Dir.AccessOptions,
2809	) Dir.AccessError!void {
2810	const ev: *Evented = @ptrCast(@alignCast(userdata));
2811
2812	var path_buffer: [PATH_MAX]u8 = undefined;
2813	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2814
2815	const mode: u32 =
2816	@as(u32, if (options.read) linux.R_OK else 0) \|
2817	@as(u32, if (options.write) linux.W_OK else 0) \|
2818	@as(u32, if (options.execute) linux.X_OK else 0);
2819	const flags: u32 = if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW;
2820
2821	var sync: CancelRegion.Sync = try .init(ev);
2822	defer sync.deinit(ev);
2823	while (true) {
2824	try sync.cancel_region.await(.nothing);
2825	switch (linux.errno(linux.faccessat(dir.handle, sub_path_posix, mode, flags))) {
2826	.SUCCESS => return,
2827	.INTR => continue,
2828	.ACCES => return error.AccessDenied,
2829	.PERM => return error.PermissionDenied,
2830	.ROFS => return error.ReadOnlyFileSystem,
2831	.LOOP => return error.SymLinkLoop,
2832	.TXTBSY => return error.FileBusy,
2833	.NOTDIR => return error.FileNotFound,
2834	.NOENT => return error.FileNotFound,
2835	.NAMETOOLONG => return error.NameTooLong,
2836	.INVAL => \|err\| return errnoBug(err),
2837	.FAULT => \|err\| return errnoBug(err),
2838	.IO => return error.InputOutput,
2839	.NOMEM => return error.SystemResources,
2840	.ILSEQ => return error.BadPathName,
2841	else => \|err\| return unexpectedErrno(err),
2842	}
2843	}
2844	}
2845
2846	fn dirCreateFile(
2847	userdata: ?*anyopaque,
2848	dir: Dir,
2849	sub_path: []const u8,
2850	flags: File.CreateFlags,
2851	) File.OpenError!File {
2852	const ev: *Evented = @ptrCast(@alignCast(userdata));
2853
2854	var path_buffer: [PATH_MAX]u8 = undefined;
2855	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2856
2857	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
2858	defer maybe_sync.deinit(ev);
2859	const fd = try ev.openat(&maybe_sync.cancel_region, dir.handle, sub_path_posix, .{
2860	.ACCMODE = if (flags.read) .RDWR else .WRONLY,
2861	.CREAT = true,
2862	.TRUNC = flags.truncate,
2863	.EXCL = flags.exclusive,
2864	.CLOEXEC = true,
2865	}, flags.permissions.toMode());
2866	errdefer ev.close(fd);
2867
2868	switch (flags.lock) {
2869	.none => {},
2870	.shared, .exclusive => try ev.flock(
2871	try maybe_sync.enterSync(ev),
2872	fd,
2873	flags.lock,
2874	if (flags.lock_nonblocking) .nonblocking else .blocking,
2875	),
2876	}
2877
2878	return .{ .handle = fd, .flags = .{ .nonblocking = false } };
2879	}
2880
2881	fn dirCreateFileAtomic(
2882	userdata: ?*anyopaque,
2883	dir: Dir,
2884	dest_path: []const u8,
2885	options: Dir.CreateFileAtomicOptions,
2886	) Dir.CreateFileAtomicError!File.Atomic {
2887	const ev: *Evented = @ptrCast(@alignCast(userdata));
2888	// Linux has O_TMPFILE, but linkat() does not support AT_REPLACE, so it's
2889	// useless when we have to make up a bogus path name to do the rename()
2890	// anyway.
2891	if (!options.replace) tmpfile: {
2892	const flags: linux.O = if (@hasField(linux.O, "TMPFILE")) .{
2893	.ACCMODE = .RDWR,
2894	.TMPFILE = true,
2895	.DIRECTORY = true,
2896	.CLOEXEC = true,
2897	} else if (@hasField(linux.O, "TMPFILE0") and !@hasField(linux.O, "TMPFILE2")) .{
2898	.ACCMODE = .RDWR,
2899	.TMPFILE0 = true,
2900	.TMPFILE1 = true,
2901	.DIRECTORY = true,
2902	.CLOEXEC = true,
2903	} else break :tmpfile;
2904
2905	const dest_dirname = Dir.path.dirname(dest_path);
2906	if (dest_dirname) \|dirname\| {
2907	// This has a nice side effect of preemptively triggering EISDIR or
2908	// ENOENT, avoiding the ambiguity below.
2909	_ = dirCreateDirPath(ev, dir, dirname, .default_dir) catch \|err\| switch (err) {
2910	// None of these make sense in this context.
2911	error.IsDir,
2912	error.Streaming,
2913	error.DiskQuota,
2914	error.PathAlreadyExists,
2915	error.LinkQuotaExceeded,
2916	error.PipeBusy,
2917	error.FileTooBig,
2918	error.DeviceBusy,
2919	error.FileLocksUnsupported,
2920	error.FileBusy,
2921	=> return error.Unexpected,
2922
2923	else => \|e\| return e,
2924	};
2925	}
2926
2927	var path_buffer: [PATH_MAX]u8 = undefined;
2928	const sub_path_posix = try pathToPosix(dest_dirname orelse ".", &path_buffer);
2929
2930	var cancel_region: CancelRegion = .init();
2931	defer cancel_region.deinit();
2932	return .{
2933	.file = .{
2934	.handle = ev.openat(
2935	&cancel_region,
2936	dir.handle,
2937	sub_path_posix,
2938	flags,
2939	options.permissions.toMode(),
2940	) catch \|err\| switch (err) {
2941	error.IsDir, error.FileNotFound => {
2942	// Ambiguous error code. It might mean the file system
2943	// does not support O_TMPFILE. Therefore, we must fall
2944	// back to not using O_TMPFILE.
2945	break :tmpfile;
2946	},
2947	error.FileTooBig => return errnoBug(.FBIG),
2948	error.DeviceBusy => return errnoBug(.BUSY), // O_EXCL not passed
2949	error.PathAlreadyExists => return errnoBug(.EXIST), // Not creating.
2950	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
2951	error.AntivirusInterference => unreachable, // Windows-only
2952	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
2953	else => \|e\| return e,
2954	},
2955	.flags = .{ .nonblocking = false },
2956	},
2957	.file_basename_hex = 0,
2958	.dest_sub_path = dest_path,
2959	.file_open = true,
2960	.file_exists = false,
2961	.close_dir_on_deinit = false,
2962	.dir = dir,
2963	};
2964	}
2965
2966	if (Dir.path.dirname(dest_path)) \|dirname\| {
2967	const new_dir = if (options.make_path)
2968	dirCreateDirPathOpen(ev, dir, dirname, .default_dir, .{}) catch \|err\| switch (err) {
2969	// None of these make sense in this context.
2970	error.IsDir,
2971	error.Streaming,
2972	error.DiskQuota,
2973	error.PathAlreadyExists,
2974	error.LinkQuotaExceeded,
2975	error.PipeBusy,
2976	error.FileTooBig,
2977	error.FileLocksUnsupported,
2978	error.DeviceBusy,
2979	=> return error.Unexpected,
2980
2981	else => \|e\| return e,
2982	}
2983	else
2984	try dirOpenDir(ev, dir, dirname, .{});
2985
2986	return ev.atomicFileInit(Dir.path.basename(dest_path), options.permissions, new_dir, true);
2987	}
2988
2989	return ev.atomicFileInit(dest_path, options.permissions, dir, false);
2990	}
2991
2992	fn atomicFileInit(
2993	ev: *Evented,
2994	dest_basename: []const u8,
2995	permissions: File.Permissions,
2996	dir: Dir,
2997	close_dir_on_deinit: bool,
2998	) Dir.CreateFileAtomicError!File.Atomic {
2999	while (true) {
3000	var random_integer: u64 = undefined;
3001	random(ev, @ptrCast(&random_integer));
3002	const tmp_sub_path = std.fmt.hex(random_integer);
3003	const file = dirCreateFile(ev, dir, &tmp_sub_path, .{
3004	.permissions = permissions,
3005	.exclusive = true,
3006	}) catch \|err\| switch (err) {
3007	error.PathAlreadyExists => continue,
3008	error.DeviceBusy => continue,
3009	error.FileBusy => continue,
3010
3011	error.IsDir => return error.Unexpected, // No path components.
3012	error.FileTooBig => return error.Unexpected, // Creating, not opening.
3013	error.FileLocksUnsupported => return error.Unexpected, // Not asking for locks.
3014	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
3015
3016	else => \|e\| return e,
3017	};
3018	return .{
3019	.file = file,
3020	.file_basename_hex = random_integer,
3021	.dest_sub_path = dest_basename,
3022	.file_open = true,
3023	.file_exists = true,
3024	.close_dir_on_deinit = close_dir_on_deinit,
3025	.dir = dir,
3026	};
3027	}
3028	}
3029
3030	fn dirOpenFile(
3031	userdata: ?*anyopaque,
3032	dir: Dir,
3033	sub_path: []const u8,
3034	flags: File.OpenFlags,
3035	) File.OpenError!File {
3036	const ev: *Evented = @ptrCast(@alignCast(userdata));
3037
3038	var path_buffer: [PATH_MAX]u8 = undefined;
3039	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3040
3041	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
3042	defer maybe_sync.deinit(ev);
3043	const fd = try ev.openat(&maybe_sync.cancel_region, dir.handle, sub_path_posix, .{
3044	.ACCMODE = switch (flags.mode) {
3045	.read_only => .RDONLY,
3046	.write_only => .WRONLY,
3047	.read_write => .RDWR,
3048	},
3049	.NOCTTY = !flags.allow_ctty,
3050	.NOFOLLOW = !flags.follow_symlinks,
3051	.CLOEXEC = true,
3052	.PATH = flags.path_only,
3053	}, 0);
3054	errdefer ev.close(fd);
3055
3056	if (!flags.allow_directory) {
3057	const is_dir = is_dir: {
3058	const s = ev.stat(&maybe_sync.cancel_region, fd) catch \|err\| switch (err) {
3059	// The directory-ness is either unknown or unknowable
3060	error.Streaming => break :is_dir false,
3061	else => \|e\| return e,
3062	};
3063	break :is_dir s.kind == .directory;
3064	};
3065	if (is_dir) return error.IsDir;
3066	}
3067
3068	switch (flags.lock) {
3069	.none => {},
3070	.shared, .exclusive => try ev.flock(
3071	try maybe_sync.enterSync(ev),
3072	fd,
3073	flags.lock,
3074	if (flags.lock_nonblocking) .nonblocking else .blocking,
3075	),
3076	}
3077
3078	return .{ .handle = fd, .flags = .{ .nonblocking = false } };
3079	}
3080
3081	fn dirClose(userdata: ?*anyopaque, dirs: []const Dir) void {
3082	const ev: *Evented = @ptrCast(@alignCast(userdata));
3083	for (dirs) \|dir\| ev.close(dir.handle);
3084	}
3085
3086	fn dirRead(userdata: ?anyopaque, dr: Dir.Reader, buffer: []Dir.Entry) Dir.Reader.Error!usize {
3087	const ev: *Evented = @ptrCast(@alignCast(userdata));
3088	var buffer_index: usize = 0;
3089	while (buffer.len - buffer_index != 0) {
3090	if (dr.end - dr.index == 0) {
3091	// Refill the buffer, unless we've already created references to
3092	// buffered data.
3093	if (buffer_index != 0) break;
3094	var sync: CancelRegion.Sync = try .init(ev);
3095	defer sync.deinit(ev);
3096	if (dr.state == .reset) {
3097	ev.lseek(&sync, dr.dir.handle, 0, linux.SEEK.SET) catch \|err\| switch (err) {
3098	error.Unseekable => return error.Unexpected,
3099	else => \|e\| return e,
3100	};
3101	dr.state = .reading;
3102	}
3103	const n = while (true) {
3104	try sync.cancel_region.await(.nothing);
3105	const rc = linux.getdents64(dr.dir.handle, dr.buffer.ptr, dr.buffer.len);
3106	switch (linux.errno(rc)) {
3107	.SUCCESS => break rc,
3108	.INTR => continue,
3109	.BADF => \|err\| return errnoBug(err), // Dir is invalid or was opened without iteration ability.
3110	.FAULT => \|err\| return errnoBug(err),
3111	.NOTDIR => \|err\| return errnoBug(err),
3112	// To be consistent across platforms, iteration
3113	// ends if the directory being iterated is deleted
3114	// during iteration. This matches the behavior of
3115	// non-Linux, non-WASI UNIX platforms.
3116	.NOENT => {
3117	dr.state = .finished;
3118	return 0;
3119	},
3120	// This can occur when reading /proc/$PID/net, or
3121	// if the provided buffer is too small. Neither
3122	// scenario is intended to be handled by this API.
3123	.INVAL => return error.Unexpected,
3124	.ACCES => return error.AccessDenied, // Lacking permission to iterate this directory.
3125	else => \|err\| return unexpectedErrno(err),
3126	}
3127	};
3128	if (n == 0) {
3129	dr.state = .finished;
3130	return 0;
3131	}
3132	dr.index = 0;
3133	dr.end = n;
3134	}
3135	// Linux aligns the header by padding after the null byte of the name
3136	// to align the next entry. This means we can find the end of the name
3137	// by looking at only the 8 bytes before the next record. However since
3138	// file names are usually short it's better to keep the machine code
3139	// simpler.
3140	//
3141	// Furthermore, I observed qemu user mode to not align this struct, so
3142	// this code makes the conservative choice to not assume alignment.
3143	const linux_entry: *align(1) linux.dirent64 = @ptrCast(&dr.buffer[dr.index]);
3144	const next_index = dr.index + linux_entry.reclen;
3145	dr.index = next_index;
3146	const name_ptr: [*]u8 = &linux_entry.name;
3147	const padded_name = name_ptr[0 .. linux_entry.reclen - @offsetOf(linux.dirent64, "name")];
3148	const name_len = std.mem.findScalar(u8, padded_name, 0).?;
3149	const name = name_ptr[0..name_len :0];
3150
3151	if (std.mem.eql(u8, name, ".") or std.mem.eql(u8, name, "..")) continue;
3152
3153	const entry_kind: File.Kind = switch (linux_entry.type) {
3154	linux.DT.BLK => .block_device,
3155	linux.DT.CHR => .character_device,
3156	linux.DT.DIR => .directory,
3157	linux.DT.FIFO => .named_pipe,
3158	linux.DT.LNK => .sym_link,
3159	linux.DT.REG => .file,
3160	linux.DT.SOCK => .unix_domain_socket,
3161	else => .unknown,
3162	};
3163	buffer[buffer_index] = .{
3164	.name = name,
3165	.kind = entry_kind,
3166	.inode = linux_entry.ino,
3167	};
3168	buffer_index += 1;
3169	}
3170	return buffer_index;
3171	}
3172
3173	fn dirRealPath(userdata: ?*anyopaque, dir: Dir, out_buffer: []u8) Dir.RealPathError!usize {
3174	const ev: *Evented = @ptrCast(@alignCast(userdata));
3175	var sync: CancelRegion.Sync = try .init(ev);
3176	defer sync.deinit(ev);
3177	return ev.realPath(&sync, dir.handle, out_buffer);
3178	}
3179
3180	fn dirRealPathFile(
3181	userdata: ?*anyopaque,
3182	dir: Dir,
3183	sub_path: []const u8,
3184	out_buffer: []u8,
3185	) Dir.RealPathFileError!usize {
3186	const ev: *Evented = @ptrCast(@alignCast(userdata));
3187
3188	var path_buffer: [PATH_MAX]u8 = undefined;
3189	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3190
3191	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
3192	defer maybe_sync.deinit(ev);
3193	const fd = ev.openat(&maybe_sync.cancel_region, dir.handle, sub_path_posix, .{
3194	.CLOEXEC = true,
3195	.PATH = true,
3196	}, 0) catch \|err\| switch (err) {
3197	error.WouldBlock => return errnoBug(.AGAIN),
3198	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
3199	else => \|e\| return e,
3200	};
3201	defer ev.close(fd);
3202	return ev.realPath(try maybe_sync.enterSync(ev), fd, out_buffer);
3203	}
3204
3205	fn dirDeleteFile(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteFileError!void {
3206	const ev: *Evented = @ptrCast(@alignCast(userdata));
3207
3208	var path_buffer: [PATH_MAX]u8 = undefined;
3209	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3210
3211	var cancel_region: CancelRegion = .init();
3212	defer cancel_region.deinit();
3213	while (true) {
3214	const thread = try cancel_region.awaitIoUring();
3215	thread.enqueue().* = .{
3216	.opcode = .UNLINKAT,
3217	.flags = 0,
3218	.ioprio = 0,
3219	.fd = dir.handle,
3220	.off = 0,
3221	.addr = @intFromPtr(sub_path_posix.ptr),
3222	.len = 0,
3223	.rw_flags = 0,
3224	.user_data = @intFromPtr(cancel_region.fiber),
3225	.buf_index = 0,
3226	.personality = 0,
3227	.splice_fd_in = 0,
3228	.addr3 = 0,
3229	.resv = 0,
3230	};
3231	ev.yield(null, .nothing);
3232	switch (cancel_region.errno()) {
3233	.SUCCESS => return,
3234	.INTR, .CANCELED => continue,
3235	.PERM => return error.PermissionDenied,
3236	.ACCES => return error.AccessDenied,
3237	.BUSY => return error.FileBusy,
3238	.FAULT => \|err\| return errnoBug(err),
3239	.IO => return error.FileSystem,
3240	.ISDIR => return error.IsDir,
3241	.LOOP => return error.SymLinkLoop,
3242	.NAMETOOLONG => return error.NameTooLong,
3243	.NOENT => return error.FileNotFound,
3244	.NOTDIR => return error.NotDir,
3245	.NOMEM => return error.SystemResources,
3246	.ROFS => return error.ReadOnlyFileSystem,
3247	.EXIST => \|err\| return errnoBug(err),
3248	.NOTEMPTY => \|err\| return errnoBug(err), // Not passing AT.REMOVEDIR
3249	.ILSEQ => return error.BadPathName,
3250	.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
3251	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
3252	else => \|err\| return unexpectedErrno(err),
3253	}
3254	}
3255	}
3256
3257	fn dirDeleteDir(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteDirError!void {
3258	const ev: *Evented = @ptrCast(@alignCast(userdata));
3259
3260	var path_buffer: [PATH_MAX]u8 = undefined;
3261	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3262
3263	var cancel_region: CancelRegion = .init();
3264	defer cancel_region.deinit();
3265	while (true) {
3266	const thread = try cancel_region.awaitIoUring();
3267	thread.enqueue().* = .{
3268	.opcode = .UNLINKAT,
3269	.flags = 0,
3270	.ioprio = 0,
3271	.fd = dir.handle,
3272	.off = 0,
3273	.addr = @intFromPtr(sub_path_posix.ptr),
3274	.len = 0,
3275	.rw_flags = linux.AT.REMOVEDIR,
3276	.user_data = @intFromPtr(cancel_region.fiber),
3277	.buf_index = 0,
3278	.personality = 0,
3279	.splice_fd_in = 0,
3280	.addr3 = 0,
3281	.resv = 0,
3282	};
3283	ev.yield(null, .nothing);
3284	switch (cancel_region.errno()) {
3285	.SUCCESS => return,
3286	.INTR, .CANCELED => continue,
3287	.ACCES => return error.AccessDenied,
3288	.PERM => return error.PermissionDenied,
3289	.BUSY => return error.FileBusy,
3290	.FAULT => \|err\| return errnoBug(err),
3291	.IO => return error.FileSystem,
3292	.ISDIR => \|err\| return errnoBug(err),
3293	.LOOP => return error.SymLinkLoop,
3294	.NAMETOOLONG => return error.NameTooLong,
3295	.NOENT => return error.FileNotFound,
3296	.NOTDIR => return error.NotDir,
3297	.NOMEM => return error.SystemResources,
3298	.ROFS => return error.ReadOnlyFileSystem,
3299	.EXIST => \|err\| return errnoBug(err),
3300	.NOTEMPTY => return error.DirNotEmpty,
3301	.ILSEQ => return error.BadPathName,
3302	.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
3303	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
3304	else => \|err\| return unexpectedErrno(err),
3305	}
3306	}
3307	}
3308
3309	fn dirRename(
3310	userdata: ?*anyopaque,
3311	old_dir: Dir,
3312	old_sub_path: []const u8,
3313	new_dir: Dir,
3314	new_sub_path: []const u8,
3315	) Dir.RenameError!void {
3316	const ev: *Evented = @ptrCast(@alignCast(userdata));
3317
3318	var old_path_buffer: [PATH_MAX]u8 = undefined;
3319	var new_path_buffer: [PATH_MAX]u8 = undefined;
3320
3321	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
3322	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
3323
3324	var cancel_region: CancelRegion = .init();
3325	defer cancel_region.deinit();
3326	return ev.renameat(
3327	&cancel_region,
3328	old_dir.handle,
3329	old_sub_path_posix,
3330	new_dir.handle,
3331	new_sub_path_posix,
3332	.{},
3333	);
3334	}
3335
3336	fn dirRenamePreserve(
3337	userdata: ?*anyopaque,
3338	old_dir: Dir,
3339	old_sub_path: []const u8,
3340	new_dir: Dir,
3341	new_sub_path: []const u8,
3342	) Dir.RenamePreserveError!void {
3343	const ev: *Evented = @ptrCast(@alignCast(userdata));
3344
3345	var old_path_buffer: [PATH_MAX]u8 = undefined;
3346	var new_path_buffer: [PATH_MAX]u8 = undefined;
3347
3348	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
3349	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
3350
3351	var cancel_region: CancelRegion = .init();
3352	defer cancel_region.deinit();
3353	return ev.renameat(
3354	&cancel_region,
3355	old_dir.handle,
3356	old_sub_path_posix,
3357	new_dir.handle,
3358	new_sub_path_posix,
3359	.{ .NOREPLACE = true },
3360	);
3361	}
3362
3363	fn dirSymLink(
3364	userdata: ?*anyopaque,
3365	dir: Dir,
3366	target_path: []const u8,
3367	sym_link_path: []const u8,
3368	flags: Dir.SymLinkFlags,
3369	) Dir.SymLinkError!void {
3370	const ev: *Evented = @ptrCast(@alignCast(userdata));
3371	_ = flags;
3372
3373	var target_path_buffer: [PATH_MAX]u8 = undefined;
3374	var sym_link_path_buffer: [PATH_MAX]u8 = undefined;
3375
3376	const target_path_posix = try pathToPosix(target_path, &target_path_buffer);
3377	const sym_link_path_posix = try pathToPosix(sym_link_path, &sym_link_path_buffer);
3378
3379	var cancel_region: CancelRegion = .init();
3380	defer cancel_region.deinit();
3381	while (true) {
3382	const thread = try cancel_region.awaitIoUring();
3383	thread.enqueue().* = .{
3384	.opcode = .SYMLINKAT,
3385	.flags = 0,
3386	.ioprio = 0,
3387	.fd = dir.handle,
3388	.off = @intFromPtr(sym_link_path_posix.ptr),
3389	.addr = @intFromPtr(target_path_posix.ptr),
3390	.len = 0,
3391	.rw_flags = 0,
3392	.user_data = @intFromPtr(cancel_region.fiber),
3393	.buf_index = 0,
3394	.personality = 0,
3395	.splice_fd_in = 0,
3396	.addr3 = 0,
3397	.resv = 0,
3398	};
3399	ev.yield(null, .nothing);
3400	switch (cancel_region.errno()) {
3401	.SUCCESS => return,
3402	.INTR, .CANCELED => continue,
3403	.FAULT => \|err\| return errnoBug(err),
3404	.INVAL => \|err\| return errnoBug(err),
3405	.ACCES => return error.AccessDenied,
3406	.PERM => return error.PermissionDenied,
3407	.DQUOT => return error.DiskQuota,
3408	.EXIST => return error.PathAlreadyExists,
3409	.IO => return error.FileSystem,
3410	.LOOP => return error.SymLinkLoop,
3411	.NAMETOOLONG => return error.NameTooLong,
3412	.NOENT => return error.FileNotFound,
3413	.NOTDIR => return error.NotDir,
3414	.NOMEM => return error.SystemResources,
3415	.NOSPC => return error.NoSpaceLeft,
3416	.ROFS => return error.ReadOnlyFileSystem,
3417	.ILSEQ => return error.BadPathName,
3418	else => \|err\| return unexpectedErrno(err),
3419	}
3420	}
3421	}
3422
3423	fn dirReadLink(
3424	userdata: ?*anyopaque,
3425	dir: Dir,
3426	sub_path: []const u8,
3427	buffer: []u8,
3428	) Dir.ReadLinkError!usize {
3429	const ev: *Evented = @ptrCast(@alignCast(userdata));
3430
3431	var sub_path_buffer: [PATH_MAX]u8 = undefined;
3432	const sub_path_posix = try pathToPosix(sub_path, &sub_path_buffer);
3433
3434	var sync: CancelRegion.Sync = try .init(ev);
3435	defer sync.deinit(ev);
3436	while (true) {
3437	try sync.cancel_region.await(.nothing);
3438	const rc = linux.readlinkat(dir.handle, sub_path_posix, buffer.ptr, buffer.len);
3439	switch (linux.errno(rc)) {
3440	.SUCCESS => return @bitCast(rc),
3441	.INTR => continue,
3442	.ACCES => return error.AccessDenied,
3443	.FAULT => \|err\| return errnoBug(err),
3444	.INVAL => return error.NotLink,
3445	.IO => return error.FileSystem,
3446	.LOOP => return error.SymLinkLoop,
3447	.NAMETOOLONG => return error.NameTooLong,
3448	.NOENT => return error.FileNotFound,
3449	.NOMEM => return error.SystemResources,
3450	.NOTDIR => return error.NotDir,
3451	.ILSEQ => return error.BadPathName,
3452	else => \|err\| return unexpectedErrno(err),
3453	}
3454	}
3455	}
3456
3457	fn dirSetOwner(
3458	userdata: ?*anyopaque,
3459	dir: Dir,
3460	owner: ?File.Uid,
3461	group: ?File.Gid,
3462	) Dir.SetOwnerError!void {
3463	const ev: *Evented = @ptrCast(@alignCast(userdata));
3464	var sync: CancelRegion.Sync = try .init(ev);
3465	defer sync.deinit(ev);
3466	try ev.fchownat(
3467	&sync,
3468	dir.handle,
3469	"",
3470	owner orelse std.math.maxInt(linux.uid_t),
3471	group orelse std.math.maxInt(linux.gid_t),
3472	linux.AT.EMPTY_PATH,
3473	);
3474	}
3475
3476	fn dirSetFileOwner(
3477	userdata: ?*anyopaque,
3478	dir: Dir,
3479	sub_path: []const u8,
3480	owner: ?File.Uid,
3481	group: ?File.Gid,
3482	options: Dir.SetFileOwnerOptions,
3483	) Dir.SetFileOwnerError!void {
3484	const ev: *Evented = @ptrCast(@alignCast(userdata));
3485	var path_buffer: [PATH_MAX]u8 = undefined;
3486	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3487	var sync: CancelRegion.Sync = try .init(ev);
3488	defer sync.deinit(ev);
3489	try ev.fchownat(
3490	&sync,
3491	dir.handle,
3492	sub_path_posix,
3493	owner orelse std.math.maxInt(linux.uid_t),
3494	group orelse std.math.maxInt(linux.gid_t),
3495	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
3496	);
3497	}
3498
3499	fn dirSetPermissions(
3500	userdata: ?*anyopaque,
3501	dir: Dir,
3502	permissions: Dir.Permissions,
3503	) Dir.SetPermissionsError!void {
3504	const ev: *Evented = @ptrCast(@alignCast(userdata));
3505	var sync: CancelRegion.Sync = try .init(ev);
3506	defer sync.deinit(ev);
3507	ev.fchmodat(
3508	&sync,
3509	dir.handle,
3510	"",
3511	permissions.toMode(),
3512	linux.AT.EMPTY_PATH,
3513	) catch \|err\| switch (err) {
3514	error.NameTooLong => return errnoBug(.NAMETOOLONG),
3515	error.BadPathName => return errnoBug(.ILSEQ),
3516	error.ProcessFdQuotaExceeded => return errnoBug(.MFILE),
3517	error.SystemFdQuotaExceeded => return errnoBug(.NFILE),
3518	error.OperationUnsupported => return errnoBug(.OPNOTSUPP),
3519	else => \|e\| return e,
3520	};
3521	}
3522
3523	fn dirSetFilePermissions(
3524	userdata: ?*anyopaque,
3525	dir: Dir,
3526	sub_path: []const u8,
3527	permissions: Dir.Permissions,
3528	options: Dir.SetFilePermissionsOptions,
3529	) Dir.SetFilePermissionsError!void {
3530	const ev: *Evented = @ptrCast(@alignCast(userdata));
3531	var path_buffer: [PATH_MAX]u8 = undefined;
3532	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3533	var sync: CancelRegion.Sync = try .init(ev);
3534	defer sync.deinit(ev);
3535	try ev.fchmodat(
3536	&sync,
3537	dir.handle,
3538	sub_path_posix,
3539	permissions.toMode(),
3540	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
3541	);
3542	}
3543
3544	fn dirSetTimestamps(
3545	userdata: ?*anyopaque,
3546	dir: Dir,
3547	sub_path: []const u8,
3548	options: Dir.SetTimestampsOptions,
3549	) Dir.SetTimestampsError!void {
3550	const ev: *Evented = @ptrCast(@alignCast(userdata));
3551	var path_buffer: [PATH_MAX]u8 = undefined;
3552	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3553	var cancel_region: CancelRegion.Sync = try .init(ev);
3554	defer cancel_region.deinit(ev);
3555	try ev.utimensat(
3556	&cancel_region,
3557	dir.handle,
3558	sub_path_posix,
3559	if (options.modify_timestamp != .now or options.access_timestamp != .now) &.{
3560	setTimestampToPosix(options.access_timestamp),
3561	setTimestampToPosix(options.modify_timestamp),
3562	} else null,
3563	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
3564	);
3565	}
3566
3567	fn dirHardLink(
3568	userdata: ?*anyopaque,
3569	old_dir: Dir,
3570	old_sub_path: []const u8,
3571	new_dir: Dir,
3572	new_sub_path: []const u8,
3573	options: Dir.HardLinkOptions,
3574	) Dir.HardLinkError!void {
3575	const ev: *Evented = @ptrCast(@alignCast(userdata));
3576
3577	var old_path_buffer: [PATH_MAX]u8 = undefined;
3578	var new_path_buffer: [PATH_MAX]u8 = undefined;
3579
3580	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
3581	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
3582
3583	var cancel_region: CancelRegion = .init();
3584	defer cancel_region.deinit();
3585	return ev.linkat(
3586	&cancel_region,
3587	old_dir.handle,
3588	old_sub_path_posix,
3589	new_dir.handle,
3590	new_sub_path_posix,
3591	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
3592	);
3593	}
3594
3595	fn fileStat(userdata: ?*anyopaque, file: File) File.StatError!File.Stat {
3596	const ev: *Evented = @ptrCast(@alignCast(userdata));
3597	var cancel_region: CancelRegion = .init();
3598	defer cancel_region.deinit();
3599	return ev.stat(&cancel_region, file.handle);
3600	}
3601
3602	fn fileLength(userdata: ?*anyopaque, file: File) File.LengthError!u64 {
3603	const ev: *Evented = @ptrCast(@alignCast(userdata));
3604	var cancel_region: CancelRegion = .init();
3605	defer cancel_region.deinit();
3606	while (true) {
3607	var statx_buf = std.mem.zeroes(linux.Statx);
3608	const thread = try cancel_region.awaitIoUring();
3609	thread.enqueue().* = .{
3610	.opcode = .STATX,
3611	.flags = 0,
3612	.ioprio = 0,
3613	.fd = file.handle,
3614	.off = @intFromPtr(&statx_buf),
3615	.addr = @intFromPtr(""),
3616	.len = @bitCast(linux.STATX{ .SIZE = true }),
3617	.rw_flags = linux.AT.EMPTY_PATH,
3618	.user_data = @intFromPtr(cancel_region.fiber),
3619	.buf_index = 0,
3620	.personality = 0,
3621	.splice_fd_in = 0,
3622	.addr3 = 0,
3623	.resv = 0,
3624	};
3625	ev.yield(null, .nothing);
3626	switch (cancel_region.errno()) {
3627	.SUCCESS => {
3628	if (!statx_buf.mask.SIZE) return error.Unexpected;
3629	return statx_buf.size;
3630	},
3631	.INTR, .CANCELED => continue,
3632	.ACCES => \|err\| return errnoBug(err),
3633	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
3634	.FAULT => \|err\| return errnoBug(err),
3635	.INVAL => \|err\| return errnoBug(err),
3636	.LOOP => \|err\| return errnoBug(err),
3637	.NAMETOOLONG => \|err\| return errnoBug(err),
3638	.NOENT => \|err\| return errnoBug(err),
3639	.NOMEM => return error.SystemResources,
3640	.NOTDIR => \|err\| return errnoBug(err),
3641	else => \|err\| return unexpectedErrno(err),
3642	}
3643	}
3644	}
3645
3646	fn fileClose(userdata: ?*anyopaque, files: []const File) void {
3647	const ev: *Evented = @ptrCast(@alignCast(userdata));
3648	var cancel_region: CancelRegion = .init();
3649	defer cancel_region.deinit();
3650	for (files) \|file\| ev.close(file.handle);
3651	}
3652
3653	fn fileWritePositional(
3654	userdata: ?*anyopaque,
3655	file: File,
3656	header: []const u8,
3657	data: []const []const u8,
3658	splat: usize,
3659	offset: u64,
3660	) File.WritePositionalError!usize {
3661	const ev: *Evented = @ptrCast(@alignCast(userdata));
3662
3663	var iovecs: [max_iovecs_len]iovec_const = undefined;
3664	var iovlen: iovlen_t = 0;
3665	addBuf(&iovecs, &iovlen, header);
3666	for (data[0 .. data.len - 1]) \|bytes\| addBuf(&iovecs, &iovlen, bytes);
3667	const pattern = data[data.len - 1];
3668	var backup_buffer: [splat_buffer_size]u8 = undefined;
3669	if (iovecs.len - iovlen != 0) switch (splat) {
3670	0 => {},
3671	1 => addBuf(&iovecs, &iovlen, pattern),
3672	else => switch (pattern.len) {
3673	0 => {},
3674	1 => {
3675	const splat_buffer = &backup_buffer;
3676	const memset_len = @min(splat_buffer.len, splat);
3677	const buf = splat_buffer[0..memset_len];
3678	@memset(buf, pattern[0]);
3679	addBuf(&iovecs, &iovlen, buf);
3680	var remaining_splat = splat - buf.len;
3681	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0) {
3682	assert(buf.len == splat_buffer.len);
3683	addBuf(&iovecs, &iovlen, splat_buffer);
3684	remaining_splat -= splat_buffer.len;
3685	}
3686	addBuf(&iovecs, &iovlen, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
3687	},
3688	else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
3689	addBuf(&iovecs, &iovlen, pattern);
3690	},
3691	},
3692	};
3693
3694	var cancel_region: CancelRegion = .init();
3695	defer cancel_region.deinit();
3696	return ev.pwritev(&cancel_region, file.handle, iovecs[0..iovlen], offset);
3697	}
3698
3699	/// This is either usize or u32. Since, either is fine, let's use the same
3700	/// `addBuf` function for both writing to a file and sending network messages.
3701	const iovlen_t = @FieldType(linux.msghdr_const, "iovlen");
3702
3703	fn addBuf(v: []iovec_const, i: *iovlen_t, bytes: []const u8) void {
3704	// OS checks ptr addr before length so zero length vectors must be omitted.
3705	if (bytes.len == 0) return;
3706	if (v.len - i.* == 0) return;
3707	v[i.*] = .{ .base = bytes.ptr, .len = bytes.len };
3708	i.* += 1;
3709	}
3710
3711	fn fileWriteFileStreaming(
3712	userdata: ?*anyopaque,
3713	file: File,
3714	header: []const u8,
3715	file_reader: *File.Reader,
3716	limit: Io.Limit,
3717	) File.Writer.WriteFileError!usize {
3718	const ev: *Evented = @ptrCast(@alignCast(userdata));
3719	_ = ev;
3720	_ = file;
3721	_ = header;
3722	_ = file_reader;
3723	_ = limit;
3724	return error.Unimplemented;
3725	}
3726
3727	fn fileWriteFilePositional(
3728	userdata: ?*anyopaque,
3729	file: File,
3730	header: []const u8,
3731	file_reader: *File.Reader,
3732	limit: Io.Limit,
3733	offset: u64,
3734	) File.WriteFilePositionalError!usize {
3735	const ev: *Evented = @ptrCast(@alignCast(userdata));
3736	_ = ev;
3737	_ = file;
3738	_ = header;
3739	_ = file_reader;
3740	_ = limit;
3741	_ = offset;
3742	return error.Unimplemented;
3743	}
3744
3745	fn fileReadPositional(
3746	userdata: ?*anyopaque,
3747	file: File,
3748	data: []const []u8,
3749	offset: u64,
3750	) File.ReadPositionalError!usize {
3751	const ev: *Evented = @ptrCast(@alignCast(userdata));
3752
3753	var iovecs_buffer: [max_iovecs_len]iovec = undefined;
3754	var i: usize = 0;
3755	for (data) \|buf\| {
3756	if (iovecs_buffer.len - i == 0) break;
3757	if (buf.len > 0) {
3758	iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len };
3759	i += 1;
3760	}
3761	}
3762	if (i == 0) return 0;
3763	const dest = iovecs_buffer[0..i];
3764	assert(dest[0].len > 0);
3765
3766	var cancel_region: CancelRegion = .init();
3767	defer cancel_region.deinit();
3768	return ev.preadv(&cancel_region, file.handle, dest, offset) catch \|err\| switch (err) {
3769	error.SocketUnconnected => return errnoBug(.NOTCONN), // not a socket
3770	error.ConnectionResetByPeer => return errnoBug(.CONNRESET), // not a socket
3771	else => \|e\| return e,
3772	};
3773	}
3774
3775	fn fileSeekBy(userdata: ?*anyopaque, file: File, offset: i64) File.SeekError!void {
3776	const ev: *Evented = @ptrCast(@alignCast(userdata));
3777	var sync: CancelRegion.Sync = try .init(ev);
3778	defer sync.deinit(ev);
3779	try ev.lseek(&sync, file.handle, @bitCast(offset), linux.SEEK.CUR);
3780	}
3781
3782	fn fileSeekTo(userdata: ?*anyopaque, file: File, offset: u64) File.SeekError!void {
3783	const ev: *Evented = @ptrCast(@alignCast(userdata));
3784	var sync: CancelRegion.Sync = try .init(ev);
3785	defer sync.deinit(ev);
3786	try ev.lseek(&sync, file.handle, offset, linux.SEEK.SET);
3787	}
3788
3789	fn fileSync(userdata: ?*anyopaque, file: File) File.SyncError!void {
3790	const ev: *Evented = @ptrCast(@alignCast(userdata));
3791	var cancel_region: CancelRegion = .init();
3792	defer cancel_region.deinit();
3793	while (true) {
3794	const thread = try cancel_region.awaitIoUring();
3795	thread.enqueue().* = .{
3796	.opcode = .FSYNC,
3797	.flags = 0,
3798	.ioprio = 0,
3799	.fd = file.handle,
3800	.off = 0,
3801	.addr = 0,
3802	.len = 0,
3803	.rw_flags = 0,
3804	.user_data = @intFromPtr(cancel_region.fiber),
3805	.buf_index = 0,
3806	.personality = 0,
3807	.splice_fd_in = 0,
3808	.addr3 = 0,
3809	.resv = 0,
3810	};
3811	ev.yield(null, .nothing);
3812	switch (cancel_region.errno()) {
3813	.SUCCESS => return,
3814	.INTR, .CANCELED => continue,
3815	.BADF => \|err\| return errnoBug(err),
3816	.INVAL => \|err\| return errnoBug(err),
3817	.ROFS => \|err\| return errnoBug(err),
3818	.IO => return error.InputOutput,
3819	.NOSPC => return error.NoSpaceLeft,
3820	.DQUOT => return error.DiskQuota,
3821	else => \|err\| return unexpectedErrno(err),
3822	}
3823	}
3824	}
3825
3826	fn fileIsTty(userdata: ?*anyopaque, file: File) Io.Cancelable!bool {
3827	const ev: *Evented = @ptrCast(@alignCast(userdata));
3828	var sync: CancelRegion.Sync = try .init(ev);
3829	defer sync.deinit(ev);
3830	while (true) {
3831	try sync.cancel_region.await(.nothing);
3832	var wsz: winsize = undefined;
3833	const rc = linux.ioctl(file.handle, linux.T.IOCGWINSZ, @intFromPtr(&wsz));
3834	switch (linux.errno(rc)) {
3835	.SUCCESS => return true,
3836	.INTR => continue,
3837	else => return false,
3838	}
3839	}
3840	}
3841
3842	fn fileEnableAnsiEscapeCodes(userdata: ?*anyopaque, file: File) File.EnableAnsiEscapeCodesError!void {
3843	const ev: *Evented = @ptrCast(@alignCast(userdata));
3844	if (!try fileIsTty(ev, file)) return error.NotTerminalDevice;
3845	}
3846
3847	fn fileSetLength(userdata: ?*anyopaque, file: File, length: u64) File.SetLengthError!void {
3848	const ev: *Evented = @ptrCast(@alignCast(userdata));
3849	var cancel_region: CancelRegion = .init();
3850	defer cancel_region.deinit();
3851	while (true) {
3852	const thread = try cancel_region.awaitIoUring();
3853	thread.enqueue().* = .{
3854	.opcode = .FTRUNCATE,
3855	.flags = 0,
3856	.ioprio = 0,
3857	.fd = file.handle,
3858	.off = length,
3859	.addr = 0,
3860	.len = 0,
3861	.rw_flags = 0,
3862	.user_data = @intFromPtr(cancel_region.fiber),
3863	.buf_index = 0,
3864	.personality = 0,
3865	.splice_fd_in = 0,
3866	.addr3 = 0,
3867	.resv = 0,
3868	};
3869	ev.yield(null, .nothing);
3870	switch (cancel_region.errno()) {
3871	.SUCCESS => return,
3872	.INTR, .CANCELED => continue,
3873	.FBIG => return error.FileTooBig,
3874	.IO => return error.InputOutput,
3875	.PERM => return error.PermissionDenied,
3876	.TXTBSY => return error.FileBusy,
3877	.BADF => \|err\| return errnoBug(err), // Handle not open for writing.
3878	.INVAL => return error.NonResizable, // This is returned for /dev/null for example.
3879	else => \|err\| return unexpectedErrno(err),
3880	}
3881	}
3882	}
3883
3884	fn fileSetOwner(
3885	userdata: ?*anyopaque,
3886	file: File,
3887	owner: ?File.Uid,
3888	group: ?File.Gid,
3889	) File.SetOwnerError!void {
3890	const ev: *Evented = @ptrCast(@alignCast(userdata));
3891	var sync: CancelRegion.Sync = try .init(ev);
3892	defer sync.deinit(ev);
3893	try ev.fchownat(
3894	&sync,
3895	file.handle,
3896	"",
3897	owner orelse std.math.maxInt(linux.uid_t),
3898	group orelse std.math.maxInt(linux.gid_t),
3899	linux.AT.EMPTY_PATH,
3900	);
3901	}
3902
3903	fn fileSetPermissions(
3904	userdata: ?*anyopaque,
3905	file: File,
3906	permissions: File.Permissions,
3907	) File.SetPermissionsError!void {
3908	const ev: *Evented = @ptrCast(@alignCast(userdata));
3909	var sync: CancelRegion.Sync = try .init(ev);
3910	defer sync.deinit(ev);
3911	ev.fchmodat(
3912	&sync,
3913	file.handle,
3914	"",
3915	permissions.toMode(),
3916	linux.AT.EMPTY_PATH,
3917	) catch \|err\| switch (err) {
3918	error.NameTooLong => return errnoBug(.NAMETOOLONG),
3919	error.BadPathName => return errnoBug(.ILSEQ),
3920	error.ProcessFdQuotaExceeded => return errnoBug(.MFILE),
3921	error.SystemFdQuotaExceeded => return errnoBug(.NFILE),
3922	error.OperationUnsupported => return errnoBug(.OPNOTSUPP),
3923	else => \|e\| return e,
3924	};
3925	}
3926
3927	fn fileSetTimestamps(
3928	userdata: ?*anyopaque,
3929	file: File,
3930	options: File.SetTimestampsOptions,
3931	) File.SetTimestampsError!void {
3932	const ev: *Evented = @ptrCast(@alignCast(userdata));
3933	var sync: CancelRegion.Sync = try .init(ev);
3934	defer sync.deinit(ev);
3935	try ev.utimensat(
3936	&sync,
3937	file.handle,
3938	"",
3939	if (options.modify_timestamp != .now or options.access_timestamp != .now) &.{
3940	setTimestampToPosix(options.access_timestamp),
3941	setTimestampToPosix(options.modify_timestamp),
3942	} else null,
3943	linux.AT.EMPTY_PATH,
3944	);
3945	}
3946
3947	fn fileLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!void {
3948	const ev: *Evented = @ptrCast(@alignCast(userdata));
3949	var sync: CancelRegion.Sync = try .init(ev);
3950	defer sync.deinit(ev);
3951	ev.flock(&sync, file.handle, lock, .blocking) catch \|err\| switch (err) {
3952	error.WouldBlock => unreachable, // blocking
3953	else => \|e\| return e,
3954	};
3955	}
3956
3957	fn fileTryLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!bool {
3958	const ev: *Evented = @ptrCast(@alignCast(userdata));
3959	var sync: CancelRegion.Sync = try .init(ev);
3960	defer sync.deinit(ev);
3961	ev.flock(&sync, file.handle, lock, switch (lock) {
3962	.none => .blocking,
3963	.shared, .exclusive => .nonblocking,
3964	}) catch \|err\| switch (err) {
3965	error.WouldBlock => return false,
3966	else => \|e\| return e,
3967	};
3968	return true;
3969	}
3970
3971	fn fileUnlock(userdata: ?*anyopaque, file: File) void {
3972	const ev: *Evented = @ptrCast(@alignCast(userdata));
3973	var sync: CancelRegion.Sync = .initBlocked(ev);
3974	defer sync.deinit(ev);
3975	ev.flock(&sync, file.handle, .none, .blocking) catch \|err\| switch (err) {
3976	error.Canceled => unreachable, // blocked
3977	error.WouldBlock => unreachable, // blocking
3978	error.SystemResources => return recoverableOsBugDetected(), // Resource deallocation.
3979	error.FileLocksUnsupported => return recoverableOsBugDetected(), // We already got the lock.
3980	error.Unexpected => return recoverableOsBugDetected(), // Resource deallocation must succeed.
3981	};
3982	}
3983
3984	fn fileDowngradeLock(userdata: ?*anyopaque, file: File) File.DowngradeLockError!void {
3985	const ev: *Evented = @ptrCast(@alignCast(userdata));
3986	var sync: CancelRegion.Sync = try .init(ev);
3987	defer sync.deinit(ev);
3988	ev.flock(&sync, file.handle, .shared, .nonblocking) catch \|err\| switch (err) {
3989	error.WouldBlock => return errnoBug(.AGAIN), // File was not locked in exclusive mode.
3990	error.SystemResources => return errnoBug(.NOLCK), // Lock already obtained.
3991	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Lock already obtained.
3992	else => \|e\| return e,
3993	};
3994	}
3995
3996	fn fileRealPath(userdata: ?*anyopaque, file: File, out_buffer: []u8) File.RealPathError!usize {
3997	const ev: *Evented = @ptrCast(@alignCast(userdata));
3998	var sync: CancelRegion.Sync = try .init(ev);
3999	defer sync.deinit(ev);
4000	return ev.realPath(&sync, file.handle, out_buffer);
4001	}
4002
4003	fn fileHardLink(
4004	userdata: ?*anyopaque,
4005	file: File,
4006	new_dir: Dir,
4007	new_sub_path: []const u8,
4008	options: File.HardLinkOptions,
4009	) File.HardLinkError!void {
4010	const ev: *Evented = @ptrCast(@alignCast(userdata));
4011
4012	var new_path_buffer: [PATH_MAX]u8 = undefined;
4013	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
4014
4015	var cancel_region: CancelRegion = .init();
4016	defer cancel_region.deinit();
4017	return ev.linkat(
4018	&cancel_region,
4019	file.handle,
4020	"",
4021	new_dir.handle,
4022	new_sub_path_posix,
4023	linux.AT.EMPTY_PATH \| @as(u32, if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW),
4024	);
4025	}
4026
4027	fn fileMemoryMapCreate(
4028	userdata: ?*anyopaque,
4029	file: File,
4030	options: File.MemoryMap.CreateOptions,
4031	) File.MemoryMap.CreateError!File.MemoryMap {
4032	const ev: *Evented = @ptrCast(@alignCast(userdata));
4033
4034	const prot: linux.PROT = .{
4035	.READ = options.protection.read,
4036	.WRITE = options.protection.write,
4037	.EXEC = options.protection.execute,
4038	};
4039	const flags: linux.MAP = .{
4040	.TYPE = .SHARED_VALIDATE,
4041	.POPULATE = options.populate,
4042	};
4043
4044	const page_align = std.heap.page_size_min;
4045
4046	var sync: CancelRegion.Sync = try .init(ev);
4047	defer sync.deinit(ev);
4048	const contents = while (true) {
4049	try sync.cancel_region.await(.nothing);
4050	const casted_offset = std.math.cast(i64, options.offset) orelse return error.Unseekable;
4051	const rc = linux.mmap(null, options.len, prot, flags, file.handle, casted_offset);
4052	switch (linux.errno(rc)) {
4053	.SUCCESS => break @as([*]align(page_align) u8, @ptrFromInt(rc))[0..options.len],
4054	.INTR => continue,
4055	.ACCES => return error.AccessDenied,
4056	.AGAIN => return error.LockedMemoryLimitExceeded,
4057	.MFILE => return error.ProcessFdQuotaExceeded,
4058	.NFILE => return error.SystemFdQuotaExceeded,
4059	.NOMEM => return error.OutOfMemory,
4060	.PERM => return error.PermissionDenied,
4061	.OVERFLOW => return error.Unseekable,
4062	.BADF => \|err\| return errnoBug(err), // Always a race condition.
4063	.INVAL => \|err\| return errnoBug(err), // Invalid parameters to mmap()
4064	.OPNOTSUPP => \|err\| return errnoBug(err), // Bad flags with MAP.SHARED_VALIDATE on Linux.
4065	else => \|err\| return unexpectedErrno(err),
4066	}
4067	};
4068	return .{
4069	.file = file,
4070	.offset = options.offset,
4071	.memory = contents,
4072	.section = {},
4073	};
4074	}
4075
4076	fn fileMemoryMapDestroy(userdata: ?anyopaque, mm: File.MemoryMap) void {
4077	const ev: *Evented = @ptrCast(@alignCast(userdata));
4078	_ = ev;
4079	const memory = mm.memory;
4080	if (memory.len == 0) return;
4081	switch (linux.errno(linux.munmap(memory.ptr, memory.len))) {
4082	.SUCCESS => {},
4083	else => \|err\| if (builtin.mode == .Debug)
4084	std.log.err("failed to unmap {d} bytes at {*}: {t}", .{ memory.len, memory.ptr, err }),
4085	}
4086	mm.* = undefined;
4087	}
4088
4089	fn fileMemoryMapSetLength(
4090	userdata: ?*anyopaque,
4091	mm: *File.MemoryMap,
4092	new_len: usize,
4093	) File.MemoryMap.SetLengthError!void {
4094	const ev: *Evented = @ptrCast(@alignCast(userdata));
4095
4096	const page_size = std.heap.pageSize();
4097	const alignment: Alignment = .fromByteUnits(page_size);
4098	const page_align = std.heap.page_size_min;
4099	const old_memory = mm.memory;
4100
4101	if (alignment.forward(new_len) == alignment.forward(old_memory.len)) {
4102	mm.memory.len = new_len;
4103	return;
4104	}
4105	const flags: linux.MREMAP = .{ .MAYMOVE = true };
4106	const addr_hint: ?[*]const u8 = null;
4107	var sync: CancelRegion.Sync = try .init(ev);
4108	defer sync.deinit(ev);
4109	const new_memory = while (true) {
4110	try sync.cancel_region.await(.nothing);
4111	const rc = linux.mremap(old_memory.ptr, old_memory.len, new_len, flags, addr_hint);
4112	switch (linux.errno(rc)) {
4113	.SUCCESS => break @as([*]align(page_align) u8, @ptrFromInt(rc))[0..new_len],
4114	.INTR => continue,
4115	.AGAIN => return error.LockedMemoryLimitExceeded,
4116	.NOMEM => return error.OutOfMemory,
4117	.INVAL => \|err\| return errnoBug(err),
4118	.FAULT => \|err\| return errnoBug(err),
4119	else => \|err\| return unexpectedErrno(err),
4120	}
4121	};
4122	mm.memory = new_memory;
4123	}
4124
4125	fn fileMemoryMapRead(userdata: ?anyopaque, mm: File.MemoryMap) File.ReadPositionalError!void {
4126	const ev: *Evented = @ptrCast(@alignCast(userdata));
4127	_ = ev;
4128	_ = mm;
4129	}
4130
4131	fn fileMemoryMapWrite(userdata: ?anyopaque, mm: File.MemoryMap) File.WritePositionalError!void {
4132	const ev: *Evented = @ptrCast(@alignCast(userdata));
4133	_ = ev;
4134	_ = mm;
4135	}
4136
4137	fn processExecutableOpen(
4138	userdata: ?*anyopaque,
4139	flags: File.OpenFlags,
4140	) process.OpenExecutableError!File {
4141	const ev: *Evented = @ptrCast(@alignCast(userdata));
4142	return dirOpenFile(ev, .{ .handle = linux.AT.FDCWD }, "/proc/self/exe", flags);
4143	}
4144
4145	fn processExecutablePath(userdata: ?*anyopaque, out_buffer: []u8) process.ExecutablePathError!usize {
4146	const ev: *Evented = @ptrCast(@alignCast(userdata));
4147	return dirReadLink(ev, .cwd(), "/proc/self/exe", out_buffer) catch \|err\| switch (err) {
4148	error.UnsupportedReparsePointType => unreachable, // Windows-only
4149	error.NetworkNotFound => unreachable, // Windows-only
4150	error.FileBusy => unreachable, // Windows-only
4151	else => \|e\| return e,
4152	};
4153	}
4154
4155	fn lockStderr(userdata: ?*anyopaque, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
4156	const ev: *Evented = @ptrCast(@alignCast(userdata));
4157	const ev_io = ev.io();
4158	ev.stderr_mutex.lockUncancelable(ev_io);
4159	errdefer ev.stderr_mutex.unlock(ev_io);
4160	return ev.initLockedStderr(terminal_mode);
4161	}
4162
4163	fn tryLockStderr(
4164	userdata: ?*anyopaque,
4165	terminal_mode: ?Io.Terminal.Mode,
4166	) Io.Cancelable!?Io.LockedStderr {
4167	const ev: *Evented = @ptrCast(@alignCast(userdata));
4168	const ev_io = ev.io();
4169	if (!ev.stderr_mutex.tryLock()) return null;
4170	errdefer ev.stderr_mutex.unlock(ev_io);
4171	return try ev.initLockedStderr(terminal_mode);
4172	}
4173
4174	fn initLockedStderr(ev: *Evented, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
4175	if (!ev.stderr_writer_initialized) {
4176	const ev_io = ev.io();
4177	const cancel_protection = swapCancelProtection(ev, .blocked);
4178	defer assert(swapCancelProtection(ev, cancel_protection) == .blocked);
4179	ev.scanEnviron() catch \|err\| switch (err) {
4180	error.Canceled => unreachable, // blocked
4181	};
4182	const NO_COLOR = ev.environ.exist.NO_COLOR;
4183	const CLICOLOR_FORCE = ev.environ.exist.CLICOLOR_FORCE;
4184	ev.stderr_mode = Io.Terminal.Mode.detect(
4185	ev_io,
4186	ev.stderr_writer.file,
4187	NO_COLOR,
4188	CLICOLOR_FORCE,
4189	) catch \|err\| switch (err) {
4190	error.Canceled => unreachable, // blocked
4191	};
4192	ev.stderr_writer_initialized = true;
4193	}
4194	return .{
4195	.file_writer = &ev.stderr_writer,
4196	.terminal_mode = terminal_mode orelse ev.stderr_mode,
4197	};
4198	}
4199
4200	fn unlockStderr(userdata: ?*anyopaque) void {
4201	const ev: *Evented = @ptrCast(@alignCast(userdata));
4202	if (ev.stderr_writer.err == null) ev.stderr_writer.interface.flush() catch {};
4203	if (ev.stderr_writer.err) \|err\| {
4204	switch (err) {
4205	error.Canceled => Thread.current().currentFiber().cancel_protection.recancel(),
4206	else => {},
4207	}
4208	ev.stderr_writer.err = null;
4209	}
4210	ev.stderr_writer.interface.end = 0;
4211	ev.stderr_writer.interface.buffer = &.{};
4212	ev.stderr_mutex.unlock(ev.io());
4213	}
4214
4215	fn processCurrentPath(userdata: ?*anyopaque, buffer: []u8) process.CurrentPathError!usize {
4216	const ev: *Evented = @ptrCast(@alignCast(userdata));
4217	var sync: CancelRegion.Sync = try .init(ev);
4218	defer sync.deinit(ev);
4219	while (true) {
4220	try sync.cancel_region.await(.nothing);
4221	switch (linux.errno(linux.getcwd(buffer.ptr, buffer.len))) {
4222	.SUCCESS => return std.mem.findScalar(u8, buffer, 0).?,
4223	.INTR => continue,
4224	.NOENT => return error.CurrentDirUnlinked,
4225	.RANGE => return error.NameTooLong,
4226	.FAULT => \|err\| return errnoBug(err),
4227	.INVAL => \|err\| return errnoBug(err),
4228	else => \|err\| return unexpectedErrno(err),
4229	}
4230	}
4231	}
4232
4233	fn processSetCurrentDir(userdata: ?*anyopaque, dir: Dir) process.SetCurrentDirError!void {
4234	const ev: *Evented = @ptrCast(@alignCast(userdata));
4235	if (dir.handle == linux.AT.FDCWD) return;
4236	var sync: CancelRegion.Sync = try .init(ev);
4237	defer sync.deinit(ev);
4238	return ev.fchdir(&sync, dir.handle);
4239	}
4240
4241	fn processSetCurrentPath(userdata: ?*anyopaque, dir_path: []const u8) ChdirError!void {
4242	const ev: *Evented = @ptrCast(@alignCast(userdata));
4243	var path_buffer: [PATH_MAX]u8 = undefined;
4244	const dir_path_posix = try pathToPosix(dir_path, &path_buffer);
4245	var sync: CancelRegion.Sync = try .init(ev);
4246	defer sync.deinit(ev);
4247	return ev.chdir(&sync, dir_path_posix);
4248	}
4249
4250	fn processReplace(userdata: ?*anyopaque, options: process.ReplaceOptions) process.ReplaceError {
4251	const ev: *Evented = @ptrCast(@alignCast(userdata));
4252
4253	try ev.scanEnviron(); // for PATH
4254	const PATH = ev.environ.string.PATH orelse default_PATH;
4255
4256	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
4257	defer arena_allocator.deinit();
4258	const arena = arena_allocator.allocator();
4259
4260	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
4261	for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
4262
4263	const env_block = env_block: {
4264	const prog_fd: i32 = -1;
4265	if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
4266	.zig_progress_fd = prog_fd,
4267	});
4268	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
4269	.zig_progress_fd = prog_fd,
4270	});
4271	};
4272
4273	var sync: CancelRegion.Sync = try .init(ev);
4274	defer sync.deinit(ev);
4275	return ev.execv(&sync, options.expand_arg0, argv_buf.ptr[0].?, argv_buf.ptr, env_block, PATH);
4276	}
4277
4278	fn processReplacePath(
4279	userdata: ?*anyopaque,
4280	dir: Dir,
4281	options: process.ReplaceOptions,
4282	) process.ReplaceError {
4283	const ev: *Evented = @ptrCast(@alignCast(userdata));
4284	_ = ev;
4285	_ = dir;
4286	_ = options;
4287	@panic("TODO processReplacePath");
4288	}
4289
4290	fn processSpawn(userdata: ?*anyopaque, options: process.SpawnOptions) process.SpawnError!process.Child {
4291	const ev: *Evented = @ptrCast(@alignCast(userdata));
4292	const spawned = try ev.spawn(options);
4293	var cancel_region: CancelRegion = .initBlocked();
4294	defer cancel_region.deinit();
4295	defer ev.close(spawned.err_fd);
4296
4297	// Wait for the child to report any errors in or before `execvpe`.
4298	var child_err: ForkBailError = undefined;
4299	ev.readAll(&cancel_region, spawned.err_fd, @ptrCast(&child_err)) catch \|read_err\| {
4300	switch (read_err) {
4301	error.Canceled => unreachable, // blocked
4302	error.EndOfStream => {
4303	// Write end closed by CLOEXEC at the time of the `execvpe` call,
4304	// indicating success.
4305	},
4306	else => {
4307	// Problem reading the error from the error reporting pipe. We
4308	// don't know if the child is alive or dead. Better to assume it is
4309	// alive so the resource does not risk being leaked.
4310	},
4311	}
4312	return .{
4313	.id = spawned.pid,
4314	.thread_handle = {},
4315	.stdin = spawned.stdin,
4316	.stdout = spawned.stdout,
4317	.stderr = spawned.stderr,
4318	.request_resource_usage_statistics = options.request_resource_usage_statistics,
4319	};
4320	};
4321	return child_err;
4322	}
4323
4324	fn processSpawnPath(
4325	userdata: ?*anyopaque,
4326	dir: Dir,
4327	options: process.SpawnOptions,
4328	) process.SpawnError!process.Child {
4329	const ev: *Evented = @ptrCast(@alignCast(userdata));
4330	_ = ev;
4331	_ = dir;
4332	_ = options;
4333	@panic("TODO processSpawnPath");
4334	}
4335
4336	const prog_fileno = @max(linux.STDIN_FILENO, linux.STDOUT_FILENO, linux.STDERR_FILENO);
4337
4338	const Spawned = struct {
4339	pid: pid_t,
4340	err_fd: fd_t,
4341	stdin: ?File,
4342	stdout: ?File,
4343	stderr: ?File,
4344	};
4345	fn spawn(ev: *Evented, options: process.SpawnOptions) process.SpawnError!Spawned {
4346	var cancel_region: CancelRegion = .init();
4347	defer cancel_region.deinit();
4348
4349	// The child process does need to access (one end of) these pipes. However,
4350	// we must initially set CLOEXEC to avoid a race condition. If another thread
4351	// is racing to spawn a different child process, we don't want it to inherit
4352	// these FDs in any scenario; that would mean that, for instance, calls to
4353	// `poll` from the parent would not report the child's stdout as closing when
4354	// expected, since the other child may retain a reference to the write end of
4355	// the pipe. So, we create the pipes with CLOEXEC initially. After fork, we
4356	// need to do something in the new child to make sure we preserve the reference
4357	// we want. We could use `fcntl` to remove CLOEXEC from the FD, but as it
4358	// turns out, we `dup2` everything anyway, so there's no need!
4359	const pipe_flags: linux.O = .{ .CLOEXEC = true };
4360
4361	const stdin_pipe = if (options.stdin == .pipe) try pipe2(pipe_flags) else undefined;
4362	errdefer if (options.stdin == .pipe) {
4363	ev.destroyPipe(stdin_pipe);
4364	};
4365
4366	const stdout_pipe = if (options.stdout == .pipe) try pipe2(pipe_flags) else undefined;
4367	errdefer if (options.stdout == .pipe) {
4368	ev.destroyPipe(stdout_pipe);
4369	};
4370
4371	const stderr_pipe = if (options.stderr == .pipe) try pipe2(pipe_flags) else undefined;
4372	errdefer if (options.stderr == .pipe) {
4373	ev.destroyPipe(stderr_pipe);
4374	};
4375
4376	const any_ignore =
4377	options.stdin == .ignore or options.stdout == .ignore or options.stderr == .ignore;
4378	const dev_null_fd = if (any_ignore) try ev.null_fd.open(ev, &cancel_region, "/dev/null", .{
4379	.ACCMODE = .RDWR,
4380	}) else undefined;
4381
4382	const prog_pipe: [2]fd_t = if (options.progress_node.index != .none) pipe: {
4383	// We use CLOEXEC for the same reason as in `pipe_flags`.
4384	const pipe = try pipe2(.{ .NONBLOCK = true, .CLOEXEC = true });
4385	_ = linux.fcntl(pipe[0], linux.F.SETPIPE_SZ, @as(u32, std.Progress.max_packet_len * 2));
4386	break :pipe pipe;
4387	} else .{ -1, -1 };
4388	errdefer ev.destroyPipe(prog_pipe);
4389
4390	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
4391	defer arena_allocator.deinit();
4392	const arena = arena_allocator.allocator();
4393
4394	// The POSIX standard does not allow malloc() between fork() and execve(),
4395	// and this allocator may be a libc allocator.
4396	// I have personally observed the child process deadlocking when it tries
4397	// to call malloc() due to a heap allocation between fork() and execve(),
4398	// in musl v1.1.24.
4399	// Additionally, we want to reduce the number of possible ways things
4400	// can fail between fork() and execve().
4401	// Therefore, we do all the allocation for the execve() before the fork().
4402	// This means we must do the null-termination of argv and env vars here.
4403	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
4404	for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
4405
4406	const env_block = env_block: {
4407	const prog_fd: i32 = if (prog_pipe[1] == -1) -1 else prog_fileno;
4408	if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
4409	.zig_progress_fd = prog_fd,
4410	});
4411	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
4412	.zig_progress_fd = prog_fd,
4413	});
4414	};
4415
4416	// This pipe communicates to the parent errors in the child between `fork` and `execvpe`.
4417	// It is closed by the child (via CLOEXEC) without writing if `execvpe` succeeds.
4418	const err_pipe: [2]fd_t = try pipe2(.{ .CLOEXEC = true });
4419	errdefer ev.destroyPipe(err_pipe);
4420
4421	try ev.scanEnviron(); // for PATH
4422	const PATH = ev.environ.string.PATH orelse default_PATH;
4423
4424	const pid_result: pid_t = fork: {
4425	const rc = linux.fork();
4426	switch (linux.errno(rc)) {
4427	.SUCCESS => break :fork @intCast(rc),
4428	.AGAIN => return error.SystemResources,
4429	.NOMEM => return error.SystemResources,
4430	.NOSYS => return error.OperationUnsupported,
4431	else => \|err\| return unexpectedErrno(err),
4432	}
4433	};
4434
4435	if (pid_result == 0) {
4436	defer comptime unreachable; // We are the child.
4437	var sync: CancelRegion.Sync = .{ .cancel_region = .initBlocked() };
4438	const err = ev.setUpChild(&sync, .{
4439	.stdin_pipe = stdin_pipe[0],
4440	.stdout_pipe = stdout_pipe[1],
4441	.stderr_pipe = stderr_pipe[1],
4442	.dev_null_fd = dev_null_fd,
4443	.prog_pipe = prog_pipe[1],
4444	.argv_buf = argv_buf,
4445	.env_block = env_block,
4446	.PATH = PATH,
4447	.spawn = options,
4448	});
4449	ev.writeAll(&sync.cancel_region, err_pipe[1], @ptrCast(&err)) catch {};
4450	const exit = if (builtin.single_threaded) linux.exit else linux.exit_group;
4451	exit(1);
4452	}
4453
4454	const pid: pid_t = @intCast(pid_result); // We are the parent.
4455	errdefer comptime unreachable; // The child is forked; we must not error from now on
4456
4457	ev.close(err_pipe[1]); // make sure only the child holds the write end open
4458
4459	if (options.stdin == .pipe) ev.close(stdin_pipe[0]);
4460	if (options.stdout == .pipe) ev.close(stdout_pipe[1]);
4461	if (options.stderr == .pipe) ev.close(stderr_pipe[1]);
4462
4463	if (prog_pipe[1] != -1) ev.close(prog_pipe[1]);
4464
4465	options.progress_node.setIpcFile(ev, .{ .handle = prog_pipe[0], .flags = .{ .nonblocking = true } });
4466
4467	return .{
4468	.pid = pid,
4469	.err_fd = err_pipe[0],
4470	.stdin = switch (options.stdin) {
4471	.pipe => .{ .handle = stdin_pipe[1], .flags = .{ .nonblocking = false } },
4472	else => null,
4473	},
4474	.stdout = switch (options.stdout) {
4475	.pipe => .{ .handle = stdout_pipe[0], .flags = .{ .nonblocking = false } },
4476	else => null,
4477	},
4478	.stderr = switch (options.stderr) {
4479	.pipe => .{ .handle = stderr_pipe[0], .flags = .{ .nonblocking = false } },
4480	else => null,
4481	},
4482	};
4483	}
4484
4485	pub const PipeError = error{
4486	SystemFdQuotaExceeded,
4487	ProcessFdQuotaExceeded,
4488	} \|\| Io.UnexpectedError;
4489	pub fn pipe2(flags: linux.O) PipeError![2]fd_t {
4490	var fds: [2]fd_t = undefined;
4491	switch (linux.errno(linux.pipe2(&fds, flags))) {
4492	.SUCCESS => return fds,
4493	.INVAL => \|err\| return errnoBug(err), // Invalid flags
4494	.NFILE => return error.SystemFdQuotaExceeded,
4495	.MFILE => return error.ProcessFdQuotaExceeded,
4496	else => \|err\| return unexpectedErrno(err),
4497	}
4498	}
4499	fn destroyPipe(ev: *Evented, pipe: [2]fd_t) void {
4500	if (pipe[0] != -1) ev.close(pipe[0]);
4501	if (pipe[0] != pipe[1]) ev.close(pipe[1]);
4502	}
4503
4504	/// Errors that can occur between fork() and execv()
4505	const ForkBailError = process.SetCurrentDirError \|\| ChdirError \|\|
4506	process.SpawnError \|\| process.ReplaceError;
4507	fn setUpChild(ev: Evented, sync: CancelRegion.Sync, options: struct {
4508	stdin_pipe: fd_t,
4509	stdout_pipe: fd_t,
4510	stderr_pipe: fd_t,
4511	dev_null_fd: fd_t,
4512	prog_pipe: fd_t,
4513	argv_buf: [:null]?[*:0]const u8,
4514	env_block: process.Environ.Block,
4515	PATH: []const u8,
4516	spawn: process.SpawnOptions,
4517	}) ForkBailError {
4518	try ev.setUpChildIo(
4519	sync,
4520	options.spawn.stdin,
4521	options.stdin_pipe,
4522	linux.STDIN_FILENO,
4523	options.dev_null_fd,
4524	);
4525	try ev.setUpChildIo(
4526	sync,
4527	options.spawn.stdout,
4528	options.stdout_pipe,
4529	linux.STDOUT_FILENO,
4530	options.dev_null_fd,
4531	);
4532	try ev.setUpChildIo(
4533	sync,
4534	options.spawn.stderr,
4535	options.stderr_pipe,
4536	linux.STDERR_FILENO,
4537	options.dev_null_fd,
4538	);
4539
4540	switch (options.spawn.cwd) {
4541	.inherit => {},
4542	.dir => \|cwd_dir\| try ev.fchdir(sync, cwd_dir.handle),
4543	.path => \|cwd_path\| {
4544	var cwd_path_buffer: [PATH_MAX]u8 = undefined;
4545	const cwd_path_posix = try pathToPosix(cwd_path, &cwd_path_buffer);
4546	try ev.chdir(sync, cwd_path_posix);
4547	},
4548	}
4549
4550	// Must happen after fchdir above, the cwd file descriptor might be
4551	// equal to prog_fileno and be clobbered by this dup2 call.
4552	if (options.prog_pipe != -1) try ev.dup2(sync, options.prog_pipe, prog_fileno);
4553
4554	if (options.spawn.gid) \|gid\| {
4555	switch (linux.errno(linux.setregid(gid, gid))) {
4556	.SUCCESS => {},
4557	.AGAIN => return error.ResourceLimitReached,
4558	.INVAL => return error.InvalidUserId,
4559	.PERM => return error.PermissionDenied,
4560	else => return error.Unexpected,
4561	}
4562	}
4563
4564	if (options.spawn.uid) \|uid\| {
4565	switch (linux.errno(linux.setreuid(uid, uid))) {
4566	.SUCCESS => {},
4567	.AGAIN => return error.ResourceLimitReached,
4568	.INVAL => return error.InvalidUserId,
4569	.PERM => return error.PermissionDenied,
4570	else => return error.Unexpected,
4571	}
4572	}
4573
4574	if (options.spawn.pgid) \|pid\| {
4575	switch (linux.errno(linux.setpgid(0, pid))) {
4576	.SUCCESS => {},
4577	.ACCES => return error.ProcessAlreadyExec,
4578	.INVAL => return error.InvalidProcessGroupId,
4579	.PERM => return error.PermissionDenied,
4580	else => return error.Unexpected,
4581	}
4582	}
4583
4584	if (options.spawn.start_suspended) {
4585	switch (linux.errno(linux.kill(0, .STOP))) {
4586	.SUCCESS => {},
4587	.PERM => return error.PermissionDenied,
4588	else => return error.Unexpected,
4589	}
4590	}
4591
4592	return ev.execv(
4593	sync,
4594	options.spawn.expand_arg0,
4595	options.argv_buf.ptr[0].?,
4596	options.argv_buf.ptr,
4597	options.env_block,
4598	options.PATH,
4599	);
4600	}
4601
4602	fn setUpChildIo(
4603	ev: *Evented,
4604	sync: *CancelRegion.Sync,
4605	stdio: process.SpawnOptions.StdIo,
4606	pipe_fd: fd_t,
4607	std_fileno: i32,
4608	dev_null_fd: fd_t,
4609	) !void {
4610	switch (stdio) {
4611	.pipe => try ev.dup2(sync, pipe_fd, std_fileno),
4612	.close => _ = linux.close(std_fileno),
4613	.inherit => {},
4614	.ignore => try ev.dup2(sync, dev_null_fd, std_fileno),
4615	.file => \|file\| {
4616	if (file.flags.nonblocking) @panic("TODO implement setUpChildIo when nonblocking file is used");
4617	try ev.dup2(sync, file.handle, std_fileno);
4618	},
4619	}
4620	}
4621
4622	pub const DupError = error{
4623	ProcessFdQuotaExceeded,
4624	SystemResources,
4625	} \|\| Io.UnexpectedError \|\| Io.Cancelable;
4626	pub fn dup2(ev: Evented, sync: CancelRegion.Sync, old_fd: fd_t, new_fd: fd_t) DupError!void {
4627	_ = ev;
4628	while (true) {
4629	try sync.cancel_region.await(.nothing);
4630	switch (linux.errno(linux.dup2(old_fd, new_fd))) {
4631	.SUCCESS => {},
4632	.BUSY, .INTR => continue,
4633	.INVAL => \|err\| return errnoBug(err), // invalid parameters
4634	.BADF => \|err\| return errnoBug(err), // use after free
4635	.MFILE => return error.ProcessFdQuotaExceeded,
4636	.NOMEM => return error.SystemResources,
4637	else => \|err\| return unexpectedErrno(err),
4638	}
4639	}
4640	}
4641
4642	fn execv(
4643	ev: *Evented,
4644	sync: *CancelRegion.Sync,
4645	arg0_expand: process.ArgExpansion,
4646	file: [*:0]const u8,
4647	child_argv: [:null]?[:0]const u8,
4648	env_block: process.Environ.PosixBlock,
4649	PATH: []const u8,
4650	) process.ReplaceError {
4651	const file_slice = std.mem.sliceTo(file, 0);
4652	if (std.mem.findScalar(u8, file_slice, '/') != null) return ev.execvPath(sync, file, child_argv, env_block);
4653
4654	// Use of PATH_MAX here is valid as the path_buf will be passed
4655	// directly to the operating system in posixExecvPath.
4656	var path_buf: [PATH_MAX]u8 = undefined;
4657	var it = std.mem.tokenizeScalar(u8, PATH, ':');
4658	var seen_eacces = false;
4659	var err: process.ReplaceError = error.FileNotFound;
4660
4661	// In case of expanding arg0 we must put it back if we return with an error.
4662	const prev_arg0 = child_argv[0];
4663	defer switch (arg0_expand) {
4664	.expand => child_argv[0] = prev_arg0,
4665	.no_expand => {},
4666	};
4667
4668	while (it.next()) \|search_path\| {
4669	const path_len = search_path.len + file_slice.len + 1;
4670	if (path_buf.len < path_len + 1) return error.NameTooLong;
4671	@memcpy(path_buf[0..search_path.len], search_path);
4672	path_buf[search_path.len] = '/';
4673	@memcpy(path_buf[search_path.len + 1 ..][0..file_slice.len], file_slice);
4674	path_buf[path_len] = 0;
4675	const full_path = path_buf[0..path_len :0].ptr;
4676	switch (arg0_expand) {
4677	.expand => child_argv[0] = full_path,
4678	.no_expand => {},
4679	}
4680	err = ev.execvPath(sync, full_path, child_argv, env_block);
4681	switch (err) {
4682	error.AccessDenied => seen_eacces = true,
4683	error.FileNotFound, error.NotDir => {},
4684	else => \|e\| return e,
4685	}
4686	}
4687	if (seen_eacces) return error.AccessDenied;
4688	return err;
4689	}
4690	/// This function ignores PATH environment variable.
4691	pub fn execvPath(
4692	ev: *Evented,
4693	sync: *CancelRegion.Sync,
4694	path: [*:0]const u8,
4695	child_argv: [:null]const ?[:0]const u8,
4696	env_block: process.Environ.PosixBlock,
4697	) process.ReplaceError {
4698	_ = ev;
4699	try sync.cancel_region.await(.nothing);
4700	switch (linux.errno(linux.execve(path, child_argv, env_block.slice.ptr))) {
4701	.FAULT => \|err\| return errnoBug(err), // Bad pointer parameter.
4702	.@"2BIG" => return error.SystemResources,
4703	.MFILE => return error.ProcessFdQuotaExceeded,
4704	.NAMETOOLONG => return error.NameTooLong,
4705	.NFILE => return error.SystemFdQuotaExceeded,
4706	.NOMEM => return error.SystemResources,
4707	.ACCES => return error.AccessDenied,
4708	.PERM => return error.PermissionDenied,
4709	.INVAL => return error.InvalidExe,
4710	.NOEXEC => return error.InvalidExe,
4711	.IO => return error.FileSystem,
4712	.LOOP => return error.FileSystem,
4713	.ISDIR => return error.IsDir,
4714	.NOENT => return error.FileNotFound,
4715	.NOTDIR => return error.NotDir,
4716	.TXTBSY => return error.FileBusy,
4717	.LIBBAD => return error.InvalidExe,
4718	else => \|err\| return unexpectedErrno(err),
4719	}
4720	}
4721
4722	fn childWait(userdata: ?anyopaque, child: process.Child) process.Child.WaitError!process.Child.Term {
4723	const ev: *Evented = @ptrCast(@alignCast(userdata));
4724
4725	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
4726	defer maybe_sync.deinit(ev);
4727	defer ev.childCleanup(child);
4728
4729	const pid = child.id.?;
4730	var info: linux.siginfo_t = undefined;
4731	while (true) {
4732	const thread = try maybe_sync.cancel_region.awaitIoUring();
4733	thread.enqueue().* = .{
4734	.opcode = .WAITID,
4735	.flags = 0,
4736	.ioprio = 0,
4737	.fd = pid,
4738	.off = @intFromPtr(&info),
4739	.addr = 0,
4740	.len = @intFromEnum(linux.P.PID),
4741	.rw_flags = 0,
4742	.user_data = @intFromPtr(maybe_sync.cancel_region.fiber),
4743	.buf_index = 0,
4744	.personality = 0,
4745	.splice_fd_in = linux.W.EXITED \|
4746	@as(i32, if (child.request_resource_usage_statistics) linux.W.NOWAIT else 0),
4747	.addr3 = 0,
4748	.resv = 0,
4749	};
4750	ev.yield(null, .nothing);
4751	switch (maybe_sync.cancel_region.errno()) {
4752	.SUCCESS => {
4753	if (child.request_resource_usage_statistics) {
4754	const sync = try maybe_sync.enterSync(ev);
4755	while (true) {
4756	try sync.cancel_region.await(.nothing);
4757	var rusage: linux.rusage = undefined;
4758	switch (linux.errno(linux.waitid(
4759	.PID,
4760	pid,
4761	&info,
4762	linux.W.EXITED \| linux.W.NOHANG,
4763	&rusage,
4764	))) {
4765	.SUCCESS => {
4766	child.resource_usage_statistics.rusage = rusage;
4767	break;
4768	},
4769	.INTR, .CANCELED => continue,
4770	.CHILD => \|err\| return errnoBug(err), // Double-free.
4771	else => \|err\| return unexpectedErrno(err),
4772	}
4773	}
4774	}
4775	const status: u32 = @bitCast(info.fields.common.second.sigchld.status);
4776	const code: linux.CLD = @enumFromInt(info.code);
4777	return switch (code) {
4778	.EXITED => .{ .exited = @truncate(status) },
4779	.KILLED, .DUMPED => .{ .signal = @enumFromInt(status) },
4780	.TRAPPED, .STOPPED => .{ .stopped = status },
4781	_, .CONTINUED => .{ .unknown = status },
4782	};
4783	},
4784	.INTR, .CANCELED => continue,
4785	.CHILD => \|err\| return errnoBug(err), // Double-free.
4786	else => \|err\| return unexpectedErrno(err),
4787	}
4788	}
4789	}
4790
4791	fn childKill(userdata: ?anyopaque, child: process.Child) void {
4792	const ev: *Evented = @ptrCast(@alignCast(userdata));
4793
4794	var maybe_sync: CancelRegion.Sync.Maybe = .{ .sync = .initBlocked(ev) };
4795	defer maybe_sync.deinit(ev);
4796	defer ev.childCleanup(child);
4797
4798	const pid = child.id.?;
4799	while (true) switch (linux.errno(linux.kill(pid, .TERM))) {
4800	.SUCCESS => break,
4801	.INTR => continue,
4802	.PERM => return,
4803	.INVAL => \|err\| return errnoBug(err) catch {},
4804	.SRCH => \|err\| return errnoBug(err) catch {},
4805	else => \|err\| return unexpectedErrno(err) catch {},
4806	};
4807	maybe_sync.leaveSync(ev);
4808
4809	var info: linux.siginfo_t = undefined;
4810	while (true) {
4811	const thread = maybe_sync.cancel_region.awaitIoUring() catch \|err\| switch (err) {
4812	error.Canceled => unreachable, // blocked
4813	};
4814	thread.enqueue().* = .{
4815	.opcode = .WAITID,
4816	.flags = 0,
4817	.ioprio = 0,
4818	.fd = pid,
4819	.off = @intFromPtr(&info),
4820	.addr = 0,
4821	.len = @intFromEnum(linux.P.PID),
4822	.rw_flags = 0,
4823	.user_data = @intFromPtr(maybe_sync.cancel_region.fiber),
4824	.buf_index = 0,
4825	.personality = 0,
4826	.splice_fd_in = linux.W.EXITED,
4827	.addr3 = 0,
4828	.resv = 0,
4829	};
4830	ev.yield(null, .nothing);
4831	switch (maybe_sync.cancel_region.errno()) {
4832	.SUCCESS => return,
4833	.INTR, .CANCELED => continue,
4834	.CHILD => \|err\| return errnoBug(err) catch {}, // Double-free.
4835	else => \|err\| return unexpectedErrno(err) catch {},
4836	}
4837	}
4838	}
4839
4840	fn childCleanup(ev: Evented, child: process.Child) void {
4841	if (child.stdin) \|*stdin\| {
4842	ev.close(stdin.handle);
4843	child.stdin = null;
4844	}
4845	if (child.stdout) \|*stdout\| {
4846	ev.close(stdout.handle);
4847	child.stdout = null;
4848	}
4849	if (child.stderr) \|*stderr\| {
4850	ev.close(stderr.handle);
4851	child.stderr = null;
4852	}
4853	child.id = null;
4854	}
4855
4856	fn progressParentFile(userdata: ?*anyopaque) std.Progress.ParentFileError!File {
4857	const ev: *Evented = @ptrCast(@alignCast(userdata));
4858	const cancel_protection = swapCancelProtection(ev, .blocked);
4859	defer assert(swapCancelProtection(ev, cancel_protection) == .blocked);
4860	ev.scanEnviron() catch \|err\| switch (err) {
4861	error.Canceled => unreachable, // blocked
4862	};
4863	return ev.environ.zig_progress_file;
4864	}
4865
4866	fn scanEnviron(ev: *Evented) Io.Cancelable!void {
4867	const ev_io = ev.io();
4868	try ev.environ_mutex.lock(ev_io);
4869	defer ev.environ_mutex.unlock(ev_io);
4870	if (ev.environ_initialized) return;
4871	ev.environ.scan(ev.allocator());
4872	ev.environ_initialized = true;
4873	}
4874
4875	fn clockResolution(userdata: ?*anyopaque, clock: Io.Clock) Io.Clock.ResolutionError!Io.Duration {
4876	const ev: *Evented = @ptrCast(@alignCast(userdata));
4877	_ = ev;
4878	const clock_id = clockToPosix(clock);
4879	var timespec: linux.timespec = undefined;
4880	return switch (linux.errno(linux.clock_getres(clock_id, &timespec))) {
4881	.SUCCESS => .fromNanoseconds(nanosecondsFromPosix(&timespec)),
4882	.INVAL => return error.ClockUnavailable,
4883	else => \|err\| return unexpectedErrno(err),
4884	};
4885	}
4886
4887	fn now(userdata: ?*anyopaque, clock: Io.Clock) Io.Timestamp {
4888	const ev: *Evented = @ptrCast(@alignCast(userdata));
4889	_ = ev;
4890	var tp: linux.timespec = undefined;
4891	switch (linux.errno(linux.clock_gettime(clockToPosix(clock), &tp))) {
4892	.SUCCESS => return timestampFromPosix(&tp),
4893	else => return .zero,
4894	}
4895	}
4896
4897	fn sleep(userdata: ?*anyopaque, timeout: Io.Timeout) Io.Cancelable!void {
4898	const ev: *Evented = @ptrCast(@alignCast(userdata));
4899
4900	const timespec: linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = timespec: switch (timeout) {
4901	.none => .{
4902	.{
4903	.sec = std.math.maxInt(i64),
4904	.nsec = std.time.ns_per_s - 1,
4905	},
4906	.awake,
4907	linux.IORING_TIMEOUT_ABS,
4908	},
4909	.duration => \|duration\| {
4910	const ns = duration.raw.toNanoseconds();
4911	break :timespec .{
4912	.{
4913	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
4914	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
4915	},
4916	duration.clock,
4917	0,
4918	};
4919	},
4920	.deadline => \|deadline\| {
4921	const ns = deadline.raw.toNanoseconds();
4922	break :timespec .{
4923	.{
4924	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
4925	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
4926	},
4927	deadline.clock,
4928	linux.IORING_TIMEOUT_ABS,
4929	};
4930	},
4931	};
4932	var cancel_region: CancelRegion = .init();
4933	defer cancel_region.deinit();
4934	const thread = try cancel_region.awaitIoUring();
4935	thread.enqueue().* = .{
4936	.opcode = .TIMEOUT,
4937	.flags = 0,
4938	.ioprio = 0,
4939	.fd = 0,
4940	.off = 0,
4941	.addr = @intFromPtr(&timespec),
4942	.len = 1,
4943	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
4944	.real => linux.IORING_TIMEOUT_REALTIME,
4945	else => 0,
4946	.boot => linux.IORING_TIMEOUT_BOOTTIME,
4947	}),
4948	.user_data = @intFromPtr(cancel_region.fiber),
4949	.buf_index = 0,
4950	.personality = 0,
4951	.splice_fd_in = 0,
4952	.addr3 = 0,
4953	.resv = 0,
4954	};
4955	ev.yield(null, .nothing);
4956	switch (cancel_region.errno()) {
4957	// Handles SUCCESS as well as clock not available and unexpected
4958	// errors. The user had a chance to check clock resolution before
4959	// getting here, which would have reported 0, making this a legal
4960	// amount of time to sleep.
4961	else => return,
4962	.INTR, .CANCELED => return error.Canceled,
4963	}
4964	}
4965
4966	fn random(userdata: ?*anyopaque, buffer: []u8) void {
4967	const ev: *Evented = @ptrCast(@alignCast(userdata));
4968	var thread: *Thread = .current();
4969	if (!thread.csprng.isInitialized()) {
4970	@branchHint(.unlikely);
4971	var seed: [Csprng.seed_len]u8 = undefined;
4972	{
4973	const ev_io = ev.io();
4974	ev.csprng_mutex.lockUncancelable(ev_io);
4975	defer ev.csprng_mutex.unlock(ev_io);
4976	if (!ev.csprng.isInitialized()) {
4977	@branchHint(.unlikely);
4978	var cancel_region: CancelRegion = .initBlocked();
4979	defer cancel_region.deinit();
4980	ev.urandomReadAll(&cancel_region, &seed) catch \|err\| switch (err) {
4981	error.Canceled => unreachable, // blocked
4982	else => fallbackSeed(ev, &seed),
4983	};
4984	ev.csprng.rng = .init(seed);
4985	thread = .current();
4986	}
4987	ev.csprng.rng.fill(&seed);
4988	}
4989	if (!thread.csprng.isInitialized()) {
4990	@branchHint(.likely);
4991	thread.csprng.rng = .init(seed);
4992	} else thread.csprng.rng.addEntropy(&seed);
4993	}
4994	thread.csprng.rng.fill(buffer);
4995	}
4996
4997	fn randomSecure(userdata: ?*anyopaque, buffer: []u8) Io.RandomSecureError!void {
4998	const ev: *Evented = @ptrCast(@alignCast(userdata));
4999	if (buffer.len == 0) return;
5000	var cancel_region: CancelRegion = .init();
5001	defer cancel_region.deinit();
5002	ev.urandomReadAll(&cancel_region, buffer) catch \|err\| switch (err) {
5003	error.Canceled => return error.Canceled,
5004	else => return error.EntropyUnavailable,
5005	};
5006	}
5007
5008	fn netListenIpUnavailable(
5009	userdata: ?*anyopaque,
5010	address: net.IpAddress,
5011	options: net.IpAddress.ListenOptions,
5012	) net.IpAddress.ListenError!net.Server {
5013	const ev: *Evented = @ptrCast(@alignCast(userdata));
5014	_ = ev;
5015	_ = address;
5016	_ = options;
5017	return error.NetworkDown;
5018	}
5019
5020	fn netAcceptUnavailable(
5021	userdata: ?*anyopaque,
5022	listen_handle: net.Socket.Handle,
5023	) net.Server.AcceptError!net.Stream {
5024	const ev: *Evented = @ptrCast(@alignCast(userdata));
5025	_ = ev;
5026	_ = listen_handle;
5027	return error.NetworkDown;
5028	}
5029
5030	fn netBindIp(
5031	userdata: ?*anyopaque,
5032	address: *const net.IpAddress,
5033	options: net.IpAddress.BindOptions,
5034	) net.IpAddress.BindError!net.Socket {
5035	const ev: *Evented = @ptrCast(@alignCast(userdata));
5036	const family = posixAddressFamily(address);
5037	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
5038	defer maybe_sync.deinit(ev);
5039	const socket_fd = try ev.socket(&maybe_sync.cancel_region, family, options);
5040	errdefer ev.close(socket_fd);
5041	var storage: PosixAddress = undefined;
5042	var addr_len = addressToPosix(address, &storage);
5043	try ev.bind(&maybe_sync.cancel_region, socket_fd, &storage.any, addr_len);
5044	try ev.getsockname(try maybe_sync.enterSync(ev), socket_fd, &storage.any, &addr_len);
5045	return .{
5046	.handle = socket_fd,
5047	.address = addressFromPosix(&storage),
5048	};
5049	}
5050
5051	fn netConnectIpUnavailable(
5052	userdata: ?*anyopaque,
5053	address: *const net.IpAddress,
5054	options: net.IpAddress.ConnectOptions,
5055	) net.IpAddress.ConnectError!net.Stream {
5056	const ev: *Evented = @ptrCast(@alignCast(userdata));
5057	_ = ev;
5058	_ = address;
5059	_ = options;
5060	return error.NetworkDown;
5061	}
5062
5063	fn netListenUnixUnavailable(
5064	userdata: ?*anyopaque,
5065	address: *const net.UnixAddress,
5066	options: net.UnixAddress.ListenOptions,
5067	) net.UnixAddress.ListenError!net.Socket.Handle {
5068	const ev: *Evented = @ptrCast(@alignCast(userdata));
5069	_ = ev;
5070	_ = address;
5071	_ = options;
5072	return error.AddressFamilyUnsupported;
5073	}
5074
5075	fn netConnectUnixUnavailable(
5076	userdata: ?*anyopaque,
5077	address: *const net.UnixAddress,
5078	) net.UnixAddress.ConnectError!net.Socket.Handle {
5079	const ev: *Evented = @ptrCast(@alignCast(userdata));
5080	_ = ev;
5081	_ = address;
5082	return error.AddressFamilyUnsupported;
5083	}
5084
5085	fn netSocketCreatePairUnavailable(
5086	userdata: ?*anyopaque,
5087	options: net.Socket.CreatePairOptions,
5088	) net.Socket.CreatePairError![2]net.Socket {
5089	_ = userdata;
5090	_ = options;
5091	return error.OperationUnsupported;
5092	}
5093
5094	fn netSendUnavailable(
5095	userdata: ?*anyopaque,
5096	handle: net.Socket.Handle,
5097	messages: []net.OutgoingMessage,
5098	flags: net.SendFlags,
5099	) struct { ?net.Socket.SendError, usize } {
5100	const ev: *Evented = @ptrCast(@alignCast(userdata));
5101	_ = ev;
5102	_ = handle;
5103	_ = messages;
5104	_ = flags;
5105	return .{ error.NetworkDown, 0 };
5106	}
5107
5108	fn netReceive(
5109	userdata: ?*anyopaque,
5110	handle: net.Socket.Handle,
5111	message_buffer: []net.IncomingMessage,
5112	data_buffer: []u8,
5113	flags: net.ReceiveFlags,
5114	timeout: Io.Timeout,
5115	) struct { ?net.Socket.ReceiveTimeoutError, usize } {
5116	const ev: *Evented = @ptrCast(@alignCast(userdata));
5117	const ev_io = ev.io();
5118
5119	var message_i: usize = 0;
5120	var data_i: usize = 0;
5121
5122	const deadline: ?struct {
5123	raw: Io.Timestamp,
5124	timespec: linux.kernel_timespec,
5125	clock: Io.Clock,
5126	} = if (timeout.toTimestamp(ev_io)) \|deadline\| deadline: {
5127	const ns = deadline.raw.toNanoseconds();
5128	break :deadline .{
5129	.raw = deadline.raw,
5130	.timespec = .{
5131	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
5132	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
5133	},
5134	.clock = deadline.clock,
5135	};
5136	} else null;
5137
5138	var cancel_region: CancelRegion = .init();
5139	defer cancel_region.deinit();
5140	while (true) {
5141	if (message_buffer.len - message_i == 0) return .{ null, message_i };
5142	const message = &message_buffer[message_i];
5143	const remaining_data_buffer = data_buffer[data_i..];
5144	var storage: PosixAddress = undefined;
5145	var iov: iovec = .{ .base = remaining_data_buffer.ptr, .len = remaining_data_buffer.len };
5146	var msg: linux.msghdr = .{
5147	.name = &storage.any,
5148	.namelen = @sizeOf(PosixAddress),
5149	.iov = (&iov)[0..1],
5150	.iovlen = 1,
5151	.control = message.control.ptr,
5152	.controllen = @intCast(message.control.len),
5153	.flags = undefined,
5154	};
5155
5156	const thread = cancel_region.awaitIoUring() catch \|err\| return .{ err, message_i };
5157	thread.enqueue().* = .{
5158	.opcode = .RECVMSG,
5159	.flags = if (deadline) \|_\| linux.IOSQE_IO_LINK else 0,
5160	.ioprio = 0,
5161	.fd = handle,
5162	.off = 0,
5163	.addr = @intFromPtr(&msg),
5164	.len = 0,
5165	.rw_flags = linux.MSG.NOSIGNAL \|
5166	@as(u32, if (flags.oob) linux.MSG.OOB else 0) \|
5167	@as(u32, if (flags.peek) linux.MSG.PEEK else 0) \|
5168	@as(u32, if (flags.trunc) linux.MSG.TRUNC else 0),
5169	.user_data = @intFromPtr(cancel_region.fiber),
5170	.buf_index = 0,
5171	.personality = 0,
5172	.splice_fd_in = 0,
5173	.addr3 = 0,
5174	.resv = 0,
5175	};
5176	if (deadline) \|deadline_ptr\| thread.enqueue(). = .{
5177	.opcode = .LINK_TIMEOUT,
5178	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
5179	.ioprio = 0,
5180	.fd = 0,
5181	.off = 0,
5182	.addr = @intFromPtr(&deadline_ptr.timespec),
5183	.len = 1,
5184	.rw_flags = linux.IORING_TIMEOUT_ABS \| @as(u32, switch (deadline_ptr.clock) {
5185	.real => linux.IORING_TIMEOUT_REALTIME,
5186	else => 0,
5187	.boot => linux.IORING_TIMEOUT_BOOTTIME,
5188	}),
5189	.user_data = @intFromEnum(Completion.Userdata.wakeup),
5190	.buf_index = 0,
5191	.personality = 0,
5192	.splice_fd_in = 0,
5193	.addr3 = 0,
5194	.resv = 0,
5195	};
5196	ev.yield(null, .nothing);
5197	const completion = cancel_region.completion();
5198	switch (completion.errno()) {
5199	.SUCCESS => {
5200	const data = remaining_data_buffer[0..@intCast(completion.result)];
5201	data_i += data.len;
5202	message.* = .{
5203	.from = addressFromPosix(&storage),
5204	.data = data,
5205	.control = if (msg.control) \|ptr\| @as([*]u8, @ptrCast(ptr))[0..msg.controllen] else message.control,
5206	.flags = .{
5207	.eor = (msg.flags & linux.MSG.EOR) != 0,
5208	.trunc = (msg.flags & linux.MSG.TRUNC) != 0,
5209	.ctrunc = (msg.flags & linux.MSG.CTRUNC) != 0,
5210	.oob = (msg.flags & linux.MSG.OOB) != 0,
5211	.errqueue = if (@hasDecl(linux.MSG, "ERRQUEUE")) (msg.flags & linux.MSG.ERRQUEUE) != 0 else false,
5212	},
5213	};
5214	message_i += 1;
5215	continue;
5216	},
5217	.AGAIN => unreachable,
5218	.INTR, .CANCELED => {
5219	if (deadline) \|d\| {
5220	if (now(ev, d.clock).nanoseconds >= d.raw.nanoseconds) return .{ error.Timeout, message_i };
5221	}
5222	continue;
5223	},
5224
5225	.BADF => \|err\| return .{ errnoBug(err), message_i },
5226	.NFILE => return .{ error.SystemFdQuotaExceeded, message_i },
5227	.MFILE => return .{ error.ProcessFdQuotaExceeded, message_i },
5228	.FAULT => \|err\| return .{ errnoBug(err), message_i },
5229	.INVAL => \|err\| return .{ errnoBug(err), message_i },
5230	.NOBUFS => return .{ error.SystemResources, message_i },
5231	.NOMEM => return .{ error.SystemResources, message_i },
5232	.NOTCONN => return .{ error.SocketUnconnected, message_i },
5233	.NOTSOCK => \|err\| return .{ errnoBug(err), message_i },
5234	.MSGSIZE => return .{ error.MessageOversize, message_i },
5235	.PIPE => return .{ error.SocketUnconnected, message_i },
5236	.OPNOTSUPP => \|err\| return .{ errnoBug(err), message_i },
5237	.CONNRESET => return .{ error.ConnectionResetByPeer, message_i },
5238	.NETDOWN => return .{ error.NetworkDown, message_i },
5239	else => \|err\| return .{ unexpectedErrno(err), message_i },
5240	}
5241	}
5242	}
5243
5244	fn netReadUnavailable(
5245	userdata: ?*anyopaque,
5246	fd: net.Socket.Handle,
5247	data: [][]u8,
5248	) net.Stream.Reader.Error!usize {
5249	const ev: *Evented = @ptrCast(@alignCast(userdata));
5250	_ = ev;
5251	_ = fd;
5252	_ = data;
5253	return error.NetworkDown;
5254	}
5255
5256	fn netWriteUnavailable(
5257	userdata: ?*anyopaque,
5258	handle: net.Socket.Handle,
5259	header: []const u8,
5260	data: []const []const u8,
5261	splat: usize,
5262	) net.Stream.Writer.Error!usize {
5263	const ev: *Evented = @ptrCast(@alignCast(userdata));
5264	_ = ev;
5265	_ = handle;
5266	_ = header;
5267	_ = data;
5268	_ = splat;
5269	return error.NetworkDown;
5270	}
5271
5272	fn netWriteFileUnavailable(
5273	userdata: ?*anyopaque,
5274	socket_handle: net.Socket.Handle,
5275	header: []const u8,
5276	file_reader: *File.Reader,
5277	limit: Io.Limit,
5278	) net.Stream.Writer.WriteFileError!usize {
5279	const ev: *Evented = @ptrCast(@alignCast(userdata));
5280	_ = ev;
5281	_ = socket_handle;
5282	_ = header;
5283	_ = file_reader;
5284	_ = limit;
5285	return error.NetworkDown;
5286	}
5287
5288	fn netClose(userdata: ?*anyopaque, handles: []const net.Socket.Handle) void {
5289	const ev: *Evented = @ptrCast(@alignCast(userdata));
5290	for (handles) \|handle\| ev.close(handle);
5291	}
5292
5293	fn netShutdown(
5294	userdata: ?*anyopaque,
5295	handle: net.Socket.Handle,
5296	how: net.ShutdownHow,
5297	) net.ShutdownError!void {
5298	const ev: *Evented = @ptrCast(@alignCast(userdata));
5299	var cancel_region: CancelRegion = .init();
5300	defer cancel_region.deinit();
5301	while (true) {
5302	const thread = try cancel_region.awaitIoUring();
5303	thread.enqueue().* = .{
5304	.opcode = .SHUTDOWN,
5305	.flags = 0,
5306	.ioprio = 0,
5307	.fd = handle,
5308	.off = 0,
5309	.addr = 0,
5310	.len = switch (how) {
5311	.recv => linux.SHUT.RD,
5312	.send => linux.SHUT.WR,
5313	.both => linux.SHUT.RDWR,
5314	},
5315	.rw_flags = 0,
5316	.user_data = @intFromPtr(cancel_region.fiber),
5317	.buf_index = 0,
5318	.personality = 0,
5319	.splice_fd_in = 0,
5320	.addr3 = 0,
5321	.resv = 0,
5322	};
5323	ev.yield(null, .nothing);
5324	switch (cancel_region.errno()) {
5325	.SUCCESS => return,
5326	.INTR, .CANCELED => continue,
5327	.BADF, .NOTSOCK, .INVAL => \|err\| return errnoBug(err),
5328	.NOTCONN => return error.SocketUnconnected,
5329	.NOBUFS => return error.SystemResources,
5330	else => \|err\| return unexpectedErrno(err),
5331	}
5332	}
5333	}
5334
5335	fn netInterfaceNameResolveUnavailable(
5336	userdata: ?*anyopaque,
5337	name: *const net.Interface.Name,
5338	) net.Interface.Name.ResolveError!net.Interface {
5339	const ev: *Evented = @ptrCast(@alignCast(userdata));
5340	_ = ev;
5341	_ = name;
5342	return error.InterfaceNotFound;
5343	}
5344
5345	fn netInterfaceNameUnavailable(
5346	userdata: ?*anyopaque,
5347	interface: net.Interface,
5348	) net.Interface.NameError!net.Interface.Name {
5349	const ev: *Evented = @ptrCast(@alignCast(userdata));
5350	_ = ev;
5351	_ = interface;
5352	return error.Unexpected;
5353	}
5354
5355	fn netLookupUnavailable(
5356	userdata: ?*anyopaque,
5357	host_name: net.HostName,
5358	resolved: *Io.Queue(net.HostName.LookupResult),
5359	options: net.HostName.LookupOptions,
5360	) net.HostName.LookupError!void {
5361	const ev: *Evented = @ptrCast(@alignCast(userdata));
5362	_ = host_name;
5363	_ = options;
5364	resolved.close(ev.io());
5365	return error.NetworkDown;
5366	}
5367
5368	fn bind(
5369	ev: *Evented,
5370	cancel_region: *CancelRegion,
5371	socket_fd: fd_t,
5372	addr: *const linux.sockaddr,
5373	addr_len: linux.socklen_t,
5374	) !void {
5375	while (true) {
5376	const thread = try cancel_region.awaitIoUring();
5377	thread.enqueue().* = .{
5378	.opcode = .BIND,
5379	.flags = 0,
5380	.ioprio = 0,
5381	.fd = socket_fd,
5382	.off = addr_len,
5383	.addr = @intFromPtr(addr),
5384	.len = 0,
5385	.rw_flags = 0,
5386	.user_data = @intFromPtr(cancel_region.fiber),
5387	.buf_index = 0,
5388	.personality = 0,
5389	.splice_fd_in = 0,
5390	.addr3 = 0,
5391	.resv = 0,
5392	};
5393	ev.yield(null, .nothing);
5394	switch (cancel_region.errno()) {
5395	.SUCCESS => return,
5396	.INTR, .CANCELED => continue,
5397	.ADDRINUSE => return error.AddressInUse,
5398	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
5399	.INVAL => \|err\| return errnoBug(err), // invalid parameters
5400	.NOTSOCK => \|err\| return errnoBug(err), // invalid `sockfd`
5401	.AFNOSUPPORT => return error.AddressFamilyUnsupported,
5402	.ADDRNOTAVAIL => return error.AddressUnavailable,
5403	.FAULT => \|err\| return errnoBug(err), // invalid `addr` pointer
5404	.NOMEM => return error.SystemResources,
5405	else => \|err\| return unexpectedErrno(err),
5406	}
5407	}
5408	}
5409
5410	fn chdir(ev: Evented, sync: CancelRegion.Sync, path: [*:0]const u8) ChdirError!void {
5411	_ = ev;
5412	while (true) {
5413	try sync.cancel_region.await(.nothing);
5414	switch (linux.errno(linux.chdir(path))) {
5415	.SUCCESS => return,
5416	.INTR => continue,
5417	.ACCES => return error.AccessDenied,
5418	.IO => return error.FileSystem,
5419	.LOOP => return error.SymLinkLoop,
5420	.NAMETOOLONG => return error.NameTooLong,
5421	.NOENT => return error.FileNotFound,
5422	.NOMEM => return error.SystemResources,
5423	.NOTDIR => return error.NotDir,
5424	.ILSEQ => return error.BadPathName,
5425	.FAULT => \|err\| return errnoBug(err),
5426	else => \|err\| return unexpectedErrno(err),
5427	}
5428	}
5429	}
5430
5431	fn close(ev: *Evented, fd: fd_t) void {
5432	_ = ev;
5433	const thread: *Thread = .current();
5434	thread.enqueue().* = .{
5435	.opcode = .CLOSE,
5436	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
5437	.ioprio = 0,
5438	.fd = fd,
5439	.off = 0,
5440	.addr = 0,
5441	.len = 0,
5442	.rw_flags = 0,
5443	.user_data = @intFromEnum(Completion.Userdata.close),
5444	.buf_index = 0,
5445	.personality = 0,
5446	.splice_fd_in = 0,
5447	.addr3 = 0,
5448	.resv = 0,
5449	};
5450	}
5451
5452	fn fchdir(ev: Evented, sync: CancelRegion.Sync, dir: fd_t) process.SetCurrentDirError!void {
5453	_ = ev;
5454	if (dir == linux.AT.FDCWD) return;
5455	while (true) {
5456	try sync.cancel_region.await(.nothing);
5457	switch (linux.errno(linux.fchdir(dir))) {
5458	.SUCCESS => return,
5459	.INTR => continue,
5460	.ACCES => return error.AccessDenied,
5461	.NOTDIR => return error.NotDir,
5462	.IO => return error.FileSystem,
5463	.BADF => \|err\| return errnoBug(err),
5464	else => \|err\| return unexpectedErrno(err),
5465	}
5466	}
5467	}
5468
5469	fn fchmodat(
5470	ev: *Evented,
5471	sync: *CancelRegion.Sync,
5472	dir: fd_t,
5473	path: [*:0]const u8,
5474	mode: linux.mode_t,
5475	flags: u32,
5476	) Dir.SetFilePermissionsError!void {
5477	_ = ev;
5478	while (true) {
5479	try sync.cancel_region.await(.nothing);
5480	switch (linux.errno(linux.fchmodat2(dir, path, mode, flags))) {
5481	.SUCCESS => return,
5482	.INTR => continue,
5483	.BADF => \|err\| return errnoBug(err),
5484	.FAULT => \|err\| return errnoBug(err),
5485	.INVAL => \|err\| return errnoBug(err),
5486	.ACCES => return error.AccessDenied,
5487	.IO => return error.InputOutput,
5488	.LOOP => return error.SymLinkLoop,
5489	.NOENT => return error.FileNotFound,
5490	.NOMEM => return error.SystemResources,
5491	.NOTDIR => return error.FileNotFound,
5492	.OPNOTSUPP => return error.OperationUnsupported,
5493	.PERM => return error.PermissionDenied,
5494	.ROFS => return error.ReadOnlyFileSystem,
5495	else => \|err\| return unexpectedErrno(err),
5496	}
5497	}
5498	}
5499
5500	fn fchownat(
5501	ev: *Evented,
5502	sync: *CancelRegion.Sync,
5503	dir: fd_t,
5504	path: [*:0]const u8,
5505	owner: linux.uid_t,
5506	group: linux.gid_t,
5507	flags: u32,
5508	) File.SetOwnerError!void {
5509	_ = ev;
5510	while (true) {
5511	try sync.cancel_region.await(.nothing);
5512	switch (linux.errno(linux.fchownat(dir, path, owner, group, flags))) {
5513	.SUCCESS => return,
5514	.INTR => continue,
5515	.BADF => \|err\| return errnoBug(err), // likely fd refers to directory opened without `Dir.OpenOptions.iterate`
5516	.FAULT => \|err\| return errnoBug(err),
5517	.INVAL => \|err\| return errnoBug(err),
5518	.ACCES => return error.AccessDenied,
5519	.IO => return error.InputOutput,
5520	.LOOP => return error.SymLinkLoop,
5521	.NOENT => return error.FileNotFound,
5522	.NOMEM => return error.SystemResources,
5523	.NOTDIR => return error.FileNotFound,
5524	.PERM => return error.PermissionDenied,
5525	.ROFS => return error.ReadOnlyFileSystem,
5526	else => \|err\| return unexpectedErrno(err),
5527	}
5528	}
5529	}
5530
5531	fn flock(
5532	ev: *Evented,
5533	sync: *CancelRegion.Sync,
5534	fd: fd_t,
5535	op: File.Lock,
5536	blocking: enum { blocking, nonblocking },
5537	) (File.LockError \|\| error{WouldBlock})!void {
5538	while (true) {
5539	try sync.cancel_region.await(.nothing);
5540	switch (linux.errno(linux.flock(fd, LOCK.NB \| @as(i32, switch (op) {
5541	.none => LOCK.UN,
5542	.shared => LOCK.SH,
5543	.exclusive => LOCK.EX,
5544	})))) {
5545	.SUCCESS => return,
5546	.INTR => continue,
5547	.BADF => \|err\| return errnoBug(err),
5548	.INVAL => \|err\| return errnoBug(err), // invalid parameters
5549	.NOLCK => return error.SystemResources,
5550	.AGAIN => {
5551	const thread = try sync.cancel_region.awaitIoUring();
5552	thread.enqueue().* = .{
5553	.opcode = .NOP,
5554	.flags = 0,
5555	.ioprio = 0,
5556	.fd = 0,
5557	.off = 0,
5558	.addr = 0,
5559	.len = 0,
5560	.rw_flags = 0,
5561	.user_data = @intFromPtr(sync.cancel_region.fiber),
5562	.buf_index = 0,
5563	.personality = 0,
5564	.splice_fd_in = 0,
5565	.addr3 = 0,
5566	.resv = 0,
5567	};
5568	ev.yield(null, .nothing);
5569	switch (sync.cancel_region.errno()) {
5570	.SUCCESS, .INTR, .CANCELED => {},
5571	else => unreachable,
5572	}
5573	switch (blocking) {
5574	.blocking => continue,
5575	.nonblocking => return error.WouldBlock,
5576	}
5577	},
5578	.OPNOTSUPP => return error.FileLocksUnsupported,
5579	else => \|err\| return unexpectedErrno(err),
5580	}
5581	}
5582	}
5583
5584	fn getsockname(
5585	ev: *Evented,
5586	sync: *CancelRegion.Sync,
5587	socket_fd: fd_t,
5588	addr: *linux.sockaddr,
5589	addr_len: *linux.socklen_t,
5590	) !void {
5591	_ = ev;
5592	while (true) {
5593	try sync.cancel_region.await(.nothing);
5594	switch (linux.errno(linux.getsockname(socket_fd, addr, addr_len))) {
5595	.SUCCESS => return,
5596	.INTR => continue,
5597	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
5598	.FAULT => \|err\| return errnoBug(err),
5599	.INVAL => \|err\| return errnoBug(err), // invalid parameters
5600	.NOTSOCK => \|err\| return errnoBug(err), // always a race condition
5601	.NOBUFS => return error.SystemResources,
5602	else => \|err\| return unexpectedErrno(err),
5603	}
5604	}
5605	}
5606
5607	fn linkat(
5608	ev: *Evented,
5609	cancel_region: *CancelRegion,
5610	old_dir: fd_t,
5611	old_path: [*:0]const u8,
5612	new_dir: fd_t,
5613	new_path: [*:0]const u8,
5614	flags: u32,
5615	) File.HardLinkError!void {
5616	while (true) {
5617	const thread = try cancel_region.awaitIoUring();
5618	thread.enqueue().* = .{
5619	.opcode = .LINKAT,
5620	.flags = 0,
5621	.ioprio = 0,
5622	.fd = old_dir,
5623	.off = @intFromPtr(new_path),
5624	.addr = @intFromPtr(old_path),
5625	.len = @bitCast(new_dir),
5626	.rw_flags = flags,
5627	.user_data = @intFromPtr(cancel_region.fiber),
5628	.buf_index = 0,
5629	.personality = 0,
5630	.splice_fd_in = 0,
5631	.addr3 = 0,
5632	.resv = 0,
5633	};
5634	ev.yield(null, .nothing);
5635	switch (cancel_region.errno()) {
5636	.SUCCESS => return,
5637	.INTR, .CANCELED => continue,
5638	.ACCES => return error.AccessDenied,
5639	.DQUOT => return error.DiskQuota,
5640	.EXIST => return error.PathAlreadyExists,
5641	.IO => return error.HardwareFailure,
5642	.LOOP => return error.SymLinkLoop,
5643	.MLINK => return error.LinkQuotaExceeded,
5644	.NAMETOOLONG => return error.NameTooLong,
5645	.NOENT => return error.FileNotFound,
5646	.NOMEM => return error.SystemResources,
5647	.NOSPC => return error.NoSpaceLeft,
5648	.NOTDIR => return error.NotDir,
5649	.PERM => return error.PermissionDenied,
5650	.ROFS => return error.ReadOnlyFileSystem,
5651	.XDEV => return error.CrossDevice,
5652	.ILSEQ => return error.BadPathName,
5653	.FAULT => \|err\| return errnoBug(err),
5654	.INVAL => \|err\| return errnoBug(err),
5655	else => \|err\| return unexpectedErrno(err),
5656	}
5657	}
5658	}
5659
5660	fn lseek(
5661	ev: *Evented,
5662	sync: *CancelRegion.Sync,
5663	fd: fd_t,
5664	offset: u64,
5665	whence: u32,
5666	) File.SeekError!void {
5667	_ = ev;
5668	while (true) {
5669	try sync.cancel_region.await(.nothing);
5670	var result: u64 = undefined;
5671	switch (linux.errno(switch (@sizeOf(usize)) {
5672	else => comptime unreachable,
5673	4 => linux.llseek(fd, offset, &result, whence),
5674	8 => linux.lseek(fd, @bitCast(offset), whence),
5675	})) {
5676	.SUCCESS => return,
5677	.INTR => continue,
5678	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
5679	.INVAL => return error.Unseekable,
5680	.OVERFLOW => return error.Unseekable,
5681	.SPIPE => return error.Unseekable,
5682	.NXIO => return error.Unseekable,
5683	else => \|err\| return unexpectedErrno(err),
5684	}
5685	}
5686	}
5687
5688	fn openat(
5689	ev: *Evented,
5690	cancel_region: *CancelRegion,
5691	dir: fd_t,
5692	path: [*:0]const u8,
5693	flags: linux.O,
5694	mode: linux.mode_t,
5695	) File.OpenError!fd_t {
5696	var mut_flags = flags;
5697	if (@hasField(linux.O, "LARGEFILE")) mut_flags.LARGEFILE = true;
5698	while (true) {
5699	const thread = try cancel_region.awaitIoUring();
5700	thread.enqueue().* = .{
5701	.opcode = .OPENAT,
5702	.flags = 0,
5703	.ioprio = 0,
5704	.fd = dir,
5705	.off = 0,
5706	.addr = @intFromPtr(path),
5707	.len = mode,
5708	.rw_flags = @bitCast(mut_flags),
5709	.user_data = @intFromPtr(cancel_region.fiber),
5710	.buf_index = 0,
5711	.personality = 0,
5712	.splice_fd_in = 0,
5713	.addr3 = 0,
5714	.resv = 0,
5715	};
5716	ev.yield(null, .nothing);
5717	const completion = cancel_region.completion();
5718	switch (completion.errno()) {
5719	.SUCCESS => return completion.result,
5720	.INTR, .CANCELED => continue,
5721	.FAULT => \|err\| return errnoBug(err),
5722	.INVAL => return error.BadPathName,
5723	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
5724	.ACCES => return error.AccessDenied,
5725	.FBIG => return error.FileTooBig,
5726	.OVERFLOW => return error.FileTooBig,
5727	.ISDIR => return error.IsDir,
5728	.LOOP => return error.SymLinkLoop,
5729	.MFILE => return error.ProcessFdQuotaExceeded,
5730	.NAMETOOLONG => return error.NameTooLong,
5731	.NFILE => return error.SystemFdQuotaExceeded,
5732	.NODEV => return error.NoDevice,
5733	.NOENT => return error.FileNotFound,
5734	.SRCH => return error.FileNotFound, // Linux when opening procfs files.
5735	.NOMEM => return error.SystemResources,
5736	.NOSPC => return error.NoSpaceLeft,
5737	.NOTDIR => return error.NotDir,
5738	.PERM => return error.PermissionDenied,
5739	.EXIST => return error.PathAlreadyExists,
5740	.BUSY => return error.DeviceBusy,
5741	.OPNOTSUPP => return error.FileLocksUnsupported,
5742	.AGAIN => return error.WouldBlock,
5743	.TXTBSY => return error.FileBusy,
5744	.NXIO => return error.NoDevice,
5745	.ILSEQ => return error.BadPathName,
5746	else => \|err\| return unexpectedErrno(err),
5747	}
5748	}
5749	}
5750
5751	fn preadv(
5752	ev: *Evented,
5753	cancel_region: *CancelRegion,
5754	fd: fd_t,
5755	iov: []const iovec,
5756	offset: ?u64,
5757	) File.Reader.Error!usize {
5758	if (iov.len == 0) return 0;
5759	const gather = iov.len > 1 or iov[0].len > 0xfffff000;
5760	while (true) {
5761	const thread = try cancel_region.awaitIoUring();
5762	thread.enqueue().* = .{
5763	.opcode = if (gather) .READV else .READ,
5764	.flags = 0,
5765	.ioprio = 0,
5766	.fd = fd,
5767	.off = offset orelse std.math.maxInt(u64),
5768	.addr = if (gather) @intFromPtr(iov.ptr) else @intFromPtr(iov[0].base),
5769	.len = @intCast(if (gather) iov.len else iov[0].len),
5770	.rw_flags = 0,
5771	.user_data = @intFromPtr(cancel_region.fiber),
5772	.buf_index = 0,
5773	.personality = 0,
5774	.splice_fd_in = 0,
5775	.addr3 = 0,
5776	.resv = 0,
5777	};
5778	ev.yield(null, .nothing);
5779	const completion = cancel_region.completion();
5780	switch (completion.errno()) {
5781	.SUCCESS => return @as(u32, @bitCast(completion.result)),
5782	.INTR, .CANCELED => continue,
5783	.INVAL => \|err\| return errnoBug(err),
5784	.FAULT => \|err\| return errnoBug(err),
5785	.AGAIN => return error.WouldBlock,
5786	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed
5787	.IO => return error.InputOutput,
5788	.ISDIR => return error.IsDir,
5789	.NOBUFS => return error.SystemResources,
5790	.NOMEM => return error.SystemResources,
5791	.NOTCONN => return error.SocketUnconnected,
5792	.CONNRESET => return error.ConnectionResetByPeer,
5793	else => \|err\| return unexpectedErrno(err),
5794	}
5795	}
5796	}
5797
5798	fn pwritev(
5799	ev: *Evented,
5800	cancel_region: *CancelRegion,
5801	fd: fd_t,
5802	iov: []const iovec_const,
5803	offset: ?u64,
5804	) File.Writer.Error!usize {
5805	if (iov.len == 0) return 0;
5806	const scatter = iov.len > 1 or iov[0].len > 0xfffff000;
5807	while (true) {
5808	const thread = try cancel_region.awaitIoUring();
5809	thread.enqueue().* = .{
5810	.opcode = if (scatter) .WRITEV else .WRITE,
5811	.flags = 0,
5812	.ioprio = 0,
5813	.fd = fd,
5814	.off = offset orelse std.math.maxInt(u64),
5815	.addr = if (scatter) @intFromPtr(iov.ptr) else @intFromPtr(iov[0].base),
5816	.len = @intCast(if (scatter) iov.len else iov[0].len),
5817	.rw_flags = 0,
5818	.user_data = @intFromPtr(cancel_region.fiber),
5819	.buf_index = 0,
5820	.personality = 0,
5821	.splice_fd_in = 0,
5822	.addr3 = 0,
5823	.resv = 0,
5824	};
5825	ev.yield(null, .nothing);
5826	const completion = cancel_region.completion();
5827	switch (completion.errno()) {
5828	.SUCCESS => return @as(u32, @bitCast(completion.result)),
5829	.INTR, .CANCELED => continue,
5830	.INVAL => \|err\| return errnoBug(err),
5831	.FAULT => \|err\| return errnoBug(err),
5832	.AGAIN => return error.WouldBlock,
5833	.BADF => return error.NotOpenForWriting, // Can be a race condition.
5834	.DESTADDRREQ => \|err\| return errnoBug(err), // `connect` was never called.
5835	.DQUOT => return error.DiskQuota,
5836	.FBIG => return error.FileTooBig,
5837	.IO => return error.InputOutput,
5838	.NOSPC => return error.NoSpaceLeft,
5839	.PERM => return error.PermissionDenied,
5840	.PIPE => return error.BrokenPipe,
5841	.CONNRESET => \|err\| return errnoBug(err), // Not a socket handle.
5842	.BUSY => return error.DeviceBusy,
5843	else => \|err\| return unexpectedErrno(err),
5844	}
5845	}
5846	}
5847
5848	fn readAll(
5849	ev: *Evented,
5850	cancel_region: *CancelRegion,
5851	fd: fd_t,
5852	buffer: []u8,
5853	) (File.Reader.Error \|\| error{EndOfStream})!void {
5854	var index: usize = 0;
5855	while (buffer.len - index != 0) {
5856	const len = try ev.preadv(cancel_region, fd, &.{
5857	.{ .base = buffer[index..].ptr, .len = buffer.len - index },
5858	}, null);
5859	if (len == 0) return error.EndOfStream;
5860	index += len;
5861	}
5862	}
5863
5864	fn realPath(
5865	ev: *Evented,
5866	sync: *CancelRegion.Sync,
5867	fd: fd_t,
5868	out_buffer: []u8,
5869	) File.RealPathError!usize {
5870	_ = ev;
5871	var procfs_buf: [std.fmt.count("/proc/self/fd/{d}\x00", .{std.math.minInt(fd_t)})]u8 = undefined;
5872	const proc_path = std.fmt.bufPrintSentinel(&procfs_buf, "/proc/self/fd/{d}", .{fd}, 0) catch
5873	unreachable;
5874	while (true) {
5875	try sync.cancel_region.await(.nothing);
5876	const rc = linux.readlink(proc_path, out_buffer.ptr, out_buffer.len);
5877	switch (linux.errno(rc)) {
5878	.SUCCESS => return rc,
5879	.INTR => continue,
5880	.ACCES => return error.AccessDenied,
5881	.FAULT => \|err\| return errnoBug(err),
5882	.IO => return error.FileSystem,
5883	.LOOP => return error.SymLinkLoop,
5884	.NAMETOOLONG => return error.NameTooLong,
5885	.NOENT => return error.FileNotFound,
5886	.NOMEM => return error.SystemResources,
5887	.NOTDIR => return error.NotDir,
5888	.ILSEQ => \|err\| return errnoBug(err),
5889	else => \|err\| return unexpectedErrno(err),
5890	}
5891	}
5892	}
5893
5894	fn renameat(
5895	ev: *Evented,
5896	cancel_region: *CancelRegion,
5897	old_dir: fd_t,
5898	old_path: [*:0]const u8,
5899	new_dir: fd_t,
5900	new_path: [*:0]const u8,
5901	flags: linux.RENAME,
5902	) Dir.RenameError!void {
5903	while (true) {
5904	const thread = try cancel_region.awaitIoUring();
5905	thread.enqueue().* = .{
5906	.opcode = .RENAMEAT,
5907	.flags = 0,
5908	.ioprio = 0,
5909	.fd = old_dir,
5910	.off = @intFromPtr(new_path),
5911	.addr = @intFromPtr(old_path),
5912	.len = @bitCast(new_dir),
5913	.rw_flags = @bitCast(flags),
5914	.user_data = @intFromPtr(cancel_region.fiber),
5915	.buf_index = 0,
5916	.personality = 0,
5917	.splice_fd_in = 0,
5918	.addr3 = 0,
5919	.resv = 0,
5920	};
5921	ev.yield(null, .nothing);
5922	switch (cancel_region.errno()) {
5923	.SUCCESS => return,
5924	.INTR, .CANCELED => continue,
5925	.ACCES => return error.AccessDenied,
5926	.PERM => return error.PermissionDenied,
5927	.BUSY => return error.FileBusy,
5928	.DQUOT => return error.DiskQuota,
5929	.ISDIR => return error.IsDir,
5930	.IO => return error.HardwareFailure,
5931	.LOOP => return error.SymLinkLoop,
5932	.MLINK => return error.LinkQuotaExceeded,
5933	.NAMETOOLONG => return error.NameTooLong,
5934	.NOENT => return error.FileNotFound,
5935	.NOTDIR => return error.NotDir,
5936	.NOMEM => return error.SystemResources,
5937	.NOSPC => return error.NoSpaceLeft,
5938	.EXIST => return error.DirNotEmpty,
5939	.NOTEMPTY => return error.DirNotEmpty,
5940	.ROFS => return error.ReadOnlyFileSystem,
5941	.XDEV => return error.CrossDevice,
5942	.ILSEQ => return error.BadPathName,
5943	.FAULT => \|err\| return errnoBug(err),
5944	.INVAL => \|err\| return errnoBug(err),
5945	else => \|err\| return unexpectedErrno(err),
5946	}
5947	}
5948	}
5949
5950	fn setsockopt(
5951	ev: *Evented,
5952	cancel_region: *CancelRegion,
5953	fd: fd_t,
5954	level: i32,
5955	opt_name: u32,
5956	option: u32,
5957	) !void {
5958	const o: []const u8 = @ptrCast(&option);
5959	while (true) {
5960	const off: extern struct {
5961	cmd_op: linux.IO_URING_SOCKET_OP,
5962	pad: u32,
5963	} align(@alignOf(u64)) = .{
5964	.cmd_op = .SETSOCKOPT,
5965	.pad = 0,
5966	};
5967	const addr: extern struct { level: i32, opt_name: u32 } align(@alignOf(u64)) = .{
5968	.level = level,
5969	.opt_name = opt_name,
5970	};
5971	const thread = try cancel_region.awaitIoUring();
5972	thread.enqueue().* = .{
5973	.opcode = .URING_CMD,
5974	.flags = 0,
5975	.ioprio = 0,
5976	.fd = fd,
5977	.off = @as(const u64, @ptrCast(&off)).,
5978	.addr = @as(const u64, @ptrCast(&addr)).,
5979	.len = 0,
5980	.rw_flags = 0,
5981	.user_data = @intFromPtr(cancel_region.fiber),
5982	.buf_index = 0,
5983	.personality = 0,
5984	.splice_fd_in = @intCast(o.len),
5985	.addr3 = @intFromPtr(o.ptr),
5986	.resv = 0,
5987	};
5988	ev.yield(null, .nothing);
5989	switch (cancel_region.errno()) {
5990	.SUCCESS => return,
5991	.INTR, .CANCELED => continue,
5992	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
5993	.NOTSOCK => \|err\| return errnoBug(err),
5994	.INVAL => \|err\| return errnoBug(err),
5995	.FAULT => \|err\| return errnoBug(err),
5996	else => \|err\| return unexpectedErrno(err),
5997	}
5998	}
5999	}
6000
6001	fn socket(
6002	ev: *Evented,
6003	cancel_region: *CancelRegion,
6004	family: linux.sa_family_t,
6005	options: net.IpAddress.BindOptions,
6006	) error{
6007	AddressFamilyUnsupported,
6008	ProtocolUnsupportedBySystem,
6009	ProcessFdQuotaExceeded,
6010	SystemFdQuotaExceeded,
6011	SystemResources,
6012	ProtocolUnsupportedByAddressFamily,
6013	SocketModeUnsupported,
6014	OptionUnsupported,
6015	Unexpected,
6016	Canceled,
6017	}!fd_t {
6018	const mode = posixSocketMode(options.mode);
6019	const protocol = posixProtocol(options.protocol);
6020	const socket_fd = while (true) {
6021	const thread = try cancel_region.awaitIoUring();
6022	thread.enqueue().* = .{
6023	.opcode = .SOCKET,
6024	.flags = 0,
6025	.ioprio = 0,
6026	.fd = family,
6027	.off = mode \| linux.SOCK.CLOEXEC,
6028	.addr = 0,
6029	.len = protocol,
6030	.rw_flags = 0,
6031	.user_data = @intFromPtr(cancel_region.fiber),
6032	.buf_index = 0,
6033	.personality = 0,
6034	.splice_fd_in = 0,
6035	.addr3 = 0,
6036	.resv = 0,
6037	};
6038	ev.yield(null, .nothing);
6039	const completion = cancel_region.completion();
6040	switch (completion.errno()) {
6041	.SUCCESS => break completion.result,
6042	.INTR, .CANCELED => continue,
6043	.AFNOSUPPORT => return error.AddressFamilyUnsupported,
6044	.INVAL => return error.ProtocolUnsupportedBySystem,
6045	.MFILE => return error.ProcessFdQuotaExceeded,
6046	.NFILE => return error.SystemFdQuotaExceeded,
6047	.NOBUFS => return error.SystemResources,
6048	.NOMEM => return error.SystemResources,
6049	.PROTONOSUPPORT => return error.ProtocolUnsupportedByAddressFamily,
6050	.PROTOTYPE => return error.SocketModeUnsupported,
6051	else => \|err\| return unexpectedErrno(err),
6052	}
6053	};
6054	errdefer ev.close(socket_fd);
6055
6056	if (options.ip6_only) {
6057	if (linux.IPV6 == void) return error.OptionUnsupported;
6058	try ev.setsockopt(cancel_region, socket_fd, linux.IPPROTO.IPV6, linux.IPV6.V6ONLY, 0);
6059	}
6060
6061	return socket_fd;
6062	}
6063
6064	fn stat(ev: Evented, cancel_region: CancelRegion, fd: fd_t) Dir.StatError!Dir.Stat {
6065	return ev.statx(cancel_region, fd, "", linux.AT.EMPTY_PATH) catch \|err\| switch (err) {
6066	error.BadPathName, error.NameTooLong => unreachable, // path is empty
6067	error.AccessDenied => return errnoBug(.ACCES),
6068	error.SymLinkLoop => return errnoBug(.LOOP),
6069	error.FileNotFound => return errnoBug(.NOENT),
6070	error.NotDir => return errnoBug(.NOTDIR),
6071	else => \|e\| return e,
6072	};
6073	}
6074
6075	fn statx(
6076	ev: *Evented,
6077	cancel_region: *CancelRegion,
6078	dir: fd_t,
6079	path: [*:0]const u8,
6080	flags: u32,
6081	) (Dir.StatError \|\| Dir.PathNameError \|\| error{ FileNotFound, NotDir, SymLinkLoop })!Dir.Stat {
6082	while (true) {
6083	var statx_buf = std.mem.zeroes(linux.Statx);
6084	const thread = try cancel_region.awaitIoUring();
6085	thread.enqueue().* = .{
6086	.opcode = .STATX,
6087	.flags = 0,
6088	.ioprio = 0,
6089	.fd = dir,
6090	.off = @intFromPtr(&statx_buf),
6091	.addr = @intFromPtr(path),
6092	.len = @bitCast(linux_statx_request),
6093	.rw_flags = flags,
6094	.user_data = @intFromPtr(cancel_region.fiber),
6095	.buf_index = 0,
6096	.personality = 0,
6097	.splice_fd_in = 0,
6098	.addr3 = 0,
6099	.resv = 0,
6100	};
6101	ev.yield(null, .nothing);
6102	switch (cancel_region.errno()) {
6103	.SUCCESS => return statFromLinux(&statx_buf),
6104	.INTR, .CANCELED => continue,
6105	.ACCES => return error.AccessDenied,
6106	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
6107	.FAULT => \|err\| return errnoBug(err),
6108	.INVAL => \|err\| return errnoBug(err),
6109	.LOOP => return error.SymLinkLoop,
6110	.NAMETOOLONG => \|err\| return errnoBug(err),
6111	.NOENT => return error.FileNotFound,
6112	.NOTDIR => return error.NotDir,
6113	.NOMEM => return error.SystemResources,
6114	else => \|err\| return unexpectedErrno(err),
6115	}
6116	}
6117	}
6118
6119	fn urandomReadAll(
6120	ev: *Evented,
6121	cancel_region: *CancelRegion,
6122	buffer: []u8,
6123	) (File.OpenError \|\| File.Reader.Error \|\| error{EndOfStream})!void {
6124	return ev.readAll(cancel_region, try ev.random_fd.open(ev, cancel_region, "/dev/urandom", .{
6125	.ACCMODE = .RDONLY,
6126	.CLOEXEC = true,
6127	}), buffer);
6128	}
6129
6130	fn utimensat(
6131	ev: *Evented,
6132	sync: *CancelRegion.Sync,
6133	dir: fd_t,
6134	path: [*:0]const u8,
6135	times: ?*const [2]linux.timespec,
6136	flags: u32,
6137	) File.SetTimestampsError!void {
6138	_ = ev;
6139	while (true) {
6140	try sync.cancel_region.await(.nothing);
6141	switch (linux.errno(linux.utimensat(dir, path, times, flags))) {
6142	.SUCCESS => return,
6143	.INTR => continue,
6144	.BADF => \|err\| return errnoBug(err), // always a race condition
6145	.FAULT => \|err\| return errnoBug(err),
6146	.INVAL => \|err\| return errnoBug(err),
6147	.ACCES => return error.AccessDenied,
6148	.PERM => return error.PermissionDenied,
6149	.ROFS => return error.ReadOnlyFileSystem,
6150	else => \|err\| return unexpectedErrno(err),
6151	}
6152	}
6153	}
6154
6155	fn writeAll(
6156	ev: *Evented,
6157	cancel_region: *CancelRegion,
6158	fd: fd_t,
6159	buffer: []const u8,
6160	) (File.Writer.Error \|\| error{EndOfStream})!void {
6161	var index: usize = 0;
6162	while (buffer.len - index != 0) {
6163	const len = try ev.pwritev(cancel_region, fd, &.{
6164	.{ .base = buffer[index..].ptr, .len = buffer.len - index },
6165	}, null);
6166	if (len == 0) return error.EndOfStream;
6167	index += len;
6168	}
6169	}
6170
6171	test {
6172	_ = Fiber.CancelProtection;
6173	}

lib/std/Io/fiber.zig created+201

...	@@ -0,0 +1,201 @@
1	pub const supported = switch (builtin.cpu.arch) {
2	.aarch64, .x86_64 => true,
3	else => false,
4	};
5
6	/// Stores the cpu state of an inactive fiber.
7	pub const Context = switch (builtin.cpu.arch) {
8	.aarch64 => extern struct {
9	sp: u64,
10	fp: u64,
11	pc: u64,
12	},
13	.x86_64 => extern struct {
14	rsp: u64,
15	rbp: u64,
16	rip: u64,
17	},
18	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
19	};
20
21	pub const Switch = extern struct { old: Context, new: Context };
22
23	/// Fills `s.old` with the current cpu state, and restores the cpu state stored in `s.new`.
24	pub inline fn contextSwitch(s: const Switch) const Switch {
25	return switch (builtin.cpu.arch) {
26	.aarch64 => asm volatile (
27	\\ ldp x0, x2, [x1]
28	\\ ldr x3, [x2, #16]
29	\\ mov x4, sp
30	\\ stp x4, fp, [x0]
31	\\ adr x5, 0f
32	\\ ldp x4, fp, [x2]
33	\\ str x5, [x0, #16]
34	\\ mov sp, x4
35	\\ br x3
36	\\0:
37	: [received_message] "={x1}" (-> *const Switch),
38	: [message_to_send] "{x1}" (s),
39	: .{
40	.x0 = true,
41	.x1 = true,
42	.x2 = true,
43	.x3 = true,
44	.x4 = true,
45	.x5 = true,
46	.x6 = true,
47	.x7 = true,
48	.x8 = true,
49	.x9 = true,
50	.x10 = true,
51	.x11 = true,
52	.x12 = true,
53	.x13 = true,
54	.x14 = true,
55	.x15 = true,
56	.x16 = true,
57	.x17 = true,
58	.x19 = true,
59	.x20 = true,
60	.x21 = true,
61	.x22 = true,
62	.x23 = true,
63	.x24 = true,
64	.x25 = true,
65	.x26 = true,
66	.x27 = true,
67	.x28 = true,
68	.x30 = true,
69	.z0 = true,
70	.z1 = true,
71	.z2 = true,
72	.z3 = true,
73	.z4 = true,
74	.z5 = true,
75	.z6 = true,
76	.z7 = true,
77	.z8 = true,
78	.z9 = true,
79	.z10 = true,
80	.z11 = true,
81	.z12 = true,
82	.z13 = true,
83	.z14 = true,
84	.z15 = true,
85	.z16 = true,
86	.z17 = true,
87	.z18 = true,
88	.z19 = true,
89	.z20 = true,
90	.z21 = true,
91	.z22 = true,
92	.z23 = true,
93	.z24 = true,
94	.z25 = true,
95	.z26 = true,
96	.z27 = true,
97	.z28 = true,
98	.z29 = true,
99	.z30 = true,
100	.z31 = true,
101	.p0 = true,
102	.p1 = true,
103	.p2 = true,
104	.p3 = true,
105	.p4 = true,
106	.p5 = true,
107	.p6 = true,
108	.p7 = true,
109	.p8 = true,
110	.p9 = true,
111	.p10 = true,
112	.p11 = true,
113	.p12 = true,
114	.p13 = true,
115	.p14 = true,
116	.p15 = true,
117	.fpcr = true,
118	.fpsr = true,
119	.ffr = true,
120	.memory = true,
121	}),
122	.x86_64 => asm volatile (
123	\\ movq 0(%%rsi), %%rax
124	\\ movq 8(%%rsi), %%rcx
125	\\ leaq 0f(%%rip), %%rdx
126	\\ movq %%rsp, 0(%%rax)
127	\\ movq %%rbp, 8(%%rax)
128	\\ movq %%rdx, 16(%%rax)
129	\\ movq 0(%%rcx), %%rsp
130	\\ movq 8(%%rcx), %%rbp
131	\\ jmpq *16(%%rcx)
132	\\0:
133	: [received_message] "={rsi}" (-> *const Switch),
134	: [message_to_send] "{rsi}" (s),
135	: .{
136	.rax = true,
137	.rcx = true,
138	.rdx = true,
139	.rbx = true,
140	.rsi = true,
141	.rdi = true,
142	.r8 = true,
143	.r9 = true,
144	.r10 = true,
145	.r11 = true,
146	.r12 = true,
147	.r13 = true,
148	.r14 = true,
149	.r15 = true,
150	.mm0 = true,
151	.mm1 = true,
152	.mm2 = true,
153	.mm3 = true,
154	.mm4 = true,
155	.mm5 = true,
156	.mm6 = true,
157	.mm7 = true,
158	.zmm0 = true,
159	.zmm1 = true,
160	.zmm2 = true,
161	.zmm3 = true,
162	.zmm4 = true,
163	.zmm5 = true,
164	.zmm6 = true,
165	.zmm7 = true,
166	.zmm8 = true,
167	.zmm9 = true,
168	.zmm10 = true,
169	.zmm11 = true,
170	.zmm12 = true,
171	.zmm13 = true,
172	.zmm14 = true,
173	.zmm15 = true,
174	.zmm16 = true,
175	.zmm17 = true,
176	.zmm18 = true,
177	.zmm19 = true,
178	.zmm20 = true,
179	.zmm21 = true,
180	.zmm22 = true,
181	.zmm23 = true,
182	.zmm24 = true,
183	.zmm25 = true,
184	.zmm26 = true,
185	.zmm27 = true,
186	.zmm28 = true,
187	.zmm29 = true,
188	.zmm30 = true,
189	.zmm31 = true,
190	.fpsr = true,
191	.fpcr = true,
192	.mxcsr = true,
193	.rflags = true,
194	.dirflag = true,
195	.memory = true,
196	}),
197	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
198	};
199	}
200
201	const builtin = @import("builtin");

lib/std/c.zig+3-10

 @@ -10724,6 +10724,7 @@ pub extern "c" fn chmod(path: [*:0]const u8, mode: mode_t) c_int;
 pub extern "c" fn fchmod(fd: fd_t, mode: mode_t) c_int;
 pub extern "c" fn fchmodat(fd: fd_t, path: [*:0]const u8, mode: mode_t, flags: c_uint) c_int;
 pub extern "c" fn fchown(fd: fd_t, owner: uid_t, group: gid_t) c_int;
+pub extern "c" fn fchownat(fd: fd_t, path: [*:0]const u8, owner: uid_t, group: gid_t, flags: c_uint) c_int;
 pub extern "c" fn umask(mode: mode_t) mode_t;
 pub extern "c" fn rmdir(path: [*:0]const u8) c_int;
 @@ -10864,6 +10865,7 @@ pub const pthread_setname_np = switch (native_os) {
 pub extern "c" fn pthread_getname_np(thread: pthread_t, name: [*:0]u8, len: usize) c_int;
 pub extern "c" fn pthread_kill(pthread_t, signal: SIG) c_int;
+pub extern "c" fn pthread_exit(ptr: ?*anyopaque) noreturn;
 pub const pthread_threadid_np = switch (native_os) {
     .driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => private.pthread_threadid_np,
 @@ -11296,16 +11298,7 @@ pub const clock_get_time = darwin.clock_get_time;
 pub const clock_serv_t = darwin.clock_serv_t;
 pub const clock_res_t = darwin.clock_res_t;
 pub const @"close$NOCANCEL" = darwin.@"close$NOCANCEL";
-pub const dispatch_function_t = darwin.dispatch_function_t;
+pub const dispatch = darwin.dispatch;
-pub const dispatch_once_f = darwin.dispatch_once_f;
-pub const dispatch_once_t = darwin.dispatch_once_t;
-pub const dispatch_release = darwin.dispatch_release;
-pub const dispatch_semaphore_create = darwin.dispatch_semaphore_create;
-pub const dispatch_semaphore_signal = darwin.dispatch_semaphore_signal;
-pub const dispatch_semaphore_t = darwin.dispatch_semaphore_t;
-pub const dispatch_semaphore_wait = darwin.dispatch_semaphore_wait;
-pub const dispatch_time = darwin.dispatch_time;
-pub const dispatch_time_t = darwin.dispatch_time_t;
 pub const fcopyfile = darwin.fcopyfile;
 pub const host_t = darwin.host_t;
 pub const integer_t = darwin.integer_t;

lib/std/c/darwin.zig+3-25

...	@@ -18,6 +18,9 @@ comptime {	...	@@ -18,6 +18,9 @@ comptime {
18	assert(builtin.os.tag.isDarwin()); // Prevent access of std.c symbols on wrong OS.	18	assert(builtin.os.tag.isDarwin()); // Prevent access of std.c symbols on wrong OS.
19	}	19	}
20		20
		21	// Grand Central Dispatch is exposed by libSystem.
		22	pub const dispatch = @import("darwin/dispatch.zig");
		23
21	pub const mach_port_t = c_uint;	24	pub const mach_port_t = c_uint;
22		25
23	pub const EXC = enum(exception_type_t) {	26	pub const EXC = enum(exception_type_t) {
...	@@ -896,31 +899,6 @@ pub const qos_class_t = enum(c_uint) {	...	@@ -896,31 +899,6 @@ pub const qos_class_t = enum(c_uint) {
896	_,	899	_,
897	};	900	};
898		901
899	// Grand Central Dispatch is exposed by libSystem.
900	pub extern "c" fn dispatch_release(object: *anyopaque) void;
901
902	pub const dispatch_semaphore_t = *opaque {};
903	pub extern "c" fn dispatch_semaphore_create(value: isize) ?dispatch_semaphore_t;
904	pub extern "c" fn dispatch_semaphore_wait(dsema: dispatch_semaphore_t, timeout: dispatch_time_t) isize;
905	pub extern "c" fn dispatch_semaphore_signal(dsema: dispatch_semaphore_t) isize;
906
907	pub const DISPATCH_TIME_NOW = @compileError("use dispatch_time_t.NOW");
908	pub const DISPATCH_TIME_FOREVER = @compileError("use dispatch_time_t.FOREVER");
909	pub const dispatch_time_t = enum(u64) {
910	NOW = 0,
911	FOREVER = ~0,
912	_,
913	};
914	pub extern "c" fn dispatch_time(when: dispatch_time_t, delta: i64) dispatch_time_t;
915
916	pub const dispatch_once_t = usize;
917	pub const dispatch_function_t = fn (?*anyopaque) callconv(.c) void;
918	pub extern fn dispatch_once_f(
919	predicate: *dispatch_once_t,
920	context: ?*anyopaque,
921	function: dispatch_function_t,
922	) void;
923
924	/// Undocumented futex-like API available on darwin 16+	902	/// Undocumented futex-like API available on darwin 16+
925	/// (macOS 10.12+, iOS 10.0+, tvOS 10.0+, watchOS 3.0+, catalyst 13.0+).	903	/// (macOS 10.12+, iOS 10.0+, tvOS 10.0+, watchOS 3.0+, catalyst 13.0+).
926	///	904	///

lib/std/c/darwin/dispatch.zig created+308

...	@@ -0,0 +1,308 @@
1	// dispatch/base.h
2	pub const function_t = const fn (?anyopaque) callconv(.c) void;
3
4	// dispatch/object.h
5	pub const object_t = *_os_object_s;
6	pub const retain = dispatch_retain;
7	pub const release = dispatch_release;
8	pub const get_context = dispatch_get_context;
9	pub const set_context = dispatch_set_context;
10	pub const set_finalizer_f = dispatch_set_finalizer_f;
11	pub const activate = dispatch_activate;
12	pub const @"suspend" = dispatch_suspend;
13	pub const @"resume" = dispatch_resume;
14
15	const _os_object_s = opaque {
16	pub const retain = dispatch_retain;
17	pub const release = dispatch_release;
18	pub const get_context = dispatch_get_context;
19	pub const set_context = dispatch_set_context;
20	pub const set_finalizer = dispatch_set_finalizer_f;
21	pub const activate = dispatch_activate;
22	pub const @"suspend" = dispatch_suspend;
23	pub const @"resume" = dispatch_resume;
24	pub const set_target_queue = dispatch_set_target_queue;
25	};
26	extern "c" fn dispatch_retain(object: object_t) void;
27	extern "c" fn dispatch_release(object: object_t) void;
28	extern "c" fn dispatch_get_context(object: object_t) ?*anyopaque;
29	extern "c" fn dispatch_set_context(object: object_t, context: ?*anyopaque) void;
30	extern "c" fn dispatch_set_finalizer_f(object: object_t, finalizer: ?function_t) void;
31	extern "c" fn dispatch_activate(object: object_t) void;
32	extern "c" fn dispatch_suspend(object: object_t) void;
33	extern "c" fn dispatch_resume(object: object_t) void;
34
35	// dispatch/once.h
36	pub const once_t = enum(isize) {
37	init = 0,
38	done = -1,
39	_,
40
41	pub inline fn once(predicate: once_t, context: ?anyopaque, function: function_t) void {
42	if (@atomicLoad(once_t, predicate, .unordered) != .done) {
43	@branchHint(.unlikely);
44	once_f(predicate, context, function);
45	} else asm volatile ("" ::: .{ .memory = true });
46	switch (builtin.mode) {
47	.Debug, .ReleaseSafe => {},
48	.ReleaseFast, .ReleaseSmall => if (@atomicLoad(once_t, predicate, .unordered) != .done)
49	unreachable,
50	}
51	}
52	};
53	pub const once_f = dispatch_once_f;
54
55	extern "c" fn dispatch_once_f(predicate: once_t, context: ?anyopaque, function: function_t) void;
56
57	// dispatch/queue.h
58	pub const queue_t = *queue_s;
59	pub const queue_global_t = queue_t;
60	pub const queue_serial_executor_t = queue_t;
61	pub const queue_serial_t = queue_t;
62	pub const queue_main_t = queue_serial_t;
63	pub const queue_concurrent_t = queue_t;
64	pub const async_f = dispatch_async_f;
65	pub const sync_f = dispatch_sync_f;
66	pub const async_and_wait_f = dispatch_async_and_wait_f;
67	pub const apply_f = dispatch_apply_f;
68	pub const get_current_queue = dispatch_get_current_queue;
69	pub inline fn get_main_queue() queue_main_t {
70	return &_dispatch_main_q;
71	}
72	pub const queue_priority_t = enum(c_long) {
73	HIGH = 2,
74	DEFAULT = 0,
75	LOW = -1,
76	BACKGROUND = std.math.minInt(i16),
77	_,
78	};
79	pub const get_global_queue = dispatch_get_global_queue;
80	pub const queue_attr_t = ?*queue_attr_s;
81	pub inline fn QUEUE_SERIAL() queue_attr_t {
82	return null;
83	}
84	pub inline fn QUEUE_INACTIVE() queue_attr_t {
85	return queue_attr_make_initially_inactive(QUEUE_SERIAL());
86	}
87	pub inline fn QUEUE_CONCURRENT() queue_attr_t {
88	return &_dispatch_queue_attr_concurrent;
89	}
90	pub inline fn QUEUE_CONCURRENT_INACTIVE() queue_attr_t {
91	return queue_attr_make_initially_inactive(QUEUE_CONCURRENT());
92	}
93	pub const queue_attr_make_initially_inactive = dispatch_queue_attr_make_initially_inactive;
94	pub const TARGET_QUEUE_DEFAULT: ?queue_t = null;
95	pub const queue_create_with_target = dispatch_queue_create_with_target;
96	pub const queue_create = dispatch_queue_create;
97	pub const CURRENT_QUEUE_LABEL: ?[*:0]const u8 = null;
98	pub const queue_get_label = dispatch_queue_get_label;
99	pub const main = dispatch_main;
100	pub const after_f = dispatch_after_f;
101
102	const queue_s = opaque {
103	pub inline fn as_object(queue: queue_t) object_t {
104	return @ptrCast(queue);
105	}
106	pub const async = async_f;
107	pub const sync = sync_f;
108	pub const async_and_wait = async_and_wait_f;
109	pub const apply = apply_f;
110	pub const get_current = get_current_queue;
111	pub const get_main = get_main_queue;
112	pub const get_global = get_global_queue;
113	pub const create_with_target = queue_create_with_target;
114	pub const create = queue_create;
115	pub const get_label = queue_get_label;
116	};
117	extern "c" fn dispatch_async_f(queue: queue_t, context: ?*anyopaque, work: function_t) void;
118	extern "c" fn dispatch_sync_f(queue: queue_t, context: ?*anyopaque, work: function_t) void;
119	extern "c" fn dispatch_async_and_wait_f(queue: queue_t, context: ?*anyopaque, work: function_t) void;
120	extern "c" fn dispatch_apply_f(iterations: usize, queue: ?queue_t, context: ?anyopaque, work: const fn (context: ?*anyopaque, iteration: usize) callconv(.c) void) void;
121	extern "c" fn dispatch_get_current_queue() queue_t;
122	extern "c" var _dispatch_main_q: queue_s;
123	extern "c" fn dispatch_get_global_queue(identifier: isize, flags: usize) queue_global_t;
124	const queue_attr_s = opaque {
125	pub inline fn as_object(queue_attr: queue_attr_t) object_t {
126	return @ptrCast(queue_attr);
127	}
128	pub const SERIAL = QUEUE_SERIAL;
129	pub const INACTIVE = QUEUE_INACTIVE;
130	pub const CONCURRENT = QUEUE_CONCURRENT;
131	pub const CONCURRENT_INACTIVE = QUEUE_CONCURRENT_INACTIVE;
132	};
133	extern "c" var _dispatch_queue_attr_concurrent: queue_attr_s;
134	extern "c" fn dispatch_queue_attr_make_initially_inactive(attr: queue_attr_t) queue_attr_t;
135	extern "c" fn dispatch_queue_create_with_target(label: ?[*:0]const u8, attr: queue_attr_t, target: ?queue_t) ?queue_t;
136	extern "c" fn dispatch_queue_create(label: ?[*:0]const u8, attr: queue_attr_t) ?queue_t;
137	extern "c" fn dispatch_queue_get_label(queue: ?queue_t) [*:0]const u8;
138	extern "c" fn dispatch_set_target_queue(object: object_t, queue: ?queue_t) void;
139	extern "c" fn dispatch_main() noreturn;
140	extern "c" fn dispatch_after_f(when: time_t, queue: queue_t, context: ?*anyopaque, work: function_t) void;
141
142	// dispatch/semaphore.h
143	pub const semaphore_t = *semaphore_s;
144	pub const semaphore_create = dispatch_semaphore_create;
145	pub const semaphore_wait = dispatch_semaphore_wait;
146	pub const semaphore_signal = dispatch_semaphore_signal;
147
148	const semaphore_s = opaque {
149	pub inline fn as_object(semaphore: semaphore_t) object_t {
150	return @ptrCast(semaphore);
151	}
152	pub const create = semaphore_create;
153	pub const wait = semaphore_wait;
154	pub const signal = semaphore_signal;
155	};
156	extern "c" fn dispatch_semaphore_create(value: isize) ?semaphore_t;
157	extern "c" fn dispatch_semaphore_wait(dsema: semaphore_t, timeout: time_t) isize;
158	extern "c" fn dispatch_semaphore_signal(dsema: semaphore_t) isize;
159
160	// dispatch/source.h
161	pub const source_t = *source_s;
162	pub const source_type_t = *const source_type_s;
163	pub const SOURCE_TYPE_DATA_ADD = &_dispatch_source_type_data_add;
164	pub const SOURCE_TYPE_DATA_OR = &_dispatch_source_type_data_or;
165	pub const SOURCE_TYPE_DATA_REPLACE = &_dispatch_source_type_data_replace;
166	pub const SOURCE_TYPE_MACH_SEND = &_dispatch_source_type_mach_send;
167	pub const SOURCE_TYPE_MACH_RECV = &_dispatch_source_type_mach_recv;
168	pub const SOURCE_TYPE_MEMORYPRESSURE = &_dispatch_source_type_memorypressure;
169	pub const SOURCE_TYPE_PROC = &_dispatch_source_type_proc;
170	pub const SOURCE_TYPE_READ = &_dispatch_source_type_read;
171	pub const SOURCE_TYPE_SIGNAL = &_dispatch_source_type_signal;
172	pub const SOURCE_TYPE_TIMER = &_dispatch_source_type_timer;
173	pub const SOURCE_TYPE_VNODE = &_dispatch_source_type_vnode;
174	pub const SOURCE_TYPE_WRITE = &_dispatch_source_type_write;
175	pub const source_mach_send_flags_t = packed struct(usize) {
176	DEAD: bool = false,
177	unused1: @Int(.unsigned, @bitSizeOf(usize) - 1) = 0,
178	};
179	pub const source_mach_recv_flags_t = packed struct(usize) {
180	unused0: @Int(.unsigned, @bitSizeOf(usize) - 0) = 0,
181	};
182	pub const source_memorypressure_flags_t = packed struct(usize) {
183	NORMAL: bool = false,
184	WARN: bool = false,
185	CRITICAL: bool = false,
186	unused3: @Int(.unsigned, @bitSizeOf(usize) - 3) = 0,
187	};
188	pub const source_proc_flags_t = packed struct(usize) {
189	unused0: u27 = 0,
190	SIGNAL: bool = false,
191	unused28: u1 = 0,
192	EXEC: bool = false,
193	FORK: bool = false,
194	EXIT: bool = false,
195	unused32: @Int(.unsigned, @bitSizeOf(usize) - 32) = 0,
196	};
197	pub const source_vnode_flags_t = packed struct(usize) {
198	DELETE: bool = false,
199	WRITE: bool = false,
200	EXTEND: bool = false,
201	ATTRIB: bool = false,
202	LINK: bool = false,
203	RENAME: bool = false,
204	REVOKE: bool = false,
205	unused7: u1 = 0,
206	FUNLOCK: bool = false,
207	unused9: @Int(.unsigned, @bitSizeOf(usize) - 9) = 0,
208	};
209	pub const source_timer_flags_t = packed struct(usize) {
210	STRICT: bool = false,
211	unused1: @Int(.unsigned, @bitSizeOf(usize) - 1) = 0,
212	};
213	pub const source_flags_t = packed union {
214	raw: usize,
215	MACH_SEND: source_mach_send_flags_t,
216	MACH_RECV: source_mach_recv_flags_t,
217	MEMORYPRESSURE: source_memorypressure_flags_t,
218	PROC: source_proc_flags_t,
219	VNODE: source_vnode_flags_t,
220	pub const none: source_flags_t = .{ .raw = 0 };
221	};
222	pub const source_create = dispatch_source_create;
223	pub const source_set_event_handler_f = dispatch_source_set_event_handler_f;
224	pub const source_set_cancel_handler_f = dispatch_source_set_cancel_handler_f;
225	pub const source_cancel = dispatch_source_cancel;
226	pub const source_testcancel = dispatch_source_testcancel;
227	pub const source_get_handle = dispatch_source_get_handle;
228	pub const source_get_mask = dispatch_source_get_mask;
229	pub const source_get_data = dispatch_source_get_data;
230	pub const source_merge_data = dispatch_source_merge_data;
231	pub const source_set_timer = dispatch_source_set_timer;
232	pub const source_set_registration_handler_f = dispatch_source_set_registration_handler_f;
233
234	const source_s = opaque {
235	pub inline fn as_object(source: source_t) object_t {
236	return @ptrCast(source);
237	}
238	pub const set_event_handler = source_set_event_handler_f;
239	pub const set_cancel_handler = source_set_cancel_handler_f;
240	pub const cancel = source_cancel;
241	pub const testcancel = source_testcancel;
242	pub const get_handle = source_get_handle;
243	pub const get_mask = source_get_mask;
244	pub const get_data = source_get_data;
245	pub const merge_data = source_merge_data;
246	pub const set_timer = source_set_timer;
247	pub const set_registration_handler = source_set_registration_handler_f;
248	};
249	const source_type_s = opaque {
250	pub const DATA_ADD = SOURCE_TYPE_DATA_ADD;
251	pub const DATA_OR = SOURCE_TYPE_DATA_OR;
252	pub const DATA_REPLACE = SOURCE_TYPE_DATA_REPLACE;
253	pub const MACH_SEND = SOURCE_TYPE_MACH_SEND;
254	pub const MACH_RECV = SOURCE_TYPE_MACH_RECV;
255	pub const MEMORYPRESSURE = SOURCE_TYPE_MEMORYPRESSURE;
256	pub const PROC = SOURCE_TYPE_PROC;
257	pub const READ = SOURCE_TYPE_READ;
258	pub const SIGNAL = SOURCE_TYPE_SIGNAL;
259	pub const TIMER = SOURCE_TYPE_TIMER;
260	pub const VNODE = SOURCE_TYPE_VNODE;
261	pub const WRITE = SOURCE_TYPE_WRITE;
262	};
263	extern "c" const _dispatch_source_type_data_add: source_type_s;
264	extern "c" const _dispatch_source_type_data_or: source_type_s;
265	extern "c" const _dispatch_source_type_data_replace: source_type_s;
266	extern "c" const _dispatch_source_type_mach_send: source_type_s;
267	extern "c" const _dispatch_source_type_mach_recv: source_type_s;
268	extern "c" const _dispatch_source_type_memorypressure: source_type_s;
269	extern "c" const _dispatch_source_type_proc: source_type_s;
270	extern "c" const _dispatch_source_type_read: source_type_s;
271	extern "c" const _dispatch_source_type_signal: source_type_s;
272	extern "c" const _dispatch_source_type_timer: source_type_s;
273	extern "c" const _dispatch_source_type_vnode: source_type_s;
274	extern "c" const _dispatch_source_type_write: source_type_s;
275	extern "c" fn dispatch_source_create(type: source_type_t, handle: usize, mask: source_flags_t, queue: ?queue_t) ?source_t;
276	extern "c" fn dispatch_source_set_event_handler_f(source: source_t, handler: ?function_t) void;
277	extern "c" fn dispatch_source_set_cancel_handler_f(source: source_t, handler: ?function_t) void;
278	extern "c" fn dispatch_source_cancel(source: source_t) void;
279	extern "c" fn dispatch_source_testcancel(source: source_t) isize;
280	extern "c" fn dispatch_source_get_handle(source: source_t) usize;
281	extern "c" fn dispatch_source_get_mask(source: source_t) source_flags_t;
282	extern "c" fn dispatch_source_get_data(source: source_t) usize;
283	extern "c" fn dispatch_source_merge_data(source: source_t, value: usize) void;
284	extern "c" fn dispatch_source_set_timer(source: source_t, start: time_t, interval: u64, leeway: u64) void;
285	extern "c" fn dispatch_source_set_registration_handler_f(source: source_t, handler: ?function_t) void;
286
287	// dispatch/time.h
288	pub const time_t = enum(u64) {
289	WALL_NOW = WALLTIME_NOW,
290	NOW = TIME_NOW,
291	FOREVER = TIME_FOREVER,
292	_,
293
294	pub const time = dispatch_time;
295	pub const walltime = dispatch_walltime;
296	pub const after = dispatch_after_f;
297	};
298	pub const WALLTIME_NOW = ~@as(u64, 1);
299	pub const TIME_NOW: u64 = 0;
300	pub const TIME_FOREVER = ~@as(u64, 0);
301	pub const time = dispatch_time;
302	pub const walltime = dispatch_walltime;
303
304	extern "c" fn dispatch_time(when: time_t, delta: i64) time_t;
305	extern "c" fn dispatch_walltime(when: ?*const std.c.timespec, delta: i64) time_t;
306
307	const builtin = @import("builtin");
308	const std = @import("std");

lib/std/debug.zig+2-6

 @@ -533,9 +533,7 @@ pub fn defaultPanic(msg: []const u8, first_trace_addr: ?usize) noreturn {
         else => {},
     }
-    // Don't try to cancel during a panic. No need to re-enable cancelation,
+    std.Options.debug_io.vtable.crashHandler(std.Options.debug_io.userdata);
-    // because the panic handler doesn't return.
-    _ = std.Options.debug_io.swapCancelProtection(.blocked);
     if (enable_segfault_handler) {
         // If a segfault happens while panicking, we want it to actually segfault, not trigger
 @@ -1535,9 +1533,7 @@ fn handleSegfault(addr: ?usize, name: []const u8, opt_ctx: ?CpuContextPtr) noret
 }
 pub fn defaultHandleSegfault(addr: ?usize, name: []const u8, opt_ctx: ?CpuContextPtr) noreturn {
-    // Don't try to cancel during a segfault. No need to re-enable cancelation,
+    std.Options.debug_io.vtable.crashHandler(std.Options.debug_io.userdata);
-    // because the segfault handler doesn't return.
-    _ = std.Options.debug_io.swapCancelProtection(.blocked);
     // There is very similar logic to the following in `defaultPanic`.
     switch (panic_stage) {

lib/std/process/Environ.zig+12

 @@ -39,6 +39,10 @@ pub const GlobalBlock = struct {
     pub const global: GlobalBlock = .{ .use_global = true };
     pub fn deinit(_: GlobalBlock, _: Allocator) void {}
+    pub fn isEmpty(block: GlobalBlock) bool {
+        return !block.use_global;
+    }
 };
 pub const PosixBlock = struct {
 @@ -51,6 +55,10 @@ pub const PosixBlock = struct {
         gpa.free(block.slice);
     }
+    pub fn isEmpty(block: PosixBlock) bool {
+        return block.slice.len == 0;
+    }
     pub const View = struct {
         slice: []const [*:0]const u8,
 @@ -72,6 +80,10 @@ pub const WindowsBlock = struct {
         gpa.free(block.slice);
     }
+    pub fn isEmpty(block: WindowsBlock) bool {
+        return block.slice[0] == 0;
+    }
     pub const View = struct {
         ptr: [*:0]const u16,

src/crash_report.zig+2

...	@@ -79,6 +79,8 @@ fn dumpCrashContext() Io.Writer.Error!void {	...	@@ -79,6 +79,8 @@ fn dumpCrashContext() Io.Writer.Error!void {
79	if (S.already_dumped) return;	79	if (S.already_dumped) return;
80	S.already_dumped = true;	80	S.already_dumped = true;
81		81
		82	std.Options.debug_io.vtable.crashHandler(std.Options.debug_io.userdata);
		83
82	// TODO: this does mean that a different thread could grab the stderr mutex between the context	84	// TODO: this does mean that a different thread could grab the stderr mutex between the context
83	// and the actual panic printing, which would be quite confusing.	85	// and the actual panic printing, which would be quite confusing.
84	const stderr = std.debug.lockStderr(&.{});	86	const stderr = std.debug.lockStderr(&.{});