Commit a28d57292f86dfdacf88453509d0b7a1a49443eb | zig

author	Jacob Young <jacobly@ziglang.org> 2026-01-06 05:10:33-05:00
committer	Jacob Young <jacobly@ziglang.org> 2026-02-09 10:47:21-05:00
log	a28d57292f86dfdacf88453509d0b7a1a49443eb
tree	f67cce22318a0eeafa7e17b15911e3a2f05ed09d
parent	b48599c5492448241a5c37cf17ff6caffa3dd302

IoUring: update to new Io APIs

19 files changed, 5884 insertions(+), 1182 deletions(-)

bootstrap.c
build.zig
lib
- std
  - Io.zig
  - Io
    File.zig
    IoUring.zig
    Threaded.zig
  - os
    linux.zig
  - process.zig
  - tar.zig
src
stage1
- config.zig.in

bootstrap.c+1

...	@@ -143,6 +143,7 @@ int main(int argc, char **argv) {	...	@@ -143,6 +143,7 @@ int main(int argc, char **argv) {
143	"pub const skip_non_native = false;\n"	143	"pub const skip_non_native = false;\n"
144	"pub const debug_gpa = false;\n"	144	"pub const debug_gpa = false;\n"
145	"pub const dev = .core;\n"	145	"pub const dev = .core;\n"
		146	"pub const io_mode: enum { threaded, evented } = .threaded;\n"
146	"pub const value_interpret_mode = .direct;\n"	147	"pub const value_interpret_mode = .direct;\n"
147	, zig_version);	148	, zig_version);
148	if (written < 100)	149	if (written < 100)

build.zig+4

lib/std/Io.zig+41-20

 @@ -378,7 +378,9 @@ pub const Operation = union(enum) {
         pub const Pending = struct {
             node: List.DoubleNode,
             tag: Tag,
-            context: [3]usize,
+            context: Context align(@max(@alignOf(usize), 4)),
+            pub const Context = [3]usize;
         };
         pub const Completion = struct {
 @@ -426,10 +428,10 @@ pub fn operate(io: Io, operation: Operation) Cancelable!Operation.Result {
 pub const Batch = struct {
     storage: []Operation.Storage,
     unused: Operation.List,
-    submissions: Operation.List,
+    submitted: Operation.List,
     pending: Operation.List,
-    completions: Operation.List,
+    completed: Operation.List,
-    context: ?*anyopaque,
+    context: ?*anyopaque align(@max(@alignOf(?*anyopaque), 4)),
     /// After calling this, it is safe to unconditionally defer a call to
     /// `cancel`. `storage` is a pre-allocated buffer of undefined memory that
 @@ -448,9 +450,9 @@ pub const Batch = struct {
                 .head = .fromIndex(0),
                 .tail = .fromIndex(storage.len - 1),
             },
-            .submissions = .empty,
+            .submitted = .empty,
             .pending = .empty,
-            .completions = .empty,
+            .completed = .empty,
             .context = null,
         };
     }
 @@ -471,20 +473,20 @@ pub const Batch = struct {
         const storage = &b.storage[index];
         const unused = storage.unused;
         switch (unused.prev) {
-            .none => b.unused.head = .none,
+            .none => b.unused.head = unused.next,
             else => |prev_index| b.storage[prev_index.toIndex()].unused.next = unused.next,
         }
         switch (unused.next) {
-            .none => b.unused.tail = .none,
+            .none => b.unused.tail = unused.prev,
             else => |next_index| b.storage[next_index.toIndex()].unused.prev = unused.prev,
         }
-        switch (b.submissions.tail) {
+        switch (b.submitted.tail) {
-            .none => b.submissions.head = .fromIndex(index),
+            .none => b.submitted.head = .fromIndex(index),
             else => |tail_index| b.storage[tail_index.toIndex()].submission.node.next = .fromIndex(index),
         }
         storage.* = .{ .submission = .{ .node = .{ .next = .none }, .operation = operation } };
-        b.submissions.tail = .fromIndex(index);
+        b.submitted.tail = .fromIndex(index);
     }
     pub const Completion = struct {
 @@ -501,13 +503,13 @@ pub const Batch = struct {
     /// Each completion returned from this function dequeues from the `Batch`.
     /// It is not required to dequeue all completions before awaiting again.
     pub fn next(b: *Batch) ?Completion {
-        const index = b.completions.head;
+        const index = b.completed.head;
         if (index == .none) return null;
         const storage = &b.storage[index.toIndex()];
         const completion = storage.completion;
         const next_index = completion.node.next;
-        b.completions.head = next_index;
+        b.completed.head = next_index;
-        if (next_index == .none) b.completions.tail = .none;
+        if (next_index == .none) b.completed.tail = .none;
         const tail_index = b.unused.tail;
         switch (tail_index) {
 @@ -551,7 +553,27 @@ pub const Batch = struct {
     /// may have successfully completed regardless of the cancel request and
     /// will appear in the iteration.
     pub fn cancel(b: *Batch, io: Io) void {
-        return io.vtable.batchCancel(io.userdata, b);
+        { // abort pending submissions
+            var tail_index = b.unused.tail;
+            defer b.unused.tail = tail_index;
+            var index = b.submitted.head;
+            errdefer b.submissions.head = index;
+            while (index != .none) {
+                const next_index = b.storage[index.toIndex()].submission.node.next;
+                switch (tail_index) {
+                    .none => b.unused.head = index,
+                    else => b.storage[tail_index.toIndex()].unused.next = index,
+                }
+                b.storage[index.toIndex()] = .{ .unused = .{ .prev = tail_index, .next = .none } };
+                tail_index = index;
+                index = next_index;
+            }
+            b.submitted = .{ .head = .none, .tail = .none };
+        }
+        io.vtable.batchCancel(io.userdata, b);
+        assert(b.submitted.head == .none and b.submitted.tail == .none);
+        assert(b.pending.head == .none and b.pending.tail == .none);
+        assert(b.context == null); // that was the last chance to deallocate resources
     }
 };
 @@ -1117,13 +1139,13 @@ pub fn recancel(io: Io) void {
 /// To modify a task's cancel protection state, see `swapCancelProtection`.
 ///
 /// For a description of cancelation and cancelation points, see `Future.cancel`.
-pub const CancelProtection = enum {
+pub const CancelProtection = enum(u1) {
     /// Any call to an `Io` function with `error.Canceled` in its error set is a cancelation point.
     ///
     /// This is the default state, which all tasks are created in.
-    unblocked,
+    unblocked = 0,
     /// No `Io` function introduces a cancelation point (`error.Canceled` will never be returned).
-    blocked,
+    blocked = 1,
 };
 /// Updates the current task's cancel protection state (see `CancelProtection`).
 ///
 @@ -1292,8 +1314,7 @@ pub fn futexWake(io: Io, comptime T: type, ptr: *align(@alignOf(u32)) const T, m
 /// shared region of code known as the "critical section".
 ///
 /// Mutex is an extern struct so that it may be used as a field inside another
-/// extern struct. Having a guaranteed memory layout including mutexes is
+/// extern struct.
-/// important for IPC over shared memory (mmap).
 pub const Mutex = extern struct {
     state: std.atomic.Value(State),

lib/std/Io/File.zig+7-2

 @@ -477,12 +477,17 @@ pub const Permissions = std.Options.FilePermissions orelse if (is_windows) enum(
     /// libc implementations use `0o666` inside `fopen` and then rely on the
     /// process-scoped "umask" setting to adjust this number for file creation.
     default_file = 0o666,
-    default_dir = 0o755,
+    /// This is the default mode given to POSIX operating systems for creating
-    executable_file = 0o777,
+    /// directories. `0o777` is "-rwxrwxrwx" which is counter-intuitive at first,
+    /// since most people would expect "-rwxr-xr-x", for example, when using
+    /// the `touch` command, which would correspond to `0o755`.
+    default_dir = 0o777,
     _,
     pub const has_executable_bit = native_os != .wasi;
+    pub const executable_file: @This() = .default_dir;
     pub fn toMode(self: @This()) std.posix.mode_t {
         return @intFromEnum(self);
     }

lib/std/Io/IoUring.zig+5380-774

...	@@ -1,21 +1,80 @@	...	@@ -1,21 +1,80 @@
1	const EventLoop = @This();	1	const addressFromPosix = Io.Threaded.addressFromPosix;
		2	const addressToPosix = Io.Threaded.addressToPosix;
		3	const Alignment = std.mem.Alignment;
		4	const Allocator = std.mem.Allocator;
		5	const Argv0 = Io.Threaded.Argv0;
		6	const assert = std.debug.assert;
2	const builtin = @import("builtin");	7	const builtin = @import("builtin");
3		8	const ChdirError = Io.Threaded.ChdirError;
4	const std = @import("../std.zig");	9	const clockToPosix = Io.Threaded.clockToPosix;
		10	const Csprng = Io.Threaded.Csprng;
		11	const default_PATH = Io.Threaded.default_PATH;
		12	const Dir = Io.Dir;
		13	const Environ = Io.Threaded.Environ;
		14	const errnoBug = Io.Threaded.errnoBug;
		15	const Evented = @This();
		16	const fallbackSeed = Io.Threaded.fallbackSeed;
		17	const fd_t = linux.fd_t;
		18	const File = Io.File;
5	const Io = std.Io;	19	const Io = std.Io;
6	const assert = std.debug.assert;	20	const IoUring = linux.IoUring;
7	const Allocator = std.mem.Allocator;	21	const iovec = std.posix.iovec;
8	const Alignment = std.mem.Alignment;	22	const iovec_const = std.posix.iovec_const;
9	const IoUring = std.os.linux.IoUring;	23	const linux = std.os.linux;
		24	const linux_statx_request = Io.Threaded.linux_statx_request;
		25	const LOCK = std.posix.LOCK;
		26	const log = std.log.scoped(.@"io-uring");
		27	const max_iovecs_len = Io.Threaded.max_iovecs_len;
		28	const nanosecondsFromPosix = Io.Threaded.nanosecondsFromPosix;
		29	const net = Io.net;
		30	const PATH_MAX = linux.PATH_MAX;
		31	const pathToPosix = Io.Threaded.pathToPosix;
		32	const pid_t = linux.pid_t;
		33	const PosixAddress = Io.Threaded.PosixAddress;
		34	const posixAddressFamily = Io.Threaded.posixAddressFamily;
		35	const posixProtocol = Io.Threaded.posixProtocol;
		36	const posixSocketMode = Io.Threaded.posixSocketMode;
		37	const process = std.process;
		38	const recoverableOsBugDetected = Io.Threaded.recoverableOsBugDetected;
		39	const setTimestampToPosix = Io.Threaded.setTimestampToPosix;
		40	const splat_buffer_size = Io.Threaded.splat_buffer_size;
		41	const statFromLinux = Io.Threaded.statFromLinux;
		42	const std = @import("../std.zig");
		43	const timestampFromPosix = Io.Threaded.timestampFromPosix;
		44	const unexpectedErrno = std.posix.unexpectedErrno;
		45	const winsize = std.posix.winsize;
10		46
11	/// Must be a thread-safe allocator.	47	backing_allocator_needs_mutex: bool,
12	gpa: Allocator,	48	backing_allocator_mutex: Io.Mutex,
13	mutex: Io.Mutex,	49	/// Does not need to be thread-safe if not used elsewhere.
14	main_fiber_buffer: [@sizeOf(Fiber) + Fiber.max_result_size]u8 align(@alignOf(Fiber)),	50	backing_allocator: Allocator,
		51	main_fiber_buffer: [
		52	std.mem.alignForward(usize, @sizeOf(Fiber), @alignOf(Completion)) + @sizeOf(Completion)
		53	]u8 align(@max(@alignOf(Fiber), @alignOf(Completion))),
15	threads: Thread.List,	54	threads: Thread.List,
16		55
		56	stderr_mutex: Io.Mutex,
		57	stderr_writer: File.Writer = .{
		58	.io = undefined,
		59	.interface = Io.File.Writer.initInterface(&.{}),
		60	.file = .stderr(),
		61	.mode = .streaming,
		62	},
		63	stderr_mode: Io.Terminal.Mode = .no_color,
		64	stderr_writer_initialized: bool = false,
		65
		66	environ_mutex: Io.Mutex,
		67	environ: Environ,
		68
		69	null_fd: CachedFd,
		70	random_fd: CachedFd,
		71
		72	csprng_mutex: Io.Mutex,
		73	csprng: Csprng,
		74
17	/// Empirically saw >128KB being used by the self-hosted backend to panic.	75	/// Empirically saw >128KB being used by the self-hosted backend to panic.
18	const idle_stack_size = 256 * 1024;	76	/// Empirically saw glibc complain about 256KB.
		77	const idle_stack_size = 512 * 1024;
19		78
20	const max_idle_search = 4;	79	const max_idle_search = 4;
21	const max_steal_ready_search = 4;	80	const max_steal_ready_search = 4;
...	@@ -23,6 +82,7 @@ const max_steal_ready_search = 4;	...	@@ -23,6 +82,7 @@ const max_steal_ready_search = 4;
23	const io_uring_entries = 64;	82	const io_uring_entries = 64;
24		83
25	const Thread = struct {	84	const Thread = struct {
		85	required_align: void align(4),
26	thread: std.Thread,	86	thread: std.Thread,
27	idle_context: Context,	87	idle_context: Context,
28	current_context: *Context,	88	current_context: *Context,
...	@@ -30,19 +90,33 @@ const Thread = struct {	...	@@ -30,19 +90,33 @@ const Thread = struct {
30	io_uring: IoUring,	90	io_uring: IoUring,
31	idle_search_index: u32,	91	idle_search_index: u32,
32	steal_ready_search_index: u32,	92	steal_ready_search_index: u32,
		93	csprng: Csprng,
33		94
34	const canceling: ?*Thread = @ptrFromInt(@alignOf(Thread));	95	threadlocal var self: ?*Thread = null;
35		96
36	threadlocal var self: *Thread = undefined;	97	noinline fn current() *Thread {
37		98	return self.?;
38	fn current() *Thread {
39	return self;
40	}	99	}
41		100
42	fn currentFiber(thread: Thread) Fiber {	101	fn currentFiber(thread: Thread) Fiber {
		102	assert(thread.current_context != &thread.idle_context);
43	return @fieldParentPtr("context", thread.current_context);	103	return @fieldParentPtr("context", thread.current_context);
44	}	104	}
45		105
		106	fn enqueue(thread: Thread) linux.io_uring_sqe {
		107	while (true) return thread.io_uring.get_sqe() catch {
		108	thread.submit();
		109	continue;
		110	};
		111	}
		112
		113	fn submit(thread: *Thread) void {
		114	_ = thread.io_uring.submit() catch \|err\| switch (err) {
		115	error.SignalInterrupt => {},
		116	else => \|e\| @panic(@errorName(e)),
		117	};
		118	}
		119
46	const List = struct {	120	const List = struct {
47	allocated: []Thread,	121	allocated: []Thread,
48	reserved: u32,	122	reserved: u32,
...	@@ -53,18 +127,109 @@ const Thread = struct {	...	@@ -53,18 +127,109 @@ const Thread = struct {
53	const Fiber = struct {	127	const Fiber = struct {
54	required_align: void align(4),	128	required_align: void align(4),
55	context: Context,	129	context: Context,
56	awaiter: ?*Fiber,	130	await_count: i32,
57	queue_next: ?*Fiber,	131	link: union {
58	cancel_thread: ?*Thread,	132	awaiter: ?*Fiber,
59	awaiting_completions: std.StaticBitSet(3),	133	group: struct { prev: ?Fiber, next: ?Fiber },
		134	},
		135	status: union(enum) {
		136	queue_next: ?*Fiber,
		137	awaiting_group: Group,
		138	},
		139	cancel_status: CancelStatus,
		140	cancel_protection: CancelProtection,
		141
		142	const CancelStatus = packed struct(u32) {
		143	requested: bool,
		144	awaiting: Awaiting,
		145
		146	const unrequested: CancelStatus = .{ .requested = false, .awaiting = .nothing };
		147
		148	const Awaiting = enum(u31) {
		149	nothing = std.math.maxInt(u31),
		150	group = std.math.maxInt(u31) - 1,
		151	select = std.math.maxInt(u31) - 2,
		152	/// An io_uring fd.
		153	_,
		154
		155	fn subWrap(lhs: Awaiting, rhs: Awaiting) Awaiting {
		156	return @enumFromInt(@intFromEnum(lhs) -% @intFromEnum(rhs));
		157	}
		158
		159	fn fromIoUringFd(fd: fd_t) Awaiting {
		160	const awaiting: Awaiting = @enumFromInt(fd);
		161	switch (awaiting) {
		162	.nothing, .group, .select => unreachable,
		163	_ => return awaiting,
		164	}
		165	}
		166
		167	fn toIoUringFd(awaiting: Awaiting) fd_t {
		168	switch (awaiting) {
		169	.nothing, .group => unreachable,
		170	_ => return @intFromEnum(awaiting),
		171	}
		172	}
		173	};
		174
		175	fn changeAwaiting(
		176	cancel_status: *CancelStatus,
		177	old_awaiting: Awaiting,
		178	new_awaiting: Awaiting,
		179	) bool {
		180	const old_cancel_status = @atomicRmw(CancelStatus, cancel_status, .Add, .{
		181	.requested = false,
		182	.awaiting = new_awaiting.subWrap(old_awaiting),
		183	}, .monotonic);
		184	assert(old_cancel_status.awaiting == old_awaiting);
		185	return old_cancel_status.requested;
		186	}
		187	};
		188
		189	const CancelProtection = packed struct {
		190	user: Io.CancelProtection,
		191	acknowledged: bool,
		192
		193	const unblocked: CancelProtection = .{ .user = .unblocked, .acknowledged = false };
		194
		195	fn check(cancel_protection: CancelProtection) Io.CancelProtection {
		196	return @enumFromInt(@intFromBool(cancel_protection != unblocked));
		197	}
		198
		199	fn acknowledge(cancel_protection: *CancelProtection) void {
		200	assert(!cancel_protection.acknowledged);
		201	cancel_protection.acknowledged = true;
		202	}
		203
		204	fn recancel(cancel_protection: *CancelProtection) void {
		205	assert(cancel_protection.acknowledged);
		206	cancel_protection.acknowledged = false;
		207	}
		208
		209	test check {
		210	try std.testing.expectEqual(Io.CancelProtection.unblocked, check(.unblocked));
		211	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
		212	.user = .unblocked,
		213	.acknowledged = true,
		214	}));
		215	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
		216	.user = .blocked,
		217	.acknowledged = false,
		218	}));
		219	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
		220	.user = .blocked,
		221	.acknowledged = true,
		222	}));
		223	}
		224	};
60		225
61	const finished: ?*Fiber = @ptrFromInt(@alignOf(Thread));	226	const finished: ?*Fiber = @ptrFromInt(@alignOf(Thread));
62		227
63	const max_result_align: Alignment = .@"16";	228	const max_result_align: Alignment = .@"16";
64	const max_result_size = max_result_align.forward(64);	229	const max_result_size = max_result_align.forward(512);
65	/// This includes any stack realignments that need to happen, and also the	230	/// This includes any stack realignments that need to happen, and also the
66	/// initial frame return address slot and argument frame, depending on target.	231	/// initial frame return address slot and argument frame, depending on target.
67	const min_stack_size = 4 * 1024 * 1024;	232	const min_stack_size = 60 * 1024 * 1024;
68	const max_context_align: Alignment = .@"16";	233	const max_context_align: Alignment = .@"16";
69	const max_context_size = max_context_align.forward(1024);	234	const max_context_size = max_context_align.forward(1024);
70	const max_closure_size: usize = @sizeOf(AsyncClosure);	235	const max_closure_size: usize = @sizeOf(AsyncClosure);
...	@@ -76,9 +241,19 @@ const Fiber = struct {	...	@@ -76,9 +241,19 @@ const Fiber = struct {
76	) + max_closure_size + max_context_size,	241	) + max_closure_size + max_context_size,
77	std.heap.page_size_max,	242	std.heap.page_size_max,
78	);	243	);
		244	comptime {
		245	assert(max_result_align.compare(.gte, .of(Completion)));
		246	assert(max_result_size >= @sizeOf(Completion));
		247	}
		248
		249	fn create(ev: Evented) error{OutOfMemory}!Fiber {
		250	return @ptrCast(try ev.allocator().alignedAlloc(u8, .of(Fiber), allocation_size));
		251	}
79		252
80	fn allocate(el: EventLoop) error{OutOfMemory}!Fiber {	253	fn destroy(fiber: *Fiber, gpa: std.mem.Allocator) void {
81	return @ptrCast(try el.gpa.alignedAlloc(u8, .of(Fiber), allocation_size));	254	log.debug("destroying {*}", .{fiber});
		255	assert(fiber.status.queue_next == null);
		256	gpa.free(fiber.allocatedSlice());
82	}	257	}
83		258
84	fn allocatedSlice(f: *Fiber) []align(@alignOf(Fiber)) u8 {	259	fn allocatedSlice(f: *Fiber) []align(@alignOf(Fiber)) u8 {
...	@@ -98,98 +273,513 @@ const Fiber = struct {	...	@@ -98,98 +273,513 @@ const Fiber = struct {
98	return @ptrFromInt(alignment.forward(@intFromPtr(f) + @sizeOf(Fiber)));	273	return @ptrFromInt(alignment.forward(@intFromPtr(f) + @sizeOf(Fiber)));
99	}	274	}
100		275
101	fn enterCancelRegion(fiber: Fiber, thread: Thread) error{Canceled}!void {	276	const Queue = struct { head: Fiber, tail: Fiber };
102	if (@cmpxchgStrong(	277
103	?*Thread,	278	/// Like a `*Fiber`, but 2 bits smaller than a pointer (because the LSBs are always 0 due to
104	&fiber.cancel_thread,	279	/// alignment) so that those two bits can be used in a `packed struct`.
105	null,	280	const PackedPtr = enum(@Int(.unsigned, @bitSizeOf(usize) - 2)) {
106	thread,	281	null = 0,
		282	all_ones = std.math.maxInt(@Int(.unsigned, @bitSizeOf(usize) - 2)),
		283	_,
		284
		285	const Split = packed struct(usize) { low: u2, high: PackedPtr };
		286	fn pack(ptr: ?*Fiber) PackedPtr {
		287	const split: Split = @bitCast(@intFromPtr(ptr));
		288	assert(split.low == 0);
		289	return split.high;
		290	}
		291	fn unpack(ptr: PackedPtr) ?*Fiber {
		292	const split: Split = .{ .low = 0, .high = ptr };
		293	return @ptrFromInt(@as(usize, @bitCast(split)));
		294	}
		295	};
		296
		297	fn requestCancel(fiber: Fiber, ev: Evented) void {
		298	const cancel_status = @atomicRmw(
		299	Fiber.CancelStatus,
		300	&fiber.cancel_status,
		301	.Or,
		302	.{ .requested = true, .awaiting = @enumFromInt(0) },
107	.acq_rel,	303	.acq_rel,
108	.acquire,	304	);
109	)) \|cancel_thread\| {	305	assert(!cancel_status.requested);
110	assert(cancel_thread == Thread.canceling);	306	switch (cancel_status.awaiting) {
		307	.nothing => {},
		308	.group => {
		309	// The awaiter received a cancelation request while awaiting a group,
		310	// so propagate the cancelation to the group.
		311	if (fiber.status.awaiting_group.cancel(ev, null)) {
		312	fiber.status = .{ .queue_next = null };
		313	_ = ev.schedule(.current(), .{ .head = fiber, .tail = fiber });
		314	}
		315	},
		316	.select => if (@atomicRmw(i32, &fiber.await_count, .Add, 1, .monotonic) == -1) {
		317	_ = ev.schedule(.current(), .{ .head = fiber, .tail = fiber });
		318	},
		319	_ => \|cancel_io_uring_fd\| {
		320	const thread: *Thread = .current();
		321	thread.enqueue().* = if (thread.io_uring.fd == @intFromEnum(cancel_io_uring_fd)) .{
		322	.opcode = .ASYNC_CANCEL,
		323	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
		324	.ioprio = 0,
		325	.fd = 0,
		326	.off = 0,
		327	.addr = @intFromPtr(fiber),
		328	.len = 0,
		329	.rw_flags = 0,
		330	.user_data = @intFromEnum(Completion.UserData.wakeup),
		331	.buf_index = 0,
		332	.personality = 0,
		333	.splice_fd_in = 0,
		334	.addr3 = 0,
		335	.resv = 0,
		336	} else .{
		337	.opcode = .MSG_RING,
		338	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
		339	.ioprio = 0,
		340	.fd = @intFromEnum(cancel_io_uring_fd),
		341	.off = @intFromPtr(fiber) \| 0b01,
		342	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
		343	.len = 0,
		344	.rw_flags = 0,
		345	.user_data = @intFromEnum(Completion.UserData.cleanup),
		346	.buf_index = 0,
		347	.personality = 0,
		348	.splice_fd_in = 0,
		349	.addr3 = 0,
		350	.resv = 0,
		351	};
		352	},
		353	}
		354	}
		355	};
		356
		357	const CancelRegion = struct {
		358	fiber: *Fiber,
		359	status: Fiber.CancelStatus,
		360	fn init() CancelRegion {
		361	const fiber = Thread.current().currentFiber();
		362	return .{
		363	.fiber = fiber,
		364	.status = .{
		365	.requested = fiber.cancel_protection.check() == .unblocked,
		366	.awaiting = .nothing,
		367	},
		368	};
		369	}
		370	fn initBlocked() CancelRegion {
		371	return .{
		372	.fiber = Thread.current().currentFiber(),
		373	.status = .{ .requested = false, .awaiting = .nothing },
		374	};
		375	}
		376	fn deinit(cancel_region: *CancelRegion) void {
		377	if (cancel_region.status.requested) _ = cancel_region.fiber.cancel_status.changeAwaiting(
		378	cancel_region.status.awaiting,
		379	.nothing,
		380	);
		381	cancel_region.* = undefined;
		382	}
		383	fn await(cancel_region: *CancelRegion, awaiting: Fiber.CancelStatus.Awaiting) Io.Cancelable!void {
		384	if (!cancel_region.status.requested) return;
		385	const status: Fiber.CancelStatus = .{ .requested = true, .awaiting = awaiting };
		386	if (cancel_region.fiber.cancel_status.changeAwaiting(
		387	cancel_region.status.awaiting,
		388	status.awaiting,
		389	)) {
		390	cancel_region.fiber.cancel_protection.acknowledge();
		391	cancel_region.status = .unrequested;
111	return error.Canceled;	392	return error.Canceled;
112	}	393	}
		394	cancel_region.status = status;
		395	}
		396	fn awaitIoUring(cancel_region: CancelRegion) Io.Cancelable!Thread {
		397	const thread: *Thread = .current();
		398	try cancel_region.await(.fromIoUringFd(thread.io_uring.fd));
		399	return thread;
113	}	400	}
		401	fn completion(cancel_region: *const CancelRegion) Completion {
		402	return cancel_region.fiber.resultPointer(Completion).*;
		403	}
		404	fn errno(cancel_region: *const CancelRegion) linux.E {
		405	return cancel_region.completion().errno();
		406	}
		407	};
114		408
115	fn exitCancelRegion(fiber: Fiber, thread: Thread) void {	409	const CachedFd = struct {
116	if (@cmpxchgStrong(	410	once: Once,
117	?*Thread,	411
118	&fiber.cancel_thread,	412	const Once = enum(fd_t) {
119	thread,	413	uninitialized = -1,
120	null,	414	initializing = -2,
121	.acq_rel,	415	/// fd
122	.acquire,	416	_,
123	)) \|cancel_thread\| assert(cancel_thread == Thread.canceling);	417
		418	fn fromFd(fd: fd_t) Once {
		419	return @enumFromInt(@as(u31, @intCast(fd)));
		420	}
		421
		422	fn toFd(once: Once) fd_t {
		423	return @as(u31, @intCast(@intFromEnum(once)));
		424	}
		425	};
		426
		427	const init: CachedFd = .{ .once = .uninitialized };
		428
		429	fn close(cached_fd: *CachedFd) void {
		430	switch (cached_fd.once) {
		431	.uninitialized => {},
		432	.initializing => unreachable,
		433	_ => \|fd\| {
		434	assert(@intFromEnum(fd) >= 0);
		435	std.posix.close(@intFromEnum(fd));
		436	cached_fd.* = .init;
		437	},
		438	}
124	}	439	}
125		440
126	const Queue = struct { head: Fiber, tail: Fiber };	441	fn open(
		442	cached_fd: *CachedFd,
		443	ev: *Evented,
		444	cancel_region: *CancelRegion,
		445	path: [*:0]const u8,
		446	flags: linux.O,
		447	) File.OpenError!fd_t {
		448	var once = @atomicLoad(Once, &cached_fd.once, .monotonic);
		449	while (true) {
		450	switch (once) {
		451	.uninitialized => {},
		452	.initializing => try futexWait(
		453	ev,
		454	@ptrCast(&cached_fd.once),
		455	@bitCast(@intFromEnum(once)),
		456	.none,
		457	),
		458	_ => \|fd\| {
		459	@branchHint(.likely);
		460	return fd.toFd();
		461	},
		462	}
		463	once = @cmpxchgWeak(
		464	Once,
		465	&cached_fd.once,
		466	.uninitialized,
		467	.initializing,
		468	.monotonic,
		469	.monotonic,
		470	) orelse {
		471	errdefer {
		472	@atomicStore(Once, &cached_fd.once, .uninitialized, .monotonic);
		473	futexWake(ev, @ptrCast(&cached_fd.once), 1);
		474	}
		475	const fd = try ev.openat(cancel_region, linux.AT.FDCWD, path, flags, 0);
		476	@atomicStore(Once, &cached_fd.once, .fromFd(fd), .monotonic);
		477	futexWake(ev, @ptrCast(&cached_fd.once), std.math.maxInt(u32));
		478	return fd;
		479	};
		480	}
		481	}
127	};	482	};
128		483
129	fn recycle(el: EventLoop, fiber: Fiber) void {	484	pub fn allocator(ev: *Evented) std.mem.Allocator {
130	std.log.debug("recyling {*}", .{fiber});	485	return if (ev.backing_allocator_needs_mutex) .{
131	assert(fiber.queue_next == null);	486	.ptr = ev,
132	el.gpa.free(fiber.allocatedSlice());	487	.vtable = &.{
		488	.alloc = alloc,
		489	.resize = resize,
		490	.remap = remap,
		491	.free = free,
		492	},
		493	} else ev.backing_allocator;
		494	}
		495
		496	fn alloc(userdata: anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[]u8 {
		497	const ev: *Evented = @ptrCast(@alignCast(userdata));
		498	const ev_io = ev.io();
		499	ev.backing_allocator_mutex.lockUncancelable(ev_io);
		500	defer ev.backing_allocator_mutex.unlock(ev_io);
		501	return ev.backing_allocator.rawAlloc(len, alignment, ret_addr);
		502	}
		503
		504	fn resize(
		505	userdata: *anyopaque,
		506	memory: []u8,
		507	alignment: std.mem.Alignment,
		508	new_len: usize,
		509	ret_addr: usize,
		510	) bool {
		511	const ev: *Evented = @ptrCast(@alignCast(userdata));
		512	const ev_io = ev.io();
		513	ev.backing_allocator_mutex.lockUncancelable(ev_io);
		514	defer ev.backing_allocator_mutex.unlock(ev_io);
		515	return ev.backing_allocator.rawResize(memory, alignment, new_len, ret_addr);
		516	}
		517
		518	fn remap(
		519	userdata: *anyopaque,
		520	memory: []u8,
		521	alignment: Alignment,
		522	new_len: usize,
		523	ret_addr: usize,
		524	) ?[*]u8 {
		525	const ev: *Evented = @ptrCast(@alignCast(userdata));
		526	const ev_io = ev.io();
		527	ev.backing_allocator_mutex.lockUncancelable(ev_io);
		528	defer ev.backing_allocator_mutex.unlock(ev_io);
		529	return ev.backing_allocator.rawRemap(memory, alignment, new_len, ret_addr);
133	}	530	}
134		531
135	pub fn io(el: *EventLoop) Io {	532	fn free(userdata: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void {
		533	const ev: *Evented = @ptrCast(@alignCast(userdata));
		534	const ev_io = ev.io();
		535	ev.backing_allocator_mutex.lockUncancelable(ev_io);
		536	defer ev.backing_allocator_mutex.unlock(ev_io);
		537	return ev.backing_allocator.rawFree(memory, alignment, ret_addr);
		538	}
		539
		540	pub fn io(ev: *Evented) Io {
136	return .{	541	return .{
137	.userdata = el,	542	.userdata = ev,
138	.vtable = &.{	543	.vtable = &.{
139	.async = async,	544	.async = async,
140	.concurrent = concurrent,	545	.concurrent = concurrent,
141	.await = await,	546	.await = await,
142	.select = select,
143	.cancel = cancel,	547	.cancel = cancel,
144	.cancelRequested = cancelRequested,
145		548
146	.mutexLock = mutexLock,	549	.groupAsync = groupAsync,
147	.mutexUnlock = mutexUnlock,	550	.groupConcurrent = groupConcurrent,
		551	.groupAwait = groupAwait,
		552	.groupCancel = groupCancel,
		553
		554	.recancel = recancel,
		555	.swapCancelProtection = swapCancelProtection,
		556	.checkCancel = checkCancel,
		557
		558	.select = select,
		559
		560	.futexWait = futexWait,
		561	.futexWaitUncancelable = futexWaitUncancelable,
		562	.futexWake = futexWake,
		563
		564	.operate = operate,
		565	.batchAwaitAsync = batchAwaitAsync,
		566	.batchAwaitConcurrent = batchAwaitConcurrent,
		567	.batchCancel = batchCancel,
148		568
149	.conditionWait = conditionWait,	569	.dirCreateDir = dirCreateDir,
150	.conditionWake = conditionWake,	570	.dirCreateDirPath = dirCreateDirPath,
		571	.dirCreateDirPathOpen = dirCreateDirPathOpen,
		572	.dirOpenDir = dirOpenDir,
		573	.dirStat = dirStat,
		574	.dirStatFile = dirStatFile,
		575	.dirAccess = dirAccess,
		576	.dirCreateFile = dirCreateFile,
		577	.dirCreateFileAtomic = dirCreateFileAtomic,
		578	.dirOpenFile = dirOpenFile,
		579	.dirClose = dirClose,
		580	.dirRead = dirRead,
		581	.dirRealPath = dirRealPath,
		582	.dirRealPathFile = dirRealPathFile,
		583	.dirDeleteFile = dirDeleteFile,
		584	.dirDeleteDir = dirDeleteDir,
		585	.dirRename = dirRename,
		586	.dirRenamePreserve = dirRenamePreserve,
		587	.dirSymLink = dirSymLink,
		588	.dirReadLink = dirReadLink,
		589	.dirSetOwner = dirSetOwner,
		590	.dirSetFileOwner = dirSetFileOwner,
		591	.dirSetPermissions = dirSetPermissions,
		592	.dirSetFilePermissions = dirSetFilePermissions,
		593	.dirSetTimestamps = dirSetTimestamps,
		594	.dirHardLink = dirHardLink,
151		595
152	.createFile = createFile,	596	.fileStat = fileStat,
153	.fileOpen = fileOpen,	597	.fileLength = fileLength,
154	.fileClose = fileClose,	598	.fileClose = fileClose,
155	.pread = pread,	599	.fileWritePositional = fileWritePositional,
156	.pwrite = pwrite,	600	.fileWriteFileStreaming = fileWriteFileStreaming,
		601	.fileWriteFilePositional = fileWriteFilePositional,
		602	.fileReadPositional = fileReadPositional,
		603	.fileSeekBy = fileSeekBy,
		604	.fileSeekTo = fileSeekTo,
		605	.fileSync = fileSync,
		606	.fileIsTty = fileIsTty,
		607	.fileEnableAnsiEscapeCodes = fileEnableAnsiEscapeCodes,
		608	.fileSupportsAnsiEscapeCodes = fileIsTty,
		609	.fileSetLength = fileSetLength,
		610	.fileSetOwner = fileSetOwner,
		611	.fileSetPermissions = fileSetPermissions,
		612	.fileSetTimestamps = fileSetTimestamps,
		613	.fileLock = fileLock,
		614	.fileTryLock = fileTryLock,
		615	.fileUnlock = fileUnlock,
		616	.fileDowngradeLock = fileDowngradeLock,
		617	.fileRealPath = fileRealPath,
		618	.fileHardLink = fileHardLink,
		619
		620	.fileMemoryMapCreate = fileMemoryMapCreate,
		621	.fileMemoryMapDestroy = fileMemoryMapDestroy,
		622	.fileMemoryMapSetLength = fileMemoryMapSetLength,
		623	.fileMemoryMapRead = fileMemoryMapRead,
		624	.fileMemoryMapWrite = fileMemoryMapWrite,
		625
		626	.processExecutableOpen = processExecutableOpen,
		627	.processExecutablePath = processExecutablePath,
		628	.lockStderr = lockStderr,
		629	.tryLockStderr = tryLockStderr,
		630	.unlockStderr = unlockStderr,
		631	.processCurrentPath = processCurrentPath,
		632	.processSetCurrentDir = processSetCurrentDir,
		633	.processReplace = processReplace,
		634	.processReplacePath = processReplacePath,
		635	.processSpawn = processSpawn,
		636	.processSpawnPath = processSpawnPath,
		637	.childWait = childWait,
		638	.childKill = childKill,
		639
		640	.progressParentFile = progressParentFile,
157		641
158	.now = now,	642	.now = now,
		643	.clockResolution = clockResolution,
159	.sleep = sleep,	644	.sleep = sleep,
		645
		646	.random = random,
		647	.randomSecure = randomSecure,
		648
		649	.netListenIp = netListenIpUnavailable,
		650	.netAccept = netAcceptUnavailable,
		651	.netBindIp = netBindIp,
		652	.netConnectIp = netConnectIpUnavailable,
		653	.netListenUnix = netListenUnixUnavailable,
		654	.netConnectUnix = netConnectUnixUnavailable,
		655	.netSocketCreatePair = netSocketCreatePairUnavailable,
		656	.netSend = netSendUnavailable,
		657	.netReceive = netReceive,
		658	.netRead = netReadUnavailable,
		659	.netWrite = netWriteUnavailable,
		660	.netWriteFile = netWriteFileUnavailable,
		661	.netClose = netClose,
		662	.netShutdown = netShutdown,
		663	.netInterfaceNameResolve = netInterfaceNameResolveUnavailable,
		664	.netInterfaceName = netInterfaceNameUnavailable,
		665	.netLookup = netLookupUnavailable,
160	},	666	},
161	};	667	};
162	}	668	}
163		669
164	pub fn init(el: *EventLoop, gpa: Allocator) !void {	670	fn fileMemoryMapSetLength(
		671	userdata: ?*anyopaque,
		672	mm: *File.MemoryMap,
		673	new_len: usize,
		674	) File.MemoryMap.SetLengthError!void {
		675	const ev: *Evented = @ptrCast(@alignCast(userdata));
		676	_ = ev;
		677	const page_size = std.heap.pageSize();
		678	const alignment: Alignment = .fromByteUnits(page_size);
		679	const page_align = std.heap.page_size_min;
		680	const old_memory = mm.memory;
		681
		682	if (alignment.forward(new_len) == alignment.forward(old_memory.len)) {
		683	mm.memory.len = new_len;
		684	return;
		685	}
		686	var cancel_region: CancelRegion = .init();
		687	defer cancel_region.deinit();
		688	const flags: linux.MREMAP = .{ .MAYMOVE = true };
		689	const addr_hint: ?[*]const u8 = null;
		690	const new_memory = while (true) {
		691	try cancel_region.await(.nothing);
		692	const rc = linux.mremap(old_memory.ptr, old_memory.len, new_len, flags, addr_hint);
		693	switch (linux.errno(rc)) {
		694	.SUCCESS => break @as([*]align(page_align) u8, @ptrFromInt(rc))[0..new_len],
		695	.INTR => continue,
		696	.AGAIN => return error.LockedMemoryLimitExceeded,
		697	.NOMEM => return error.OutOfMemory,
		698	.INVAL => \|err\| return errnoBug(err),
		699	.FAULT => \|err\| return errnoBug(err),
		700	else => \|err\| return unexpectedErrno(err),
		701	}
		702	};
		703	mm.memory = new_memory;
		704	}
		705
		706	fn fileMemoryMapRead(userdata: ?anyopaque, mm: File.MemoryMap) File.ReadPositionalError!void {
		707	const ev: *Evented = @ptrCast(@alignCast(userdata));
		708	_ = ev;
		709	_ = mm;
		710	}
		711
		712	fn fileMemoryMapWrite(userdata: ?anyopaque, mm: File.MemoryMap) File.WritePositionalError!void {
		713	const ev: *Evented = @ptrCast(@alignCast(userdata));
		714	_ = ev;
		715	_ = mm;
		716	}
		717
		718	pub const InitOptions = struct {
		719	backing_allocator_needs_mutex: bool = true,
		720
		721	/// Affects the following operations:
		722	/// * `processExecutablePath` on OpenBSD and Haiku.
		723	argv0: Argv0 = .empty,
		724	/// Affects the following operations:
		725	/// * `fileIsTty`
		726	/// * `processSpawn`, `processSpawnPath`, `processReplace`, `processReplacePath`
		727	environ: process.Environ,
		728	};
		729
		730	pub fn init(ev: *Evented, backing_allocator: Allocator, options: InitOptions) !void {
165	const threads_size = @max(std.Thread.getCpuCount() catch 1, 1) * @sizeOf(Thread);	731	const threads_size = @max(std.Thread.getCpuCount() catch 1, 1) * @sizeOf(Thread);
166	const idle_stack_end_offset = std.mem.alignForward(usize, threads_size + idle_stack_size, std.heap.page_size_max);	732	const idle_stack_end_offset =
167	const allocated_slice = try gpa.alignedAlloc(u8, .of(Thread), idle_stack_end_offset);	733	std.mem.alignForward(usize, threads_size + idle_stack_size, std.heap.page_size_max);
168	errdefer gpa.free(allocated_slice);	734	const allocated_slice = try backing_allocator.alignedAlloc(u8, .of(Thread), idle_stack_end_offset);
169	el.* = .{	735	errdefer backing_allocator.free(allocated_slice);
170	.gpa = gpa,	736	ev.* = .{
171	.mutex = .{},	737	.backing_allocator_needs_mutex = options.backing_allocator_needs_mutex,
		738	.backing_allocator_mutex = .init,
		739	.backing_allocator = backing_allocator,
172	.main_fiber_buffer = undefined,	740	.main_fiber_buffer = undefined,
173	.threads = .{	741	.threads = .{
174	.allocated = @ptrCast(allocated_slice[0..threads_size]),	742	.allocated = @ptrCast(allocated_slice[0..threads_size]),
175	.reserved = 1,	743	.reserved = 1,
176	.active = 1,	744	.active = 1,
177	},	745	},
		746
		747	.stderr_mutex = .init,
		748	.stderr_writer = .{
		749	.io = ev.io(),
		750	.interface = Io.File.Writer.initInterface(&.{}),
		751	.file = .stderr(),
		752	.mode = .streaming,
		753	},
		754	.stderr_mode = .no_color,
		755	.stderr_writer_initialized = false,
		756
		757	.environ_mutex = .init,
		758	.environ = .{ .process_environ = options.environ },
		759
		760	.null_fd = .init,
		761	.random_fd = .init,
		762
		763	.csprng_mutex = .init,
		764	.csprng = .uninitialized,
178	};	765	};
179	const main_fiber: *Fiber = @ptrCast(&el.main_fiber_buffer);	766	const main_fiber: *Fiber = @ptrCast(&ev.main_fiber_buffer);
180	main_fiber.* = .{	767	main_fiber.* = .{
181	.required_align = {},	768	.required_align = {},
182	.context = undefined,	769	.context = undefined,
183	.awaiter = null,	770	.await_count = 0,
184	.queue_next = null,	771	.link = .{ .awaiter = null },
185	.cancel_thread = null,	772	.status = .{ .queue_next = null },
186	.awaiting_completions = .initEmpty(),	773	.cancel_status = .unrequested,
		774	.cancel_protection = .unblocked,
187	};	775	};
188	const main_thread = &el.threads.allocated[0];	776	const main_thread = &ev.threads.allocated[0];
189	Thread.self = main_thread;	777	Thread.self = main_thread;
190	const idle_stack_end: [*]align(16) usize = @ptrCast(@alignCast(allocated_slice[idle_stack_end_offset..].ptr));	778	const idle_stack_end: [*]align(16) usize =
191	(idle_stack_end - 1)[0..1].* = .{@intFromPtr(el)};	779	@ptrCast(@alignCast(allocated_slice[idle_stack_end_offset..].ptr));
		780	(idle_stack_end - 1)[0..1].* = .{@intFromPtr(ev)};
192	main_thread.* = .{	781	main_thread.* = .{
		782	.required_align = {},
193	.thread = undefined,	783	.thread = undefined,
194	.idle_context = switch (builtin.cpu.arch) {	784	.idle_context = switch (builtin.cpu.arch) {
195	.aarch64 => .{	785	.aarch64 => .{
...	@@ -206,42 +796,56 @@ pub fn init(el: *EventLoop, gpa: Allocator) !void {	...	@@ -206,42 +796,56 @@ pub fn init(el: *EventLoop, gpa: Allocator) !void {
206	},	796	},
207	.current_context = &main_fiber.context,	797	.current_context = &main_fiber.context,
208	.ready_queue = null,	798	.ready_queue = null,
209	.io_uring = try IoUring.init(io_uring_entries, 0),	799	.io_uring = try .init(
		800	io_uring_entries,
		801	linux.IORING_SETUP_COOP_TASKRUN \| linux.IORING_SETUP_SINGLE_ISSUER,
		802	),
210	.idle_search_index = 1,	803	.idle_search_index = 1,
211	.steal_ready_search_index = 1,	804	.steal_ready_search_index = 1,
		805	.csprng = .uninitialized,
212	};	806	};
213	errdefer main_thread.io_uring.deinit();	807	errdefer main_thread.io_uring.deinit();
214	std.log.debug("created main idle {*}", .{&main_thread.idle_context});	808	log.debug("created main idle {*}", .{&main_thread.idle_context});
215	std.log.debug("created main {*}", .{main_fiber});	809	log.debug("created main {*}", .{main_fiber});
216	}	810	}
217		811
218	pub fn deinit(el: *EventLoop) void {	812	pub fn deinit(ev: *Evented) void {
219	const active_threads = @atomicLoad(u32, &el.threads.active, .acquire);	813	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
220	for (el.threads.allocated[0..active_threads]) \|*thread\| {	814	for (ev.threads.allocated[0..active_threads]) \|*thread\| {
221	const ready_fiber = @atomicLoad(?*Fiber, &thread.ready_queue, .monotonic);	815	const ready_fiber = @atomicLoad(?*Fiber, &thread.ready_queue, .monotonic);
222	assert(ready_fiber == null or ready_fiber == Fiber.finished); // pending async	816	assert(ready_fiber == null or ready_fiber == Fiber.finished); // pending async
223	}	817	}
224	el.yield(null, .exit);	818	ev.yield(null, .exit);
225	const allocated_ptr: [*]align(@alignOf(Thread)) u8 = @ptrCast(@alignCast(el.threads.allocated.ptr));	819	ev.threads.allocated[0].io_uring.deinit();
226	const idle_stack_end_offset = std.mem.alignForward(usize, el.threads.allocated.len * @sizeOf(Thread) + idle_stack_size, std.heap.page_size_max);	820	ev.null_fd.close();
227	for (el.threads.allocated[1..active_threads]) \|*thread\| thread.thread.join();	821	ev.random_fd.close();
228	el.gpa.free(allocated_ptr[0..idle_stack_end_offset]);	822	const allocated_ptr: [*]align(@alignOf(Thread)) u8 = @ptrCast(@alignCast(ev.threads.allocated.ptr));
229	el.* = undefined;	823	const idle_stack_end_offset = std.mem.alignForward(
		824	usize,
		825	ev.threads.allocated.len * @sizeOf(Thread) + idle_stack_size,
		826	std.heap.page_size_max,
		827	);
		828	for (ev.threads.allocated[1..active_threads]) \|*thread\| thread.thread.join();
		829	assert(active_threads == ev.threads.active); // spawned threads while there was no pending async?
		830	ev.backing_allocator.free(allocated_ptr[0..idle_stack_end_offset]);
		831	ev.* = undefined;
230	}	832	}
231		833
232	fn findReadyFiber(el: EventLoop, thread: Thread) ?*Fiber {	834	fn findReadyFiber(ev: Evented, thread: Thread) ?*Fiber {
233	if (@atomicRmw(?*Fiber, &thread.ready_queue, .Xchg, Fiber.finished, .acquire)) \|ready_fiber\| {	835	if (@atomicRmw(?*Fiber, &thread.ready_queue, .Xchg, Fiber.finished, .acquire)) \|ready_fiber\| {
234	@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.queue_next, .release);	836	@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.status.queue_next, .release);
235	ready_fiber.queue_next = null;	837	ready_fiber.status.queue_next = null;
236	return ready_fiber;	838	return ready_fiber;
237	}	839	}
238	const active_threads = @atomicLoad(u32, &el.threads.active, .acquire);	840	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
239	for (0..@min(max_steal_ready_search, active_threads)) \|_\| {	841	for (0..@min(max_steal_ready_search, active_threads)) \|_\| {
240	defer thread.steal_ready_search_index += 1;	842	defer thread.steal_ready_search_index += 1;
241	if (thread.steal_ready_search_index == active_threads) thread.steal_ready_search_index = 0;	843	if (thread.steal_ready_search_index == active_threads) thread.steal_ready_search_index = 0;
242	const steal_ready_search_thread = &el.threads.allocated[0..active_threads][thread.steal_ready_search_index];	844	const steal_ready_search_thread =
		845	&ev.threads.allocated[0..active_threads][thread.steal_ready_search_index];
243	if (steal_ready_search_thread == thread) continue;	846	if (steal_ready_search_thread == thread) continue;
244	const ready_fiber = @atomicLoad(?*Fiber, &steal_ready_search_thread.ready_queue, .acquire) orelse continue;	847	const ready_fiber =
		848	@atomicLoad(?*Fiber, &steal_ready_search_thread.ready_queue, .acquire) orelse continue;
245	if (ready_fiber == Fiber.finished) continue;	849	if (ready_fiber == Fiber.finished) continue;
246	if (@cmpxchgWeak(	850	if (@cmpxchgWeak(
247	?*Fiber,	851	?*Fiber,
...	@@ -251,8 +855,8 @@ fn findReadyFiber(el: EventLoop, thread: Thread) ?*Fiber {	...	@@ -251,8 +855,8 @@ fn findReadyFiber(el: EventLoop, thread: Thread) ?*Fiber {
251	.acquire,	855	.acquire,
252	.monotonic,	856	.monotonic,
253	)) \|_\| continue;	857	)) \|_\| continue;
254	@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.queue_next, .release);	858	@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.status.queue_next, .release);
255	ready_fiber.queue_next = null;	859	ready_fiber.status.queue_next = null;
256	return ready_fiber;	860	return ready_fiber;
257	}	861	}
258	// couldn't find anything to do, so we are now open for business	862	// couldn't find anything to do, so we are now open for business
...	@@ -260,9 +864,9 @@ fn findReadyFiber(el: EventLoop, thread: Thread) ?*Fiber {	...	@@ -260,9 +864,9 @@ fn findReadyFiber(el: EventLoop, thread: Thread) ?*Fiber {
260	return null;	864	return null;
261	}	865	}
262		866
263	fn yield(el: EventLoop, maybe_ready_fiber: ?Fiber, pending_task: SwitchMessage.PendingTask) void {	867	fn yield(ev: Evented, maybe_ready_fiber: ?Fiber, pending_task: SwitchMessage.PendingTask) void {
264	const thread: *Thread = .current();	868	const thread: *Thread = .current();
265	const ready_context = if (maybe_ready_fiber orelse el.findReadyFiber(thread)) \|ready_fiber\|	869	const ready_context = if (maybe_ready_fiber orelse ev.findReadyFiber(thread)) \|ready_fiber\|
266	&ready_fiber.context	870	&ready_fiber.context
267	else	871	else
268	&thread.idle_context;	872	&thread.idle_context;
...	@@ -273,25 +877,25 @@ fn yield(el: EventLoop, maybe_ready_fiber: ?Fiber, pending_task: SwitchMessage	...	@@ -273,25 +877,25 @@ fn yield(el: EventLoop, maybe_ready_fiber: ?Fiber, pending_task: SwitchMessage
273	},	877	},
274	.pending_task = pending_task,	878	.pending_task = pending_task,
275	};	879	};
276	std.log.debug("switching from {} to {}", .{ message.contexts.prev, message.contexts.ready });	880	log.debug("switching from {} to {}", .{ message.contexts.prev, message.contexts.ready });
277	contextSwitch(&message).handle(el);	881	contextSwitch(&message).handle(ev);
278	}	882	}
279		883
280	fn schedule(el: EventLoop, thread: Thread, ready_queue: Fiber.Queue) void {	884	fn schedule(ev: Evented, thread: Thread, ready_queue: Fiber.Queue) bool {
281	{	885	{
282	var fiber = ready_queue.head;	886	var fiber = ready_queue.head;
283	while (true) {	887	while (true) {
284	std.log.debug("scheduling {*}", .{fiber});	888	log.debug("scheduling {*}", .{fiber});
285	fiber = fiber.queue_next orelse break;	889	fiber = fiber.status.queue_next orelse break;
286	}	890	}
287	assert(fiber == ready_queue.tail);	891	assert(fiber == ready_queue.tail);
288	}	892	}
289	// shared fields of previous `Thread` must be initialized before later ones are marked as active	893	// shared fields of previous `Thread` must be initialized before later ones are marked as active
290	const new_thread_index = @atomicLoad(u32, &el.threads.active, .acquire);	894	const new_thread_index = @atomicLoad(u32, &ev.threads.active, .acquire);
291	for (0..@min(max_idle_search, new_thread_index)) \|_\| {	895	for (0..@min(max_idle_search, new_thread_index)) \|_\| {
292	defer thread.idle_search_index += 1;	896	defer thread.idle_search_index += 1;
293	if (thread.idle_search_index == new_thread_index) thread.idle_search_index = 0;	897	if (thread.idle_search_index == new_thread_index) thread.idle_search_index = 0;
294	const idle_search_thread = &el.threads.allocated[0..new_thread_index][thread.idle_search_index];	898	const idle_search_thread = &ev.threads.allocated[0..new_thread_index][thread.idle_search_index];
295	if (idle_search_thread == thread) continue;	899	if (idle_search_thread == thread) continue;
296	if (@cmpxchgWeak(	900	if (@cmpxchgWeak(
297	?*Fiber,	901	?*Fiber,
...	@@ -301,13 +905,13 @@ fn schedule(el: EventLoop, thread: Thread, ready_queue: Fiber.Queue) void {	...	@@ -301,13 +905,13 @@ fn schedule(el: EventLoop, thread: Thread, ready_queue: Fiber.Queue) void {
301	.release,	905	.release,
302	.monotonic,	906	.monotonic,
303	)) \|_\| continue;	907	)) \|_\| continue;
304	getSqe(&thread.io_uring).* = .{	908	thread.enqueue().* = .{
305	.opcode = .MSG_RING,	909	.opcode = .MSG_RING,
306	.flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS,	910	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
307	.ioprio = 0,	911	.ioprio = 0,
308	.fd = idle_search_thread.io_uring.fd,	912	.fd = idle_search_thread.io_uring.fd,
309	.off = @intFromEnum(Completion.UserData.wakeup),	913	.off = @intFromEnum(Completion.UserData.wakeup),
310	.addr = 0,	914	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
311	.len = 0,	915	.len = 0,
312	.rw_flags = 0,	916	.rw_flags = 0,
313	.user_data = @intFromEnum(Completion.UserData.wakeup),	917	.user_data = @intFromEnum(Completion.UserData.wakeup),
...	@@ -317,145 +921,221 @@ fn schedule(el: EventLoop, thread: Thread, ready_queue: Fiber.Queue) void {	...	@@ -317,145 +921,221 @@ fn schedule(el: EventLoop, thread: Thread, ready_queue: Fiber.Queue) void {
317	.addr3 = 0,	921	.addr3 = 0,
318	.resv = 0,	922	.resv = 0,
319	};	923	};
320	return;	924	return true;
321	}	925	}
322	spawn_thread: {	926	spawn_thread: {
323	// previous failed reservations must have completed before retrying	927	// previous failed reservations must have completed before retrying
324	if (new_thread_index == el.threads.allocated.len or @cmpxchgWeak(	928	if (new_thread_index == ev.threads.allocated.len or @cmpxchgWeak(
325	u32,	929	u32,
326	&el.threads.reserved,	930	&ev.threads.reserved,
327	new_thread_index,	931	new_thread_index,
328	new_thread_index + 1,	932	new_thread_index + 1,
329	.acquire,	933	.acquire,
330	.monotonic,	934	.monotonic,
331	) != null) break :spawn_thread;	935	) != null) break :spawn_thread;
332	const new_thread = &el.threads.allocated[new_thread_index];	936	const new_thread = &ev.threads.allocated[new_thread_index];
333	const next_thread_index = new_thread_index + 1;	937	const next_thread_index = new_thread_index + 1;
		938	var params = std.mem.zeroInit(linux.io_uring_params, .{
		939	.flags = linux.IORING_SETUP_ATTACH_WQ \|
		940	linux.IORING_SETUP_R_DISABLED \|
		941	linux.IORING_SETUP_COOP_TASKRUN \|
		942	linux.IORING_SETUP_SINGLE_ISSUER,
		943	.wq_fd = @as(u32, @intCast(ev.threads.allocated[0].io_uring.fd)),
		944	});
334	new_thread.* = .{	945	new_thread.* = .{
		946	.required_align = {},
335	.thread = undefined,	947	.thread = undefined,
336	.idle_context = undefined,	948	.idle_context = undefined,
337	.current_context = &new_thread.idle_context,	949	.current_context = &new_thread.idle_context,
338	.ready_queue = ready_queue.head,	950	.ready_queue = ready_queue.head,
339	.io_uring = IoUring.init(io_uring_entries, 0) catch \|err\| {	951	.io_uring = IoUring.init_params(io_uring_entries, &params) catch \|err\| {
340	@atomicStore(u32, &el.threads.reserved, new_thread_index, .release);	952	@atomicStore(u32, &ev.threads.reserved, new_thread_index, .release);
341	// no more access to `thread` after giving up reservation	953	// no more access to `thread` after giving up reservation
342	std.log.warn("unable to create worker thread due to io_uring init failure: {s}", .{@errorName(err)});	954	log.warn("unable to create worker thread due to io_uring init failure: {s}", .{
		955	@errorName(err),
		956	});
343	break :spawn_thread;	957	break :spawn_thread;
344	},	958	},
345	.idle_search_index = 0,	959	.idle_search_index = 0,
346	.steal_ready_search_index = 0,	960	.steal_ready_search_index = 0,
		961	.csprng = .uninitialized,
347	};	962	};
348	new_thread.thread = std.Thread.spawn(.{	963	new_thread.thread = std.Thread.spawn(.{
349	.stack_size = idle_stack_size,	964	.stack_size = idle_stack_size,
350	.allocator = el.gpa,	965	.allocator = ev.allocator(),
351	}, threadEntry, .{ el, new_thread_index }) catch \|err\| {	966	}, threadEntry, .{ ev, new_thread_index }) catch \|err\| {
352	new_thread.io_uring.deinit();	967	new_thread.io_uring.deinit();
353	@atomicStore(u32, &el.threads.reserved, new_thread_index, .release);	968	@atomicStore(u32, &ev.threads.reserved, new_thread_index, .release);
354	// no more access to `thread` after giving up reservation	969	// no more access to `thread` after giving up reservation
355	std.log.warn("unable to create worker thread due spawn failure: {s}", .{@errorName(err)});	970	log.warn("unable to create worker thread due spawn failure: {s}", .{@errorName(err)});
356	break :spawn_thread;	971	break :spawn_thread;
357	};	972	};
358	// shared fields of `Thread` must be initialized before being marked active	973	// shared fields of `Thread` must be initialized before being marked active
359	@atomicStore(u32, &el.threads.active, next_thread_index, .release);	974	@atomicStore(u32, &ev.threads.active, next_thread_index, .release);
360	return;	975	return false;
361	}	976	}
362	// nobody wanted it, so just queue it on ourselves	977	// nobody wanted it, so just queue it on ourselves
363	while (@cmpxchgWeak(	978	while (@cmpxchgWeak(
364	?*Fiber,	979	?*Fiber,
365	&thread.ready_queue,	980	&thread.ready_queue,
366	ready_queue.tail.queue_next,	981	ready_queue.tail.status.queue_next,
367	ready_queue.head,	982	ready_queue.head,
368	.acq_rel,	983	.acq_rel,
369	.acquire,	984	.acquire,
370	)) \|old_head\| ready_queue.tail.queue_next = old_head;	985	)) \|old_head\| ready_queue.tail.status.queue_next = old_head;
		986	return false;
371	}	987	}
372		988
373	fn mainIdle(el: EventLoop, message: const SwitchMessage) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Context)))) noreturn {	989	fn mainIdle(
374	message.handle(el);	990	ev: *Evented,
375	el.idle(&el.threads.allocated[0]);	991	message: *const SwitchMessage,
376	el.yield(@ptrCast(&el.main_fiber_buffer), .nothing);	992	) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Context)))) noreturn {
		993	message.handle(ev);
		994	ev.idle(&ev.threads.allocated[0]);
		995	ev.yield(@ptrCast(&ev.main_fiber_buffer), .nothing);
377	unreachable; // switched to dead fiber	996	unreachable; // switched to dead fiber
378	}	997	}
379		998
380	fn threadEntry(el: *EventLoop, index: u32) void {	999	fn threadEntry(ev: *Evented, index: u32) void {
381	const thread: *Thread = &el.threads.allocated[index];	1000	const thread: *Thread = &ev.threads.allocated[index];
382	Thread.self = thread;	1001	Thread.self = thread;
383	std.log.debug("created thread idle {*}", .{&thread.idle_context});	1002	defer thread.io_uring.deinit();
384	el.idle(thread);	1003	log.debug("created thread idle {*}", .{&thread.idle_context});
		1004	switch (linux.errno(linux.io_uring_register(thread.io_uring.fd, .REGISTER_ENABLE_RINGS, null, 0))) {
		1005	.SUCCESS => ev.idle(thread),
		1006	else => \|err\| @panic(@tagName(err)),
		1007	}
385	}	1008	}
386		1009
387	const Completion = struct {	1010	const Completion = struct {
		1011	result: i32,
		1012	flags: u32,
		1013
388	const UserData = enum(usize) {	1014	const UserData = enum(usize) {
389	unused,	1015	unused,
390	wakeup,	1016	wakeup,
		1017	futex_wake,
391	cleanup,	1018	cleanup,
392	exit,	1019	exit,
393	/// *Fiber	1020	/// If bit 0 is 1, a pointer to the `context` field of `Io.Batch.Storage.Pending`.
		1021	/// If bits 0 and 1 are 0, a `*Fiber`.
394	_,	1022	_,
395	};	1023	};
396	result: i32,	1024
397	flags: u32,	1025	fn errno(completion: Completion) linux.E {
		1026	return linux.errno(@bitCast(@as(isize, completion.result)));
		1027	}
398	};	1028	};
399		1029
400	fn idle(el: EventLoop, thread: Thread) void {	1030	fn idle(ev: Evented, thread: Thread) void {
401	var maybe_ready_fiber: ?*Fiber = null;	1031	var maybe_ready_fiber: ?*Fiber = null;
402	while (true) {	1032	while (true) {
403	while (maybe_ready_fiber orelse el.findReadyFiber(thread)) \|ready_fiber\| {	1033	while (maybe_ready_fiber orelse ev.findReadyFiber(thread)) \|ready_fiber\| {
404	el.yield(ready_fiber, .nothing);	1034	ev.yield(ready_fiber, .nothing);
405	maybe_ready_fiber = null;	1035	maybe_ready_fiber = null;
406	}	1036	}
407	_ = thread.io_uring.submit_and_wait(1) catch \|err\| switch (err) {	1037	_ = thread.io_uring.submit_and_wait(1) catch \|err\| switch (err) {
408	error.SignalInterrupt => std.log.warn("submit_and_wait failed with SignalInterrupt", .{}),	1038	error.SignalInterrupt => {},
409	else => \|e\| @panic(@errorName(e)),	1039	else => \|e\| @panic(@errorName(e)),
410	};	1040	};
411	var cqes_buffer: [io_uring_entries]std.os.linux.io_uring_cqe = undefined;	1041	var cqes_buffer: [io_uring_entries]linux.io_uring_cqe = undefined;
412	var maybe_ready_queue: ?Fiber.Queue = null;	1042	var maybe_ready_queue: ?Fiber.Queue = null;
413	for (cqes_buffer[0 .. thread.io_uring.copy_cqes(&cqes_buffer, 0) catch \|err\| switch (err) {	1043	for (cqes_buffer[0 .. thread.io_uring.copy_cqes(&cqes_buffer, 0) catch \|err\| switch (err) {
414	error.SignalInterrupt => cqes_len: {	1044	error.SignalInterrupt => 0,
415	std.log.warn("copy_cqes failed with SignalInterrupt", .{});
416	break :cqes_len 0;
417	},
418	else => \|e\| @panic(@errorName(e)),	1045	else => \|e\| @panic(@errorName(e)),
419	}]) \|cqe\| switch (@as(Completion.UserData, @enumFromInt(cqe.user_data))) {	1046	}]) \|cqe\| if (cqe.flags & linux.IORING_CQE_F_SKIP == 0) switch (@as(
		1047	Completion.UserData,
		1048	@enumFromInt(cqe.user_data),
		1049	)) {
420	.unused => unreachable, // bad submission queued?	1050	.unused => unreachable, // bad submission queued?
421	.wakeup => {},	1051	.wakeup => {},
		1052	.futex_wake => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
		1053	.SUCCESS => recoverableOsBugDetected(), // success is skipped
		1054	.INVAL => {}, // invalid futex_wait() on ptr done elsewhere
		1055	.INTR, .CANCELED => recoverableOsBugDetected(), // `Completion.UserData.futex_wake` is not cancelable
		1056	.FAULT => {}, // pointer became invalid while doing the wake
		1057	else => recoverableOsBugDetected(), // deadlock due to operating system bug
		1058	},
422	.cleanup => @panic("failed to notify other threads that we are exiting"),	1059	.cleanup => @panic("failed to notify other threads that we are exiting"),
423	.exit => {	1060	.exit => {
424	assert(maybe_ready_fiber == null and maybe_ready_queue == null); // pending async	1061	assert(maybe_ready_fiber == null and maybe_ready_queue == null); // pending async
425	return;	1062	return;
426	},	1063	},
427	_ => switch (errno(cqe.res)) {	1064	_ => if (@as(?*Fiber, ready_fiber: switch (@as(u2, @truncate(cqe.user_data))) {
428	.INTR => getSqe(&thread.io_uring).* = .{	1065	0b00 => {
429	.opcode = .ASYNC_CANCEL,	1066	const ready_fiber: *Fiber = @ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
430	.flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS,	1067	ready_fiber.resultPointer(Completion).* = .{
431	.ioprio = 0,
432	.fd = 0,
433	.off = 0,
434	.addr = cqe.user_data,
435	.len = 0,
436	.rw_flags = 0,
437	.user_data = @intFromEnum(Completion.UserData.wakeup),
438	.buf_index = 0,
439	.personality = 0,
440	.splice_fd_in = 0,
441	.addr3 = 0,
442	.resv = 0,
443	},
444	else => {
445	const fiber: *Fiber = @ptrFromInt(cqe.user_data);
446	assert(fiber.queue_next == null);
447	fiber.resultPointer(Completion).* = .{
448	.result = cqe.res,	1068	.result = cqe.res,
449	.flags = cqe.flags,	1069	.flags = cqe.flags,
450	};	1070	};
451	if (maybe_ready_fiber == null) maybe_ready_fiber = fiber else if (maybe_ready_queue) \|*ready_queue\| {	1071	break :ready_fiber ready_fiber;
452	ready_queue.tail.queue_next = fiber;	1072	},
453	ready_queue.tail = fiber;	1073	0b01 => {
454	} else maybe_ready_queue = .{ .head = fiber, .tail = fiber };	1074	thread.enqueue().* = .{
		1075	.opcode = .ASYNC_CANCEL,
		1076	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
		1077	.ioprio = 0,
		1078	.fd = 0,
		1079	.off = 0,
		1080	.addr = cqe.user_data & ~@as(usize, 0b11),
		1081	.len = 0,
		1082	.rw_flags = 0,
		1083	.user_data = @intFromEnum(Completion.UserData.wakeup),
		1084	.buf_index = 0,
		1085	.personality = 0,
		1086	.splice_fd_in = 0,
		1087	.addr3 = 0,
		1088	.resv = 0,
		1089	};
		1090	break :ready_fiber null;
		1091	},
		1092	0b10 => {
		1093	const context: *Io.Operation.Storage.Pending.Context =
		1094	@ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
		1095	const batch: *Io.Batch = @ptrFromInt(context[0]);
		1096	var next: usize = 0b00;
		1097	context[0..3].* = .{ next, @as(u32, @bitCast(cqe.res)), cqe.flags };
		1098	while (true) {
		1099	next = @cmpxchgWeak(
		1100	usize,
		1101	@as(*usize, @ptrCast(&batch.context)),
		1102	next,
		1103	cqe.user_data,
		1104	.release,
		1105	.acquire,
		1106	) orelse break;
		1107	context[0] = next;
		1108	}
		1109	break :ready_fiber switch (@as(u2, @truncate(next))) {
		1110	0b00, 0b01 => @ptrFromInt(next & ~@as(usize, 0b11)),
		1111	0b10, 0b11 => null,
		1112	};
		1113	},
		1114	0b11 => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
		1115	.SUCCESS => unreachable, // no event count specified
		1116	.TIME => {
		1117	const context: *usize = @ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
		1118	const fiber = @atomicRmw(usize, context, .Add, 0b01, .acquire);
		1119	break :ready_fiber switch (@as(u2, @truncate(fiber))) {
		1120	else => unreachable, // timeout completed multiple times
		1121	0b00 => @ptrFromInt(fiber & ~@as(usize, 0b11)),
		1122	0b10 => null,
		1123	};
		1124	},
		1125	.CANCELED => null, // user data may have been invalidated
		1126	else => \|err\| unexpectedErrno(err) catch null,
455	},	1127	},
		1128	})) \|ready_fiber\| {
		1129	assert(ready_fiber.status.queue_next == null);
		1130	if (maybe_ready_fiber == null) {
		1131	maybe_ready_fiber = ready_fiber;
		1132	} else if (maybe_ready_queue) \|*ready_queue\| {
		1133	ready_queue.tail.status.queue_next = ready_fiber;
		1134	ready_queue.tail = ready_fiber;
		1135	} else maybe_ready_queue = .{ .head = ready_fiber, .tail = ready_fiber };
456	},	1136	},
457	};	1137	};
458	if (maybe_ready_queue) \|ready_queue\| el.schedule(thread, ready_queue);	1138	if (maybe_ready_queue) \|ready_queue\| _ = ev.schedule(thread, ready_queue);
459	}	1139	}
460	}	1140	}
461		1141
...	@@ -469,113 +1149,68 @@ const SwitchMessage = struct {	...	@@ -469,113 +1149,68 @@ const SwitchMessage = struct {
469	const PendingTask = union(enum) {	1149	const PendingTask = union(enum) {
470	nothing,	1150	nothing,
471	reschedule,	1151	reschedule,
472	recycle: *Fiber,	1152	await: u31,
473	register_awaiter: ?Fiber,	1153	group_await: Group,
474	register_select: []const *Io.AnyFuture,	1154	group_cancel: Group,
475	mutex_lock: struct {	1155	batch_await: *Io.Batch,
476	prev_state: Io.Mutex.State,	1156	destroy,
477	mutex: *Io.Mutex,
478	},
479	condition_wait: struct {
480	cond: *Io.Condition,
481	mutex: *Io.Mutex,
482	},
483	exit,	1157	exit,
484	};	1158	};
485		1159
486	fn handle(message: const SwitchMessage, el: EventLoop) void {	1160	fn handle(message: const SwitchMessage, ev: Evented) void {
487	const thread: *Thread = .current();	1161	const thread: *Thread = .current();
488	thread.current_context = message.contexts.ready;	1162	thread.current_context = message.contexts.ready;
489	switch (message.pending_task) {	1163	switch (message.pending_task) {
490	.nothing => {},	1164	.nothing => {},
491	.reschedule => if (message.contexts.prev != &thread.idle_context) {	1165	.reschedule => if (message.contexts.prev != &thread.idle_context) {
492	const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));	1166	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
493	assert(prev_fiber.queue_next == null);	1167	assert(fiber.status.queue_next == null);
494	el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });	1168	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
495	},
496	.recycle => \|fiber\| {
497	el.recycle(fiber);
498	},	1169	},
499	.register_awaiter => \|awaiter\| {	1170	.await => \|count\| {
500	const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));	1171	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
501	assert(prev_fiber.queue_next == null);	1172	if (@atomicRmw(i32, &fiber.await_count, .Sub, count, .monotonic) > 0)
502	if (@atomicRmw(?*Fiber, awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished)	1173	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
503	el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });
504	},	1174	},
505	.register_select => \|futures\| {	1175	.group_await => \|group\| {
506	const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));	1176	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
507	assert(prev_fiber.queue_next == null);	1177	if (group.await(ev, fiber))
508	for (futures) \|any_future\| {	1178	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
509	const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));
510	if (@atomicRmw(?*Fiber, &future_fiber.awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished) {
511	const closure: *AsyncClosure = .fromFiber(future_fiber);
512	if (!@atomicRmw(bool, &closure.already_awaited, .Xchg, true, .seq_cst)) {
513	el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });
514	}
515	}
516	}
517	},	1179	},
518	.mutex_lock => \|mutex_lock\| {	1180	.group_cancel => \|group\| {
519	const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));	1181	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
520	assert(prev_fiber.queue_next == null);	1182	if (group.cancel(ev, fiber))
521	var prev_state = mutex_lock.prev_state;	1183	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
522	while (switch (prev_state) {
523	else => next_state: {
524	prev_fiber.queue_next = @ptrFromInt(@intFromEnum(prev_state));
525	break :next_state @cmpxchgWeak(
526	Io.Mutex.State,
527	&mutex_lock.mutex.state,
528	prev_state,
529	@enumFromInt(@intFromPtr(prev_fiber)),
530	.release,
531	.acquire,
532	);
533	},
534	.unlocked => @cmpxchgWeak(
535	Io.Mutex.State,
536	&mutex_lock.mutex.state,
537	.unlocked,
538	.locked_once,
539	.acquire,
540	.acquire,
541	) orelse {
542	prev_fiber.queue_next = null;
543	el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });
544	return;
545	},
546	}) \|next_state\| prev_state = next_state;
547	},	1184	},
548	.condition_wait => \|condition_wait\| {	1185	.batch_await => \|batch\| {
549	const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));	1186	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
550	assert(prev_fiber.queue_next == null);
551	const cond_impl = prev_fiber.resultPointer(ConditionImpl);
552	cond_impl.* = .{
553	.tail = prev_fiber,
554	.event = .queued,
555	};
556	if (@cmpxchgStrong(	1187	if (@cmpxchgStrong(
557	?*Fiber,	1188	?*anyopaque,
558	@as(?Fiber, @ptrCast(&condition_wait.cond.state)),	1189	&batch.context,
559	null,	1190	null,
560	prev_fiber,	1191	fiber,
561	.release,	1192	.release,
562	.acquire,	1193	.monotonic,
563	)) \|waiting_fiber\| {	1194	)) \|head\| {
564	const waiting_cond_impl = waiting_fiber.?.resultPointer(ConditionImpl);	1195	assert(@as(u2, @truncate(@intFromPtr(head))) != 0b00);
565	assert(waiting_cond_impl.tail.queue_next == null);	1196	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
566	waiting_cond_impl.tail.queue_next = prev_fiber;
567	waiting_cond_impl.tail = prev_fiber;
568	}	1197	}
569	condition_wait.mutex.unlock(el.io());
570	},	1198	},
571	.exit => for (el.threads.allocated[0..@atomicLoad(u32, &el.threads.active, .acquire)]) \|*each_thread\| {	1199	.destroy => {
572	getSqe(&thread.io_uring).* = .{	1200	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
		1201	fiber.destroy(ev.backing_allocator);
		1202	ev.backing_allocator_mutex.unlock(ev.io());
		1203	},
		1204	.exit => for (
		1205	ev.threads.allocated[0..@atomicLoad(u32, &ev.threads.active, .acquire)],
		1206	) \|*each_thread\| {
		1207	thread.enqueue().* = .{
573	.opcode = .MSG_RING,	1208	.opcode = .MSG_RING,
574	.flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS,	1209	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
575	.ioprio = 0,	1210	.ioprio = 0,
576	.fd = each_thread.io_uring.fd,	1211	.fd = each_thread.io_uring.fd,
577	.off = @intFromEnum(Completion.UserData.exit),	1212	.off = @intFromEnum(Completion.UserData.exit),
578	.addr = 0,	1213	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
579	.len = 0,	1214	.len = 0,
580	.rw_flags = 0,	1215	.rw_flags = 0,
581	.user_data = @intFromEnum(Completion.UserData.cleanup),	1216	.user_data = @intFromEnum(Completion.UserData.cleanup),
...	@@ -784,65 +1419,73 @@ inline fn contextSwitch(message: const SwitchMessage) const SwitchMessage {	...	@@ -784,65 +1419,73 @@ inline fn contextSwitch(message: const SwitchMessage) const SwitchMessage {
784		1419
785	fn mainIdleEntry() callconv(.naked) void {	1420	fn mainIdleEntry() callconv(.naked) void {
786	switch (builtin.cpu.arch) {	1421	switch (builtin.cpu.arch) {
787	.x86_64 => asm volatile (
788	\\ movq (%%rsp), %%rdi
789	\\ jmp %[mainIdle:P]
790	:
791	: [mainIdle] "X" (&mainIdle),
792	),
793	.aarch64 => asm volatile (	1422	.aarch64 => asm volatile (
794	\\ ldr x0, [sp, #-8]	1423	\\ ldr x0, [sp, #-8]
795	\\ b %[mainIdle]	1424	\\ b %[mainIdle]
796	:	1425	:
797	: [mainIdle] "X" (&mainIdle),	1426	: [mainIdle] "X" (&mainIdle),
798	),	1427	),
799	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
800	}
801	}
802
803	fn fiberEntry() callconv(.naked) void {
804	switch (builtin.cpu.arch) {
805	.x86_64 => asm volatile (	1428	.x86_64 => asm volatile (
806	\\ leaq 8(%%rsp), %%rdi	1429	\\ movq (%%rsp), %%rdi
807	\\ jmp %[AsyncClosure_call:P]	1430	\\ jmp %[mainIdle:P]
808	:	1431	:
809	: [AsyncClosure_call] "X" (&AsyncClosure.call),	1432	: [mainIdle] "X" (&mainIdle),
810	),	1433	),
811	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),	1434	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
812	}	1435	}
813	}	1436	}
814		1437
815	const AsyncClosure = struct {	1438	const AsyncClosure = struct {
816	event_loop: *EventLoop,	1439	ev: *Evented,
817	fiber: *Fiber,	1440	fiber: *Fiber,
818	start: const fn (context: const anyopaque, result: *anyopaque) void,	1441	start: const fn (context: const anyopaque, result: *anyopaque) void,
819	result_align: Alignment,	1442	result_align: Alignment,
820	already_awaited: bool,	1443
		1444	fn fromFiber(fiber: Fiber) AsyncClosure {
		1445	return @ptrFromInt(Fiber.max_context_align.max(.of(AsyncClosure)).backward(
		1446	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
		1447	) - @sizeOf(AsyncClosure));
		1448	}
821		1449
822	fn contextPointer(closure: AsyncClosure) []align(Fiber.max_context_align.toByteUnits()) u8 {	1450	fn contextPointer(closure: AsyncClosure) []align(Fiber.max_context_align.toByteUnits()) u8 {
823	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(AsyncClosure));	1451	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(AsyncClosure));
824	}	1452	}
825		1453
826	fn call(closure: AsyncClosure, message: const SwitchMessage) callconv(.withStackAlign(.c, @alignOf(AsyncClosure))) noreturn {	1454	fn entry() callconv(.naked) void {
827	message.handle(closure.event_loop);	1455	switch (builtin.cpu.arch) {
		1456	.aarch64 => asm volatile (
		1457	\\ mov x0, sp
		1458	\\ b %[call]
		1459	:
		1460	: [call] "X" (&call),
		1461	),
		1462	.x86_64 => asm volatile (
		1463	\\ leaq 8(%%rsp), %%rdi
		1464	\\ jmp %[call:P]
		1465	:
		1466	: [call] "X" (&call),
		1467	),
		1468	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
		1469	}
		1470	}
		1471
		1472	fn call(
		1473	closure: *AsyncClosure,
		1474	message: *const SwitchMessage,
		1475	) callconv(.withStackAlign(.c, @alignOf(AsyncClosure))) noreturn {
		1476	message.handle(closure.ev);
828	const fiber = closure.fiber;	1477	const fiber = closure.fiber;
829	std.log.debug("{*} performing async", .{fiber});	1478	log.debug("{*} performing async", .{fiber});
830	closure.start(closure.contextPointer(), fiber.resultBytes(closure.result_align));	1479	closure.start(closure.contextPointer(), fiber.resultBytes(closure.result_align));
831	const awaiter = @atomicRmw(?*Fiber, &fiber.awaiter, .Xchg, Fiber.finished, .acq_rel);	1480	closure.ev.yield(
832	const ready_awaiter = r: {	1481	if (@atomicRmw(?*Fiber, &fiber.link.awaiter, .Xchg, Fiber.finished, .acq_rel)) \|awaiter\|
833	const a = awaiter orelse break :r null;	1482	if (@atomicRmw(i32, &awaiter.await_count, .Add, 1, .monotonic) == -1) awaiter else null
834	if (@atomicRmw(bool, &closure.already_awaited, .Xchg, true, .acq_rel)) break :r null;	1483	else
835	break :r a;	1484	null,
836	};	1485	.nothing,
837	closure.event_loop.yield(ready_awaiter, .nothing);	1486	);
838	unreachable; // switched to dead fiber	1487	unreachable; // switched to dead fiber
839	}	1488	}
840
841	fn fromFiber(fiber: Fiber) AsyncClosure {
842	return @ptrFromInt(Fiber.max_context_align.max(.of(AsyncClosure)).backward(
843	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
844	) - @sizeOf(AsyncClosure));
845	}
846	};	1489	};
847		1490
848	fn async(	1491	fn async(
...	@@ -853,7 +1496,8 @@ fn async(	...	@@ -853,7 +1496,8 @@ fn async(
853	context_alignment: Alignment,	1496	context_alignment: Alignment,
854	start: const fn (context: const anyopaque, result: *anyopaque) void,	1497	start: const fn (context: const anyopaque, result: *anyopaque) void,
855	) ?*std.Io.AnyFuture {	1498	) ?*std.Io.AnyFuture {
856	return concurrent(userdata, result.len, result_alignment, context, context_alignment, start) catch {	1499	const ev: *Evented = @ptrCast(@alignCast(userdata));
		1500	return concurrent(ev, result.len, result_alignment, context, context_alignment, start) catch {
857	start(context.ptr, result.ptr);	1501	start(context.ptr, result.ptr);
858	return null;	1502	return null;
859	};	1503	};
...	@@ -872,626 +1516,4588 @@ fn concurrent(	...	@@ -872,626 +1516,4588 @@ fn concurrent(
872	assert(result_len <= Fiber.max_result_size); // TODO	1516	assert(result_len <= Fiber.max_result_size); // TODO
873	assert(context.len <= Fiber.max_context_size); // TODO	1517	assert(context.len <= Fiber.max_context_size); // TODO
874		1518
875	const event_loop: *EventLoop = @ptrCast(@alignCast(userdata));	1519	const ev: *Evented = @ptrCast(@alignCast(userdata));
876	const fiber = try Fiber.allocate(event_loop);	1520	const fiber = Fiber.create(ev) catch \|err\| switch (err) {
877	std.log.debug("allocated {*}", .{fiber});	1521	error.OutOfMemory => return error.ConcurrencyUnavailable,
		1522	};
		1523	log.debug("allocated {*}", .{fiber});
878		1524
879	const closure: *AsyncClosure = .fromFiber(fiber);	1525	const closure: *AsyncClosure = .fromFiber(fiber);
880	fiber.* = .{	1526	fiber.* = .{
881	.required_align = {},	1527	.required_align = {},
882	.context = switch (builtin.cpu.arch) {	1528	.context = switch (builtin.cpu.arch) {
883	.x86_64 => .{
884	.rsp = @intFromPtr(closure) - @sizeOf(usize),
885	.rbp = 0,
886	.rip = @intFromPtr(&fiberEntry),
887	},
888	.aarch64 => .{	1529	.aarch64 => .{
889	.sp = @intFromPtr(closure),	1530	.sp = @intFromPtr(closure),
890	.fp = 0,	1531	.fp = 0,
891	.pc = @intFromPtr(&fiberEntry),	1532	.pc = @intFromPtr(&AsyncClosure.entry),
		1533	},
		1534	.x86_64 => .{
		1535	.rsp = @intFromPtr(closure) - @sizeOf(usize),
		1536	.rbp = 0,
		1537	.rip = @intFromPtr(&AsyncClosure.entry),
892	},	1538	},
893	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),	1539	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
894	},	1540	},
895	.awaiter = null,	1541	.await_count = 0,
896	.queue_next = null,	1542	.link = .{ .awaiter = null },
897	.cancel_thread = null,	1543	.status = .{ .queue_next = null },
898	.awaiting_completions = .initEmpty(),	1544	.cancel_status = .unrequested,
		1545	.cancel_protection = .unblocked,
899	};	1546	};
900	closure.* = .{	1547	closure.* = .{
901	.event_loop = event_loop,	1548	.ev = ev,
902	.fiber = fiber,	1549	.fiber = fiber,
903	.start = start,	1550	.start = start,
904	.result_align = result_alignment,	1551	.result_align = result_alignment,
905	.already_awaited = false,
906	};	1552	};
907	@memcpy(closure.contextPointer(), context);	1553	@memcpy(closure.contextPointer(), context);
908		1554
909	event_loop.schedule(.current(), .{ .head = fiber, .tail = fiber });	1555	const thread: *Thread = .current();
		1556	if (ev.schedule(thread, .{ .head = fiber, .tail = fiber })) thread.submit();
910	return @ptrCast(fiber);	1557	return @ptrCast(fiber);
911	}	1558	}
912		1559
913	fn await(	1560	fn await(
914	userdata: ?*anyopaque,	1561	userdata: ?*anyopaque,
915	any_future: *std.Io.AnyFuture,	1562	future: *std.Io.AnyFuture,
916	result: []u8,	1563	result: []u8,
917	result_alignment: Alignment,	1564	result_alignment: Alignment,
918	) void {	1565	) void {
919	const event_loop: *EventLoop = @ptrCast(@alignCast(userdata));	1566	const ev: *Evented = @ptrCast(@alignCast(userdata));
920	const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));	1567	const fiber = Thread.current().currentFiber();
921	if (@atomicLoad(?*Fiber, &future_fiber.awaiter, .acquire) != Fiber.finished)	1568	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
922	event_loop.yield(null, .{ .register_awaiter = &future_fiber.awaiter });	1569	if (@atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, fiber, .acq_rel)) \|awaiter\| {
		1570	assert(awaiter == Fiber.finished);
		1571	} else while (true) {
		1572	ev.yield(null, .{ .await = 1 });
		1573	const awaiter = @atomicLoad(?*Fiber, &future_fiber.link.awaiter, .acquire);
		1574	if (awaiter == Fiber.finished) break;
		1575	assert(awaiter == fiber); // spurious wakeup
		1576	}
923	@memcpy(result, future_fiber.resultBytes(result_alignment));	1577	@memcpy(result, future_fiber.resultBytes(result_alignment));
924	event_loop.recycle(future_fiber);	1578	future_fiber.destroy(ev.allocator());
925	}	1579	}
926		1580
927	fn select(userdata: ?anyopaque, futures: []const Io.AnyFuture) usize {	1581	fn cancel(
928	const el: *EventLoop = @ptrCast(@alignCast(userdata));	1582	userdata: ?*anyopaque,
		1583	future: *std.Io.AnyFuture,
		1584	result: []u8,
		1585	result_alignment: Alignment,
		1586	) void {
		1587	const ev: *Evented = @ptrCast(@alignCast(userdata));
		1588	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
		1589	future_fiber.requestCancel(ev);
		1590	await(ev, future, result, result_alignment);
		1591	}
		1592
		1593	const Group = struct {
		1594	ptr: *Io.Group,
		1595
		1596	const List = packed struct(usize) {
		1597	cancel_requested: bool,
		1598	awaiter_delayed: bool,
		1599	fibers: Fiber.PackedPtr,
		1600	};
		1601	fn listPtr(group: Group) *List {
		1602	return @ptrCast(&group.ptr.token);
		1603	}
		1604
		1605	const Mutex = packed struct(u32) {
		1606	locked: bool,
		1607	contended: bool,
		1608	shared2: u30,
		1609	};
		1610	fn mutexPtr(group: Group) *Mutex {
		1611	return switch (comptime builtin.cpu.arch.endian()) {
		1612	.little => @ptrCast(&group.ptr.state),
		1613	.big => @ptrCast(@alignCast(
		1614	@as([*]u8, @ptrCast(&group.ptr.state)) + @sizeOf(usize) - @sizeOf(u32),
		1615	)),
		1616	};
		1617	}
		1618
		1619	const Awaiter = packed struct(usize) {
		1620	locked: bool,
		1621	contended: bool,
		1622	awaiter: Fiber.PackedPtr,
		1623	};
		1624	fn awaiterPtr(group: Group) *Awaiter {
		1625	return @ptrCast(&group.ptr.state);
		1626	}
		1627
		1628	fn lock(group: Group, ev: *Evented) void {
		1629	const mutex = group.mutexPtr();
		1630	{
		1631	const old_state = @atomicRmw(
		1632	Mutex,
		1633	mutex,
		1634	.Or,
		1635	.{ .locked = true, .contended = false, .shared2 = 0 },
		1636	.acquire,
		1637	);
		1638	if (!old_state.locked) {
		1639	@branchHint(.likely);
		1640	return;
		1641	}
		1642	if (old_state.contended) {
		1643	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
		1644	}
		1645	}
		1646	while (true) {
		1647	var old_state = @atomicRmw(
		1648	Mutex,
		1649	mutex,
		1650	.Or,
		1651	.{ .locked = true, .contended = true, .shared2 = 0 },
		1652	.acquire,
		1653	);
		1654	if (!old_state.locked) {
		1655	@branchHint(.likely);
		1656	return;
		1657	}
		1658	old_state.contended = true;
		1659	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
		1660	}
		1661	}
		1662
		1663	fn unlock(group: Group, ev: *Evented) void {
		1664	const mutex = group.mutexPtr();
		1665	const old_state = @atomicRmw(
		1666	Mutex,
		1667	mutex,
		1668	.And,
		1669	.{ .locked = false, .contended = false, .shared2 = std.math.maxInt(u30) },
		1670	.release,
		1671	);
		1672	assert(old_state.locked);
		1673	if (old_state.contended) futexWake(ev, @ptrCast(mutex), 1);
		1674	}
929		1675
930	// Optimization to avoid the yield below.	1676	fn addFiber(group: Group, ev: Evented, fiber: Fiber) void {
931	for (futures, 0..) \|any_future, i\| {	1677	group.lock(ev);
932	const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));	1678	defer group.unlock(ev);
933	if (@atomicLoad(?*Fiber, &future_fiber.awaiter, .acquire) == Fiber.finished)	1679	const list_ptr = group.listPtr();
934	return i;	1680	const list = @atomicLoad(List, list_ptr, .monotonic);
		1681	if (list.cancel_requested) fiber.cancel_status = .{ .requested = true, .awaiting = .nothing };
		1682	const old_head = list.fibers.unpack();
		1683	if (old_head) \|head\| head.link.group.prev = fiber;
		1684	fiber.link.group.next = old_head;
		1685	@atomicStore(List, list_ptr, .{
		1686	.cancel_requested = list.cancel_requested,
		1687	.awaiter_delayed = list.awaiter_delayed,
		1688	.fibers = .pack(fiber),
		1689	}, .monotonic);
935	}	1690	}
936		1691
937	el.yield(null, .{ .register_select = futures });	1692	fn removeFiber(group: Group, ev: Evented, fiber: Fiber) ?*Fiber {
		1693	group.lock(ev);
		1694	defer group.unlock(ev);
		1695	const list_ptr = group.listPtr();
		1696	const list = @atomicLoad(List, list_ptr, .monotonic);
		1697	if (fiber.link.group.next) \|next\| next.link.group.prev = fiber.link.group.prev;
		1698	if (fiber.link.group.prev) \|prev\| {
		1699	prev.link.group.next = fiber.link.group.next;
		1700	} else if (fiber.link.group.next) \|new_head\| {
		1701	@atomicStore(List, list_ptr, .{
		1702	.cancel_requested = list.cancel_requested,
		1703	.awaiter_delayed = list.awaiter_delayed,
		1704	.fibers = .pack(new_head),
		1705	}, .monotonic);
		1706	} else if (@atomicLoad(Awaiter, group.awaiterPtr(), .monotonic).awaiter.unpack()) \|awaiter\| {
		1707	if (!awaiter.cancel_status.changeAwaiting(.group, .nothing) or list.cancel_requested) {
		1708	@atomicStore(List, list_ptr, .{
		1709	.cancel_requested = false,
		1710	.awaiter_delayed = false,
		1711	.fibers = .null,
		1712	}, .release);
		1713	assert(awaiter.status.awaiting_group.ptr == group.ptr);
		1714	awaiter.status = .{ .queue_next = null };
		1715	return awaiter;
		1716	}
		1717	// Race with `Fiber.requestCancel`
		1718	@atomicStore(List, list_ptr, .{
		1719	.cancel_requested = false,
		1720	.awaiter_delayed = true,
		1721	.fibers = .null,
		1722	}, .monotonic);
		1723	} else @atomicStore(List, list_ptr, .{
		1724	.cancel_requested = false,
		1725	.awaiter_delayed = false,
		1726	.fibers = .null,
		1727	}, .release);
		1728	return null;
		1729	}
938		1730
939	std.log.debug("back from select yield", .{});	1731	fn await(group: Group, ev: Evented, awaiter: Fiber) bool {
		1732	group.lock(ev);
		1733	defer group.unlock(ev);
		1734	if (@atomicLoad(List, group.listPtr(), .monotonic).fibers.unpack()) \|_\| {
		1735	if (group.registerAwaiter(awaiter) and awaiter.cancel_protection.check() == .unblocked) {
		1736	// The awaiter already had an unacknowledged cancelation request before
		1737	// attempting to await a group, so propagate the cancelation to the group.
		1738	assert(!group.cancelLocked(ev, null));
		1739	}
		1740	return false;
		1741	}
		1742	return true;
		1743	}
940		1744
941	const my_thread: *Thread = .current();	1745	fn cancel(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
942	const my_fiber = my_thread.currentFiber();	1746	group.lock(ev);
943	var result: ?usize = null;	1747	defer group.unlock(ev);
		1748	return group.cancelLocked(ev, maybe_awaiter);
		1749	}
944		1750
945	for (futures, 0..) \|any_future, i\| {	1751	/// Assumes the mutex is held.
946	const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));	1752	fn cancelLocked(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
947	if (@cmpxchgStrong(?*Fiber, &future_fiber.awaiter, my_fiber, null, .seq_cst, .seq_cst)) \|awaiter\| {	1753	const list_ptr = group.listPtr();
948	if (awaiter == Fiber.finished) {	1754	const list = @atomicRmw(
949	if (result == null) result = i;	1755	List,
950	} else if (awaiter) \|a\| {	1756	list_ptr,
951	const closure: *AsyncClosure = .fromFiber(a);	1757	.Add,
952	closure.already_awaited = false;	1758	.{ .cancel_requested = true, .awaiter_delayed = false, .fibers = .null },
		1759	.monotonic,
		1760	);
		1761	assert(!list.cancel_requested);
		1762	if (list.fibers.unpack()) \|head\| {
		1763	var maybe_fiber: ?*Fiber = head;
		1764	while (maybe_fiber) \|fiber\| {
		1765	fiber.requestCancel(ev);
		1766	maybe_fiber = fiber.link.group.next;
953	}	1767	}
954	} else {	1768	if (maybe_awaiter) \|awaiter\| _ = group.registerAwaiter(awaiter);
955	const closure: *AsyncClosure = .fromFiber(my_fiber);	1769	return false;
956	closure.already_awaited = false;
957	}	1770	}
		1771	@atomicStore(
		1772	List,
		1773	list_ptr,
		1774	.{ .cancel_requested = false, .awaiter_delayed = false, .fibers = .null },
		1775	.release,
		1776	);
		1777	return if (maybe_awaiter) \|_\| true else list.awaiter_delayed;
958	}	1778	}
959		1779
960	return result.?;	1780	/// Assumes the mutex is held.
961	}	1781	fn registerAwaiter(group: Group, awaiter: *Fiber) bool {
		1782	assert(awaiter.status.queue_next == null);
		1783	awaiter.status = .{ .awaiting_group = group };
		1784	assert(@atomicRmw(
		1785	Awaiter,
		1786	group.awaiterPtr(),
		1787	.Add,
		1788	.{ .locked = false, .contended = false, .awaiter = .pack(awaiter) },
		1789	.monotonic,
		1790	).awaiter == .null);
		1791	return awaiter.cancel_status.changeAwaiting(.nothing, .group);
		1792	}
962		1793
963	fn cancel(	1794	const AsyncClosure = struct {
		1795	ev: *Evented,
		1796	group: Group,
		1797	fiber: *Fiber,
		1798	start: const fn (context: const anyopaque) Io.Cancelable!void,
		1799
		1800	fn fromFiber(fiber: Fiber) Group.AsyncClosure {
		1801	return @ptrFromInt(Fiber.max_context_align.max(.of(Group.AsyncClosure)).backward(
		1802	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
		1803	) - @sizeOf(Group.AsyncClosure));
		1804	}
		1805
		1806	fn contextPointer(
		1807	closure: *Group.AsyncClosure,
		1808	) [*]align(Fiber.max_context_align.toByteUnits()) u8 {
		1809	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(Group.AsyncClosure));
		1810	}
		1811
		1812	fn entry() callconv(.naked) void {
		1813	switch (builtin.cpu.arch) {
		1814	.aarch64 => asm volatile (
		1815	\\ mov x0, sp
		1816	\\ b %[call]
		1817	:
		1818	: [call] "X" (&call),
		1819	),
		1820	.x86_64 => asm volatile (
		1821	\\ leaq 8(%%rsp), %%rdi
		1822	\\ jmp %[call:P]
		1823	:
		1824	: [call] "X" (&call),
		1825	),
		1826	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
		1827	}
		1828	}
		1829
		1830	fn call(
		1831	closure: *Group.AsyncClosure,
		1832	message: *const SwitchMessage,
		1833	) callconv(.withStackAlign(.c, @alignOf(Group.AsyncClosure))) noreturn {
		1834	message.handle(closure.ev);
		1835	assert(closure.fiber.status.queue_next == null);
		1836	log.debug("{*} performing group async", .{closure.fiber});
		1837	const result = closure.start(closure.contextPointer());
		1838	const ev = closure.ev;
		1839	const group = closure.group;
		1840	const fiber = closure.fiber;
		1841	const cancel_acknowledged = fiber.cancel_protection.acknowledged;
		1842	if (result) {
		1843	assert(!cancel_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
		1844	} else \|err\| switch (err) {
		1845	error.Canceled => assert(cancel_acknowledged), // group task returned `error.Canceled` but was never canceled
		1846	}
		1847	const awaiter = group.removeFiber(ev, fiber);
		1848	ev.backing_allocator_mutex.lockUncancelable(ev.io());
		1849	ev.yield(awaiter, .destroy);
		1850	unreachable; // switched to dead fiber
		1851	}
		1852	};
		1853	};
		1854
		1855	fn groupAsync(
964	userdata: ?*anyopaque,	1856	userdata: ?*anyopaque,
965	any_future: *std.Io.AnyFuture,	1857	type_erased: *Io.Group,
966	result: []u8,	1858	context: []const u8,
967	result_alignment: Alignment,	1859	context_alignment: Alignment,
		1860	start: const fn (context: const anyopaque) Io.Cancelable!void,
968	) void {	1861	) void {
969	const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));	1862	const ev: *Evented = @ptrCast(@alignCast(userdata));
970	if (@atomicRmw(	1863	return groupConcurrent(ev, type_erased, context, context_alignment, start) catch {
971	?*Thread,	1864	const fiber = Thread.current().currentFiber();
972	&future_fiber.cancel_thread,	1865	const pre_acknowledged = fiber.cancel_protection.acknowledged;
973	.Xchg,	1866	const result = start(context.ptr);
974	Thread.canceling,	1867	const post_acknowledged = fiber.cancel_protection.acknowledged;
975	.acq_rel,	1868	if (result) {
976	)) \|cancel_thread\| if (cancel_thread != Thread.canceling) {	1869	if (pre_acknowledged) {
977	getSqe(&Thread.current().io_uring).* = .{	1870	assert(post_acknowledged); // group task called `recancel` but was not canceled
978	.opcode = .MSG_RING,	1871	} else {
979	.flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS,	1872	assert(!post_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
980	.ioprio = 0,	1873	}
981	.fd = cancel_thread.io_uring.fd,	1874	} else \|err\| switch (err) {
982	.off = @intFromPtr(future_fiber),	1875	// Don't swallow the cancelation: make it visible to the `Group.async` caller.
983	.addr = 0,	1876	error.Canceled => {
984	.len = @bitCast(-@as(i32, @intFromEnum(std.os.linux.E.INTR))),	1877	assert(!pre_acknowledged); // group task called `recancel` but was not canceled
985	.rw_flags = 0,	1878	assert(post_acknowledged); // group task returned `error.Canceled` but was never canceled
986	.user_data = @intFromEnum(Completion.UserData.cleanup),	1879	recancel(userdata);
987	.buf_index = 0,	1880	},
988	.personality = 0,	1881	}
989	.splice_fd_in = 0,
990	.addr3 = 0,
991	.resv = 0,
992	};
993	};	1882	};
994	await(userdata, any_future, result, result_alignment);
995	}
996
997	fn cancelRequested(userdata: ?*anyopaque) bool {
998	_ = userdata;
999	return @atomicLoad(?*Thread, &Thread.current().currentFiber().cancel_thread, .acquire) == Thread.canceling;
1000	}	1883	}
1001		1884
1002	fn createFile(	1885	fn groupConcurrent(
1003	userdata: ?*anyopaque,	1886	userdata: ?*anyopaque,
1004	dir: Io.Dir,	1887	type_erased: *Io.Group,
1005	sub_path: []const u8,	1888	context: []const u8,
1006	flags: Io.File.CreateFlags,	1889	context_alignment: Alignment,
1007	) Io.File.OpenError!Io.File {	1890	start: const fn (context: const anyopaque) Io.Cancelable!void,
1008	const el: *EventLoop = @ptrCast(@alignCast(userdata));	1891	) Io.ConcurrentError!void {
1009	const thread: *Thread = .current();	1892	assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
1010	const iou = &thread.io_uring;	1893	assert(context.len <= Fiber.max_context_size); // TODO
1011	const fiber = thread.currentFiber();
1012	try fiber.enterCancelRegion(thread);
1013
1014	const posix = std.posix;
1015	const sub_path_c = try posix.toPosixPath(sub_path);
1016		1894
1017	var os_flags: posix.O = .{	1895	const ev: *Evented = @ptrCast(@alignCast(userdata));
1018	.ACCMODE = if (flags.read) .RDWR else .WRONLY,	1896	const group: Group = .{ .ptr = type_erased };
1019	.CREAT = true,	1897	const fiber = Fiber.create(ev) catch \|err\| switch (err) {
1020	.TRUNC = flags.truncate,	1898	error.OutOfMemory => return error.ConcurrencyUnavailable,
1021	.EXCL = flags.exclusive,
1022	};	1899	};
1023	if (@hasField(posix.O, "LARGEFILE")) os_flags.LARGEFILE = true;	1900	log.debug("allocated {*}", .{fiber});
1024	if (@hasField(posix.O, "CLOEXEC")) os_flags.CLOEXEC = true;
1025		1901
1026	// Use the O locking flags if the os supports them to acquire the lock	1902	const closure: *Group.AsyncClosure = .fromFiber(fiber);
1027	// atomically. Note that the NONBLOCK flag is removed after the openat()	1903	fiber.* = .{
1028	// call is successful.	1904	.required_align = {},
1029	const has_flock_open_flags = @hasField(posix.O, "EXLOCK");	1905	.context = switch (builtin.cpu.arch) {
1030	if (has_flock_open_flags) switch (flags.lock) {	1906	.aarch64 => .{
1031	.none => {},	1907	.sp = @intFromPtr(closure),
1032	.shared => {	1908	.fp = 0,
1033	os_flags.SHLOCK = true;	1909	.pc = @intFromPtr(&Group.AsyncClosure.entry),
1034	os_flags.NONBLOCK = flags.lock_nonblocking;	1910	},
1035	},	1911	.x86_64 => .{
1036	.exclusive => {	1912	.rsp = @intFromPtr(closure) - @sizeOf(usize),
1037	os_flags.EXLOCK = true;	1913	.rbp = 0,
1038	os_flags.NONBLOCK = flags.lock_nonblocking;	1914	.rip = @intFromPtr(&Group.AsyncClosure.entry),
		1915	},
		1916	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1039	},	1917	},
		1918	.await_count = 0,
		1919	.link = .{ .group = .{ .prev = null, .next = null } },
		1920	.status = .{ .queue_next = null },
		1921	.cancel_status = .unrequested,
		1922	.cancel_protection = .unblocked,
1040	};	1923	};
1041	const have_flock = @TypeOf(posix.system.flock) != void;	1924	closure.* = .{
1042		1925	.ev = ev,
1043	if (have_flock and !has_flock_open_flags and flags.lock != .none) {	1926	.group = group,
1044	@panic("TODO");	1927	.fiber = fiber,
1045	}	1928	.start = start,
1046
1047	if (has_flock_open_flags and flags.lock_nonblocking) {
1048	@panic("TODO");
1049	}
1050
1051	getSqe(iou).* = .{
1052	.opcode = .OPENAT,
1053	.flags = 0,
1054	.ioprio = 0,
1055	.fd = dir.handle,
1056	.off = 0,
1057	.addr = @intFromPtr(&sub_path_c),
1058	.len = @intCast(flags.mode),
1059	.rw_flags = @bitCast(os_flags),
1060	.user_data = @intFromPtr(fiber),
1061	.buf_index = 0,
1062	.personality = 0,
1063	.splice_fd_in = 0,
1064	.addr3 = 0,
1065	.resv = 0,
1066	};	1929	};
		1930	@memcpy(closure.contextPointer(), context);
		1931	group.addFiber(ev, fiber);
		1932	const thread: *Thread = .current();
		1933	if (ev.schedule(thread, .{ .head = fiber, .tail = fiber })) thread.submit();
		1934	}
1067		1935
1068	el.yield(null, .nothing);	1936	fn groupAwait(
1069	fiber.exitCancelRegion(thread);	1937	userdata: ?*anyopaque,
1070		1938	type_erased: *Io.Group,
1071	const completion = fiber.resultPointer(Completion);	1939	initial_token: *anyopaque,
1072	switch (errno(completion.result)) {	1940	) Io.Cancelable!void {
1073	.SUCCESS => return .{ .handle = completion.result },	1941	const ev: *Evented = @ptrCast(@alignCast(userdata));
1074	.INTR => unreachable,	1942	_ = initial_token;
1075	.CANCELED => return error.Canceled,	1943	ev.yield(null, .{ .group_await = .{ .ptr = type_erased } });
		1944	}
1076		1945
1077	.FAULT => unreachable,	1946	fn groupCancel(userdata: ?anyopaque, type_erased: Io.Group, initial_token: *anyopaque) void {
1078	.INVAL => return error.BadPathName,	1947	const ev: *Evented = @ptrCast(@alignCast(userdata));
1079	.BADF => unreachable,	1948	_ = initial_token;
1080	.ACCES => return error.AccessDenied,	1949	ev.yield(null, .{ .group_cancel = .{ .ptr = type_erased } });
1081	.FBIG => return error.FileTooBig,
1082	.OVERFLOW => return error.FileTooBig,
1083	.ISDIR => return error.IsDir,
1084	.LOOP => return error.SymLinkLoop,
1085	.MFILE => return error.ProcessFdQuotaExceeded,
1086	.NAMETOOLONG => return error.NameTooLong,
1087	.NFILE => return error.SystemFdQuotaExceeded,
1088	.NODEV => return error.NoDevice,
1089	.NOENT => return error.FileNotFound,
1090	.NOMEM => return error.SystemResources,
1091	.NOSPC => return error.NoSpaceLeft,
1092	.NOTDIR => return error.NotDir,
1093	.PERM => return error.PermissionDenied,
1094	.EXIST => return error.PathAlreadyExists,
1095	.BUSY => return error.DeviceBusy,
1096	.OPNOTSUPP => return error.FileLocksUnsupported,
1097	.AGAIN => return error.WouldBlock,
1098	.TXTBSY => return error.FileBusy,
1099	.NXIO => return error.NoDevice,
1100	else => \|err\| return posix.unexpectedErrno(err),
1101	}
1102	}	1950	}
1103		1951
1104	fn fileOpen(	1952	fn recancel(userdata: ?*anyopaque) void {
1105	userdata: ?*anyopaque,	1953	const ev: *Evented = @ptrCast(@alignCast(userdata));
1106	dir: Io.Dir,	1954	_ = ev;
1107	sub_path: []const u8,	1955	const cancel_protection = &Thread.current().currentFiber().cancel_protection;
1108	flags: Io.File.OpenFlags,	1956	assert(cancel_protection.acknowledged);
1109	) Io.File.OpenError!Io.File {	1957	cancel_protection.acknowledged = false;
1110	const el: *EventLoop = @ptrCast(@alignCast(userdata));	1958	}
1111	const thread: *Thread = .current();
1112	const iou = &thread.io_uring;
1113	const fiber = thread.currentFiber();
1114	try fiber.enterCancelRegion(thread);
1115		1959
1116	const posix = std.posix;	1960	fn swapCancelProtection(userdata: ?*anyopaque, new: Io.CancelProtection) Io.CancelProtection {
1117	const sub_path_c = try posix.toPosixPath(sub_path);	1961	const ev: *Evented = @ptrCast(@alignCast(userdata));
		1962	_ = ev;
		1963	const cancel_protection = &Thread.current().currentFiber().cancel_protection;
		1964	defer cancel_protection.user = new;
		1965	return cancel_protection.user;
		1966	}
1118		1967
1119	var os_flags: posix.O = .{	1968	fn checkCancel(userdata: ?*anyopaque) Io.Cancelable!void {
1120	.ACCMODE = switch (flags.mode) {	1969	const ev: *Evented = @ptrCast(@alignCast(userdata));
1121	.read_only => .RDONLY,	1970	_ = ev;
1122	.write_only => .WRONLY,	1971	const fiber = Thread.current().currentFiber();
1123	.read_write => .RDWR,	1972	switch (fiber.cancel_protection.check()) {
		1973	.blocked => {},
		1974	.unblocked => if (@atomicLoad(Fiber.CancelStatus, &fiber.cancel_status, .monotonic).requested) {
		1975	fiber.cancel_protection.acknowledge();
		1976	return error.Canceled;
1124	},	1977	},
1125	};
1126
1127	if (@hasField(posix.O, "CLOEXEC")) os_flags.CLOEXEC = true;
1128	if (@hasField(posix.O, "LARGEFILE")) os_flags.LARGEFILE = true;
1129	if (@hasField(posix.O, "NOCTTY")) os_flags.NOCTTY = !flags.allow_ctty;
1130
1131	// Use the O locking flags if the os supports them to acquire the lock
1132	// atomically.
1133	const has_flock_open_flags = @hasField(posix.O, "EXLOCK");
1134	if (has_flock_open_flags) {
1135	// Note that the NONBLOCK flag is removed after the openat() call
1136	// is successful.
1137	switch (flags.lock) {
1138	.none => {},
1139	.shared => {
1140	os_flags.SHLOCK = true;
1141	os_flags.NONBLOCK = flags.lock_nonblocking;
1142	},
1143	.exclusive => {
1144	os_flags.EXLOCK = true;
1145	os_flags.NONBLOCK = flags.lock_nonblocking;
1146	},
1147	}
1148	}	1978	}
1149	const have_flock = @TypeOf(posix.system.flock) != void;	1979	}
1150		1980
1151	if (have_flock and !has_flock_open_flags and flags.lock != .none) {	1981	fn select(userdata: ?anyopaque, futures: []const Io.AnyFuture) Io.Cancelable!usize {
1152	@panic("TODO");	1982	const ev: *Evented = @ptrCast(@alignCast(userdata));
		1983	var cancel_region: CancelRegion = .init();
		1984	defer cancel_region.deinit();
		1985	var await_count: u31, var result = for (futures, 0..) \|future, future_index\| {
		1986	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
		1987	if (@atomicRmw(
		1988	?*Fiber,
		1989	&future_fiber.link.awaiter,
		1990	.Xchg,
		1991	cancel_region.fiber,
		1992	.acq_rel,
		1993	)) \|awaiter\| {
		1994	assert(awaiter == Fiber.finished);
		1995	break .{ @intCast(future_index), future_index };
		1996	}
		1997	} else result: {
		1998	const await_count: u31 = @intCast(futures.len);
		1999	cancel_region.await(.select) catch \|err\| switch (err) {
		2000	error.Canceled => \|e\| break :result .{ await_count + 1, e },
		2001	};
		2002	ev.yield(null, .{ .await = 1 });
		2003	cancel_region.await(.nothing) catch \|err\| switch (err) {
		2004	error.Canceled => \|e\| break :result .{ await_count, e },
		2005	};
		2006	break :result .{ await_count - 1, futures.len };
		2007	};
		2008	for (futures[0 .. result catch futures.len], 0..) \|future, future_index\| {
		2009	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
		2010	const awaiter = @atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, null, .monotonic);
		2011	if (awaiter == Fiber.finished) {
		2012	@atomicStore(?*Fiber, &future_fiber.link.awaiter, Fiber.finished, .monotonic);
		2013	result = if (result) \|finished_index\| @min(future_index, finished_index) else \|e\| e;
		2014	} else {
		2015	assert(awaiter == cancel_region.fiber);
		2016	await_count -= 1;
		2017	}
1153	}	2018	}
1154		2019	// Equivalent to `ev.yield(null, .{ .await = await_count });`,
1155	if (has_flock_open_flags and flags.lock_nonblocking) {	2020	// but avoiding a context switch in the common case.
1156	@panic("TODO");	2021	switch (std.math.order(
		2022	@atomicRmw(i32, &cancel_region.fiber.await_count, .Sub, await_count, .monotonic),
		2023	await_count,
		2024	)) {
		2025	.lt => ev.yield(null, .{ .await = 0 }),
		2026	.eq => {},
		2027	.gt => unreachable,
1157	}	2028	}
		2029	return result;
		2030	}
1158		2031
1159	getSqe(iou).* = .{	2032	fn futexWait(
1160	.opcode = .OPENAT,	2033	userdata: ?*anyopaque,
1161	.flags = 0,	2034	ptr: *const u32,
		2035	expected: u32,
		2036	timeout: Io.Timeout,
		2037	) Io.Cancelable!void {
		2038	const ev: *Evented = @ptrCast(@alignCast(userdata));
		2039	if (builtin.single_threaded) unreachable; // Deadlock.
		2040	const timespec: ?linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = timespec: switch (timeout) {
		2041	.none => .{
		2042	null,
		2043	.awake,
		2044	linux.IORING_TIMEOUT_ABS,
		2045	},
		2046	.duration => \|duration\| {
		2047	const ns = duration.raw.toNanoseconds();
		2048	break :timespec .{
		2049	.{
		2050	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
		2051	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
		2052	},
		2053	duration.clock,
		2054	0,
		2055	};
		2056	},
		2057	.deadline => \|deadline\| {
		2058	const ns = deadline.raw.toNanoseconds();
		2059	break :timespec .{
		2060	.{
		2061	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
		2062	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
		2063	},
		2064	deadline.clock,
		2065	linux.IORING_TIMEOUT_ABS,
		2066	};
		2067	},
		2068	};
		2069	var cancel_region: CancelRegion = .init();
		2070	defer cancel_region.deinit();
		2071	const thread = try cancel_region.awaitIoUring();
		2072	thread.enqueue().* = .{
		2073	.opcode = .FUTEX_WAIT,
		2074	.flags = if (timespec) \|_\| linux.IOSQE_IO_LINK else 0,
1162	.ioprio = 0,	2075	.ioprio = 0,
1163	.fd = dir.handle,	2076	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
1164	.off = 0,	2077	.off = expected,
1165	.addr = @intFromPtr(&sub_path_c),	2078	.addr = @intFromPtr(ptr),
1166	.len = 0,	2079	.len = 0,
1167	.rw_flags = @bitCast(os_flags),	2080	.rw_flags = 0,
1168	.user_data = @intFromPtr(fiber),	2081	.user_data = @intFromPtr(cancel_region.fiber),
1169	.buf_index = 0,	2082	.buf_index = 0,
1170	.personality = 0,	2083	.personality = 0,
1171	.splice_fd_in = 0,	2084	.splice_fd_in = 0,
1172	.addr3 = 0,	2085	.addr3 = std.math.maxInt(u32),
1173	.resv = 0,	2086	.resv = 0,
1174	};	2087	};
1175		2088	if (timespec) \|timespec_ptr\| thread.enqueue(). = .{
1176	el.yield(null, .nothing);	2089	.opcode = .LINK_TIMEOUT,
1177	fiber.exitCancelRegion(thread);	2090	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1178
1179	const completion = fiber.resultPointer(Completion);
1180	switch (errno(completion.result)) {
1181	.SUCCESS => return .{ .handle = completion.result },
1182	.INTR => unreachable,
1183	.CANCELED => return error.Canceled,
1184
1185	.FAULT => unreachable,
1186	.INVAL => return error.BadPathName,
1187	.BADF => unreachable,
1188	.ACCES => return error.AccessDenied,
1189	.FBIG => return error.FileTooBig,
1190	.OVERFLOW => return error.FileTooBig,
1191	.ISDIR => return error.IsDir,
1192	.LOOP => return error.SymLinkLoop,
1193	.MFILE => return error.ProcessFdQuotaExceeded,
1194	.NAMETOOLONG => return error.NameTooLong,
1195	.NFILE => return error.SystemFdQuotaExceeded,
1196	.NODEV => return error.NoDevice,
1197	.NOENT => return error.FileNotFound,
1198	.NOMEM => return error.SystemResources,
1199	.NOSPC => return error.NoSpaceLeft,
1200	.NOTDIR => return error.NotDir,
1201	.PERM => return error.PermissionDenied,
1202	.EXIST => return error.PathAlreadyExists,
1203	.BUSY => return error.DeviceBusy,
1204	.OPNOTSUPP => return error.FileLocksUnsupported,
1205	.AGAIN => return error.WouldBlock,
1206	.TXTBSY => return error.FileBusy,
1207	.NXIO => return error.NoDevice,
1208	else => \|err\| return posix.unexpectedErrno(err),
1209	}
1210	}
1211
1212	fn fileClose(userdata: ?*anyopaque, file: Io.File) void {
1213	const el: *EventLoop = @ptrCast(@alignCast(userdata));
1214	const thread: *Thread = .current();
1215	const iou = &thread.io_uring;
1216	const fiber = thread.currentFiber();
1217
1218	getSqe(iou).* = .{
1219	.opcode = .CLOSE,
1220	.flags = 0,
1221	.ioprio = 0,	2091	.ioprio = 0,
1222	.fd = file.handle,	2092	.fd = 0,
1223	.off = 0,	2093	.off = 0,
1224	.addr = 0,	2094	.addr = @intFromPtr(timespec_ptr),
1225	.len = 0,	2095	.len = 1,
1226	.rw_flags = 0,	2096	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
1227	.user_data = @intFromPtr(fiber),	2097	.real => linux.IORING_TIMEOUT_REALTIME,
		2098	else => 0,
		2099	.boot => linux.IORING_TIMEOUT_BOOTTIME,
		2100	}),
		2101	.user_data = @intFromEnum(Completion.UserData.wakeup),
1228	.buf_index = 0,	2102	.buf_index = 0,
1229	.personality = 0,	2103	.personality = 0,
1230	.splice_fd_in = 0,	2104	.splice_fd_in = 0,
1231	.addr3 = 0,	2105	.addr3 = 0,
1232	.resv = 0,	2106	.resv = 0,
1233	};	2107	};
1234		2108	ev.yield(null, .nothing);
1235	el.yield(null, .nothing);	2109	switch (cancel_region.errno()) {
1236		2110	.SUCCESS => {}, // notified by `wake()`
1237	const completion = fiber.resultPointer(Completion);	2111	.INTR, .CANCELED => {}, // caller's responsibility to retry
1238	switch (errno(completion.result)) {	2112	.AGAIN => {}, // ptr.* != expect
1239	.SUCCESS => return,	2113	.INVAL => {}, // possibly timeout overflow
1240	.INTR => unreachable,	2114	.TIMEDOUT => unreachable,
1241	.CANCELED => return,	2115	.FAULT => recoverableOsBugDetected(), // ptr was invalid
1242		2116	else => recoverableOsBugDetected(),
1243	.BADF => unreachable, // Always a race condition.
1244	else => return,
1245	}	2117	}
1246	}	2118	}
1247		2119
1248	fn pread(userdata: ?*anyopaque, file: Io.File, buffer: []u8, offset: std.posix.off_t) Io.File.PReadError!usize {	2120	fn futexWaitUncancelable(userdata: ?anyopaque, ptr: const u32, expected: u32) void {
1249	const el: *EventLoop = @ptrCast(@alignCast(userdata));	2121	const ev: *Evented = @ptrCast(@alignCast(userdata));
1250	const thread: *Thread = .current();	2122	if (builtin.single_threaded) unreachable; // Deadlock.
1251	const iou = &thread.io_uring;	2123	var cancel_region: CancelRegion = .initBlocked();
1252	const fiber = thread.currentFiber();	2124	defer cancel_region.deinit();
1253	try fiber.enterCancelRegion(thread);	2125	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
1254		2126	error.Canceled => unreachable, // blocked
1255	getSqe(iou).* = .{	2127	};
1256	.opcode = .READ,	2128	thread.enqueue().* = .{
		2129	.opcode = .FUTEX_WAIT,
1257	.flags = 0,	2130	.flags = 0,
1258	.ioprio = 0,	2131	.ioprio = 0,
1259	.fd = file.handle,	2132	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
1260	.off = @bitCast(offset),	2133	.off = expected,
1261	.addr = @intFromPtr(buffer.ptr),	2134	.addr = @intFromPtr(ptr),
1262	.len = @min(buffer.len, 0x7ffff000),	2135	.len = 0,
1263	.rw_flags = 0,	2136	.rw_flags = 0,
1264	.user_data = @intFromPtr(fiber),	2137	.user_data = @intFromPtr(cancel_region.fiber),
1265	.buf_index = 0,	2138	.buf_index = 0,
1266	.personality = 0,	2139	.personality = 0,
1267	.splice_fd_in = 0,	2140	.splice_fd_in = 0,
1268	.addr3 = 0,	2141	.addr3 = std.math.maxInt(u32),
1269	.resv = 0,	2142	.resv = 0,
1270	};	2143	};
1271		2144	ev.yield(null, .nothing);
1272	el.yield(null, .nothing);	2145	switch (cancel_region.errno()) {
1273	fiber.exitCancelRegion(thread);	2146	.SUCCESS => {}, // notified by `wake()`
1274		2147	.INTR, .CANCELED => {}, // caller's responsibility to retry
1275	const completion = fiber.resultPointer(Completion);	2148	.AGAIN => {}, // ptr.* != expect
1276	switch (errno(completion.result)) {	2149	.INVAL => {}, // possibly timeout overflow
1277	.SUCCESS => return @as(u32, @bitCast(completion.result)),	2150	.FAULT => recoverableOsBugDetected(), // ptr was invalid
1278	.INTR => unreachable,	2151	else => recoverableOsBugDetected(),
1279	.CANCELED => return error.Canceled,
1280
1281	.INVAL => unreachable,
1282	.FAULT => unreachable,
1283	.NOENT => return error.ProcessNotFound,
1284	.AGAIN => return error.WouldBlock,
1285	.BADF => return error.NotOpenForReading, // Can be a race condition.
1286	.IO => return error.InputOutput,
1287	.ISDIR => return error.IsDir,
1288	.NOBUFS => return error.SystemResources,
1289	.NOMEM => return error.SystemResources,
1290	.NOTCONN => return error.SocketUnconnected,
1291	.CONNRESET => return error.ConnectionResetByPeer,
1292	.TIMEDOUT => return error.Timeout,
1293	.NXIO => return error.Unseekable,
1294	.SPIPE => return error.Unseekable,
1295	.OVERFLOW => return error.Unseekable,
1296	else => \|err\| return std.posix.unexpectedErrno(err),
1297	}	2152	}
1298	}	2153	}
1299		2154
1300	fn pwrite(userdata: ?*anyopaque, file: Io.File, buffer: []const u8, offset: std.posix.off_t) Io.File.PWriteError!usize {	2155	fn futexWake(userdata: ?anyopaque, ptr: const u32, max_waiters: u32) void {
1301	const el: *EventLoop = @ptrCast(@alignCast(userdata));	2156	const ev: *Evented = @ptrCast(@alignCast(userdata));
		2157	_ = ev;
		2158	if (builtin.single_threaded) unreachable; // Nothing to wake up.
1302	const thread: *Thread = .current();	2159	const thread: *Thread = .current();
1303	const iou = &thread.io_uring;	2160	thread.enqueue().* = .{
1304	const fiber = thread.currentFiber();	2161	.opcode = .FUTEX_WAKE,
1305	try fiber.enterCancelRegion(thread);	2162	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1306
1307	getSqe(iou).* = .{
1308	.opcode = .WRITE,
1309	.flags = 0,
1310	.ioprio = 0,	2163	.ioprio = 0,
1311	.fd = file.handle,	2164	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
1312	.off = @bitCast(offset),	2165	.off = max_waiters,
1313	.addr = @intFromPtr(buffer.ptr),	2166	.addr = @intFromPtr(ptr),
1314	.len = @min(buffer.len, 0x7ffff000),	2167	.len = 0,
1315	.rw_flags = 0,	2168	.rw_flags = 0,
1316	.user_data = @intFromPtr(fiber),	2169	.user_data = @intFromEnum(Completion.UserData.futex_wake),
1317	.buf_index = 0,	2170	.buf_index = 0,
1318	.personality = 0,	2171	.personality = 0,
1319	.splice_fd_in = 0,	2172	.splice_fd_in = 0,
1320	.addr3 = 0,	2173	.addr3 = std.math.maxInt(u32),
1321	.resv = 0,	2174	.resv = 0,
1322	};	2175	};
		2176	thread.submit();
		2177	}
1323		2178
1324	el.yield(null, .nothing);	2179	fn operate(userdata: ?*anyopaque, operation: Io.Operation) Io.Cancelable!Io.Operation.Result {
1325	fiber.exitCancelRegion(thread);	2180	const ev: *Evented = @ptrCast(@alignCast(userdata));
1326		2181	switch (operation) {
1327	const completion = fiber.resultPointer(Completion);	2182	.file_read_streaming => \|o\| return .{
1328	switch (errno(completion.result)) {	2183	.file_read_streaming = ev.fileReadStreaming(o.file, o.data) catch \|err\| switch (err) {
1329	.SUCCESS => return @as(u32, @bitCast(completion.result)),	2184	error.Canceled => \|e\| return e,
1330	.INTR => unreachable,	2185	else => \|e\| e,
1331	.CANCELED => return error.Canceled,	2186	},
1332		2187	},
1333	.INVAL => return error.InvalidArgument,	2188	.file_write_streaming => \|o\| return .{
1334	.FAULT => unreachable,	2189	.file_write_streaming = ev.fileWriteStreaming(o.file, o.header, o.data, o.splat) catch \|err\| switch (err) {
1335	.NOENT => return error.ProcessNotFound,	2190	error.Canceled => \|e\| return e,
1336	.AGAIN => return error.WouldBlock,	2191	else => \|e\| e,
1337	.BADF => return error.NotOpenForWriting, // can be a race condition.	2192	},
1338	.DESTADDRREQ => unreachable, // `connect` was never called.	2193	},
1339	.DQUOT => return error.DiskQuota,	2194	.device_io_control => \|*o\| return .{ .device_io_control = try deviceIoControl(o) },
1340	.FBIG => return error.FileTooBig,
1341	.IO => return error.InputOutput,
1342	.NOSPC => return error.NoSpaceLeft,
1343	.ACCES => return error.AccessDenied,
1344	.PERM => return error.PermissionDenied,
1345	.PIPE => return error.BrokenPipe,
1346	.NXIO => return error.Unseekable,
1347	.SPIPE => return error.Unseekable,
1348	.OVERFLOW => return error.Unseekable,
1349	.BUSY => return error.DeviceBusy,
1350	.CONNRESET => return error.ConnectionResetByPeer,
1351	.MSGSIZE => return error.MessageOversize,
1352	else => \|err\| return std.posix.unexpectedErrno(err),
1353	}	2195	}
1354	}	2196	}
1355		2197
1356	fn now(userdata: ?*anyopaque, clockid: std.posix.clockid_t) Io.ClockGetTimeError!Io.Timestamp {	2198	fn fileReadStreaming(ev: *Evented, file: File, data: []const []u8) File.Reader.Error!usize {
1357	_ = userdata;	2199	var iovecs_buffer: [max_iovecs_len]iovec = undefined;
1358	const timespec = try std.posix.clock_gettime(clockid);	2200	var i: usize = 0;
1359	return @enumFromInt(@as(i128, timespec.sec) * std.time.ns_per_s + timespec.nsec);	2201	for (data) \|buf\| {
1360	}	2202	if (iovecs_buffer.len - i == 0) break;
		2203	if (buf.len != 0) {
		2204	iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len };
		2205	i += 1;
		2206	}
		2207	}
		2208	const dest = iovecs_buffer[0..i];
		2209	assert(dest[0].len > 0);
1361		2210
1362	fn sleep(userdata: ?*anyopaque, clockid: std.posix.clockid_t, deadline: Io.Deadline) Io.SleepError!void {	2211	var cancel_region: CancelRegion = .init();
1363	const el: *EventLoop = @ptrCast(@alignCast(userdata));	2212	defer cancel_region.deinit();
1364	const thread: *Thread = .current();	2213	return ev.preadv(&cancel_region, file.handle, dest, null);
1365	const iou = &thread.io_uring;	2214	}
1366	const fiber = thread.currentFiber();
1367	try fiber.enterCancelRegion(thread);
1368		2215
1369	const deadline_nanoseconds: i96 = switch (deadline) {	2216	fn fileWriteStreaming(
1370	.duration => \|duration\| duration.nanoseconds,	2217	ev: *Evented,
1371	.timestamp => \|timestamp\| @intFromEnum(timestamp),	2218	file: File,
		2219	header: []const u8,
		2220	data: []const []const u8,
		2221	splat: usize,
		2222	) File.Writer.Error!usize {
		2223	var iovecs: [max_iovecs_len]iovec_const = undefined;
		2224	var iovlen: iovlen_t = 0;
		2225	addBuf(&iovecs, &iovlen, header);
		2226	for (data[0 .. data.len - 1]) \|bytes\| addBuf(&iovecs, &iovlen, bytes);
		2227	const pattern = data[data.len - 1];
		2228	if (iovecs.len - iovlen != 0) switch (splat) {
		2229	0 => {},
		2230	1 => addBuf(&iovecs, &iovlen, pattern),
		2231	else => switch (pattern.len) {
		2232	0 => {},
		2233	1 => {
		2234	var backup_buffer: [splat_buffer_size]u8 = undefined;
		2235	const splat_buffer = &backup_buffer;
		2236	const memset_len = @min(splat_buffer.len, splat);
		2237	const buf = splat_buffer[0..memset_len];
		2238	@memset(buf, pattern[0]);
		2239	addBuf(&iovecs, &iovlen, buf);
		2240	var remaining_splat = splat - buf.len;
		2241	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0) {
		2242	assert(buf.len == splat_buffer.len);
		2243	addBuf(&iovecs, &iovlen, splat_buffer);
		2244	remaining_splat -= splat_buffer.len;
		2245	}
		2246	addBuf(&iovecs, &iovlen, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
		2247	},
		2248	else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
		2249	addBuf(&iovecs, &iovlen, pattern);
		2250	},
		2251	},
1372	};	2252	};
1373	const timespec: std.os.linux.kernel_timespec = .{	2253
1374	.sec = @intCast(@divFloor(deadline_nanoseconds, std.time.ns_per_s)),	2254	var cancel_region: CancelRegion = .init();
1375	.nsec = @intCast(@mod(deadline_nanoseconds, std.time.ns_per_s)),	2255	defer cancel_region.deinit();
		2256	return ev.pwritev(&cancel_region, file.handle, iovecs[0..iovlen], null);
		2257	}
		2258
		2259	fn deviceIoControl(o: *const Io.Operation.DeviceIoControl) Io.Cancelable!i32 {
		2260	var cancel_region: CancelRegion = .init();
		2261	defer cancel_region.deinit();
		2262	while (true) {
		2263	try cancel_region.await(.nothing);
		2264	const rc = linux.ioctl(o.file.handle, @bitCast(o.code), @intFromPtr(o.arg));
		2265	switch (linux.errno(rc)) {
		2266	.SUCCESS => return @bitCast(@as(u32, @truncate(rc))),
		2267	.INTR => continue,
		2268	else => \|err\| return -@as(i32, @intFromEnum(err)),
		2269	}
		2270	}
		2271	}
		2272
		2273	fn batchAwaitAsync(userdata: ?anyopaque, batch: Io.Batch) Io.Cancelable!void {
		2274	const ev: *Evented = @ptrCast(@alignCast(userdata));
		2275	var cancel_region: CancelRegion = .init();
		2276	defer cancel_region.deinit();
		2277	batchDrainSubmitted(batch, &cancel_region, false) catch \|err\| switch (err) {
		2278	error.ConcurrencyUnavailable => unreachable, // passed concurrency=false
		2279	else => \|e\| return e,
1376	};	2280	};
1377	getSqe(iou).* = .{	2281	while (true) {
1378	.opcode = .TIMEOUT,	2282	batchDrainReady(batch) catch \|err\| switch (err) {
1379	.flags = 0,	2283	error.Timeout => unreachable, // no timeout
1380	.ioprio = 0,	2284	};
1381	.fd = 0,	2285	if (batch.completed.head != .none) return;
		2286	ev.yield(null, .{ .batch_await = batch });
		2287	}
		2288	}
		2289
		2290	fn batchAwaitConcurrent(
		2291	userdata: ?*anyopaque,
		2292	batch: *Io.Batch,
		2293	timeout: Io.Timeout,
		2294	) Io.Batch.AwaitConcurrentError!void {
		2295	const ev: *Evented = @ptrCast(@alignCast(userdata));
		2296	var cancel_region: CancelRegion = .init();
		2297	defer cancel_region.deinit();
		2298	try batchDrainSubmitted(batch, &cancel_region, true);
		2299	const timespec: linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = while (true) {
		2300	batchDrainReady(batch) catch \|err\| switch (err) {
		2301	error.Timeout => unreachable, // no timeout
		2302	};
		2303	if (batch.completed.head != .none) return;
		2304	switch (timeout) {
		2305	.none => ev.yield(null, .{ .batch_await = batch }),
		2306	.duration => \|duration\| {
		2307	const ns = duration.raw.toNanoseconds();
		2308	break .{
		2309	.{
		2310	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
		2311	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
		2312	},
		2313	duration.clock,
		2314	0,
		2315	};
		2316	},
		2317	.deadline => \|deadline\| {
		2318	const ns = deadline.raw.toNanoseconds();
		2319	break .{
		2320	.{
		2321	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
		2322	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
		2323	},
		2324	deadline.clock,
		2325	linux.IORING_TIMEOUT_ABS,
		2326	};
		2327	},
		2328	}
		2329	};
		2330	{
		2331	const thread = try cancel_region.awaitIoUring();
		2332	thread.enqueue().* = .{
		2333	.opcode = .TIMEOUT,
		2334	.flags = 0,
		2335	.ioprio = 0,
		2336	.fd = 0,
		2337	.off = 0,
		2338	.addr = @intFromPtr(&timespec),
		2339	.len = 1,
		2340	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
		2341	.real => linux.IORING_TIMEOUT_REALTIME,
		2342	else => 0,
		2343	.boot => linux.IORING_TIMEOUT_BOOTTIME,
		2344	}),
		2345	.user_data = @intFromPtr(&batch.context) \| 0b11,
		2346	.buf_index = 0,
		2347	.personality = 0,
		2348	.splice_fd_in = 0,
		2349	.addr3 = 0,
		2350	.resv = 0,
		2351	};
		2352	}
		2353	while (batch.completed.head == .none) {
		2354	ev.yield(null, .{ .batch_await = batch });
		2355	batchDrainReady(batch) catch \|err\| switch (err) {
		2356	error.Timeout => \|e\| return if (batch.completed.head == .none) e,
		2357	};
		2358	if (batch.completed.head == .none) continue;
		2359	}
		2360	const thread = try cancel_region.awaitIoUring();
		2361	thread.enqueue().* = .{
		2362	.opcode = .TIMEOUT_REMOVE,
		2363	.flags = 0,
		2364	.ioprio = 0,
		2365	.fd = 0,
1382	.off = 0,	2366	.off = 0,
1383	.addr = @intFromPtr(&timespec),	2367	.addr = @intFromPtr(&batch.context) \| 0b11,
1384	.len = 1,	2368	.len = 0,
1385	.rw_flags = @as(u32, switch (deadline) {	2369	.rw_flags = 0,
1386	.duration => 0,	2370	.user_data = @intFromPtr(cancel_region.fiber),
1387	.timestamp => std.os.linux.IORING_TIMEOUT_ABS,
1388	}) \| @as(u32, switch (clockid) {
1389	.REALTIME => std.os.linux.IORING_TIMEOUT_REALTIME,
1390	.MONOTONIC => 0,
1391	.BOOTTIME => std.os.linux.IORING_TIMEOUT_BOOTTIME,
1392	else => return error.UnsupportedClock,
1393	}),
1394	.user_data = @intFromPtr(fiber),
1395	.buf_index = 0,	2371	.buf_index = 0,
1396	.personality = 0,	2372	.personality = 0,
1397	.splice_fd_in = 0,	2373	.splice_fd_in = 0,
1398	.addr3 = 0,	2374	.addr3 = 0,
1399	.resv = 0,	2375	.resv = 0,
1400	};	2376	};
		2377	ev.yield(null, .nothing);
		2378	switch (cancel_region.errno()) {
		2379	.SUCCESS => return,
		2380	.BUSY, .NOENT => {},
		2381	else => \|err\| unexpectedErrno(err) catch {},
		2382	}
		2383	while (true) {
		2384	batchDrainReady(batch) catch \|err\| switch (err) {
		2385	error.Timeout => return,
		2386	};
		2387	ev.yield(null, .{ .batch_await = batch });
		2388	}
		2389	}
1401		2390
1402	el.yield(null, .nothing);	2391	/// If `concurrency` is false, `error.ConcurrencyUnavailable` is unreachable.
1403	fiber.exitCancelRegion(thread);	2392	fn batchDrainSubmitted(
1404		2393	batch: *Io.Batch,
1405	const completion = fiber.resultPointer(Completion);	2394	cancel_region: *CancelRegion,
1406	switch (errno(completion.result)) {	2395	concurrency: bool,
1407	.SUCCESS, .TIME => return,	2396	) (Io.ConcurrentError \|\| Io.Cancelable)!void {
1408	.INTR => unreachable,	2397	var index = batch.submitted.head;
1409	.CANCELED => return error.Canceled,	2398	if (index == .none) return;
		2399	errdefer batch.submitted.head = index;
		2400	const thread = try cancel_region.awaitIoUring();
		2401	while (index != .none) {
		2402	const storage = &batch.storage[index.toIndex()];
		2403	const next_index = storage.submission.node.next;
		2404	if (@as(?Io.Operation.Result, operation: switch (storage.submission.operation) {
		2405	.file_read_streaming => \|o\| {
		2406	const buffer = for (o.data) \|buffer\| {
		2407	if (buffer.len != 0) break buffer;
		2408	} else break :operation .{ .file_read_streaming = 0 };
		2409	const fd = o.file.handle;
		2410	storage.* = .{ .pending = .{
		2411	.node = .{ .prev = batch.pending.tail, .next = .none },
		2412	.tag = .file_read_streaming,
		2413	.context = undefined,
		2414	} };
		2415	thread.enqueue().* = .{
		2416	.opcode = .READ,
		2417	.flags = 0,
		2418	.ioprio = 0,
		2419	.fd = fd,
		2420	.off = std.math.maxInt(u64),
		2421	.addr = @intFromPtr(buffer.ptr),
		2422	.len = @min(buffer.len, 0xfffff000),
		2423	.rw_flags = 0,
		2424	.user_data = @intFromPtr(&storage.pending.context) \| 0b10,
		2425	.buf_index = 0,
		2426	.personality = 0,
		2427	.splice_fd_in = 0,
		2428	.addr3 = 0,
		2429	.resv = 0,
		2430	};
		2431	break :operation null;
		2432	},
		2433	.file_write_streaming => \|o\| {
		2434	const buffer = buffer: {
		2435	if (o.header.len != 0) break :buffer o.header;
		2436	for (o.data[0 .. o.data.len - 1]) \|buffer\| {
		2437	if (buffer.len != 0) break :buffer buffer;
		2438	}
		2439	if (o.splat > 0) break :buffer o.data[o.data.len - 1];
		2440	break :operation .{ .file_write_streaming = 0 };
		2441	};
		2442	const fd = o.file.handle;
		2443	storage.* = .{ .pending = .{
		2444	.node = .{ .prev = batch.pending.tail, .next = .none },
		2445	.tag = .file_write_streaming,
		2446	.context = undefined,
		2447	} };
		2448	thread.enqueue().* = .{
		2449	.opcode = .WRITE,
		2450	.flags = 0,
		2451	.ioprio = 0,
		2452	.fd = fd,
		2453	.off = std.math.maxInt(u64),
		2454	.addr = @intFromPtr(buffer.ptr),
		2455	.len = @min(buffer.len, 0xfffff000),
		2456	.rw_flags = 0,
		2457	.user_data = @intFromPtr(&storage.pending.context) \| 0b10,
		2458	.buf_index = 0,
		2459	.personality = 0,
		2460	.splice_fd_in = 0,
		2461	.addr3 = 0,
		2462	.resv = 0,
		2463	};
		2464	break :operation null;
		2465	},
		2466	.device_io_control => \|o\| if (concurrency)
		2467	return error.ConcurrencyUnavailable
		2468	else
		2469	.{ .device_io_control = try deviceIoControl(&o) },
		2470	})) \|result\| {
		2471	switch (batch.completed.tail) {
		2472	.none => batch.completed.head = index,
		2473	else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next = index,
		2474	}
		2475	batch.completed.tail = index;
		2476	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
		2477	} else {
		2478	switch (batch.pending.tail) {
		2479	.none => batch.pending.head = index,
		2480	else => \|tail_index\| batch.storage[tail_index.toIndex()].pending.node.next = index,
		2481	}
		2482	batch.pending.tail = index;
		2483	storage.pending.context[0] = @intFromPtr(batch);
		2484	}
		2485	index = next_index;
		2486	}
		2487	batch.submitted = .{ .head = .none, .tail = .none };
		2488	}
1410		2489
1411	else => \|err\| return std.posix.unexpectedErrno(err),	2490	fn batchDrainReady(batch: *Io.Batch) Io.Timeout.Error!void {
		2491	while (@atomicRmw(?*anyopaque, &batch.context, .Xchg, null, .acquire)) \|head\| {
		2492	var next: usize = @intFromPtr(head);
		2493	var timeout = false;
		2494	while (cond: switch (@as(u2, @truncate(next))) {
		2495	0b00 => if (timeout) return error.Timeout else false,
		2496	0b01 => {
		2497	assert(!timeout);
		2498	return error.Timeout;
		2499	},
		2500	0b10 => true,
		2501	0b11 => {
		2502	assert(!timeout);
		2503	timeout = true;
		2504	break :cond true;
		2505	},
		2506	}) {
		2507	var context: *Io.Operation.Storage.Pending.Context = @ptrFromInt(next & ~@as(usize, 0b11));
		2508	next = context[0];
		2509	const completion: Completion = .{
		2510	.result = @bitCast(@as(u32, @intCast(context[1]))),
		2511	.flags = @intCast(context[2]),
		2512	};
		2513	const pending: *Io.Operation.Storage.Pending = @fieldParentPtr("context", context);
		2514	const storage: *Io.Operation.Storage = @fieldParentPtr("pending", pending);
		2515	const index: Io.Operation.OptionalIndex = .fromIndex(storage - batch.storage.ptr);
		2516	assert(completion.flags & linux.IORING_CQE_F_SKIP == 0);
		2517	switch (pending.node.prev) {
		2518	.none => batch.pending.head = pending.node.next,
		2519	else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.next =
		2520	pending.node.next,
		2521	}
		2522	switch (pending.node.next) {
		2523	.none => batch.pending.tail = pending.node.prev,
		2524	else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.prev =
		2525	pending.node.prev,
		2526	}
		2527	if (@as(?Io.Operation.Result, result: switch (pending.tag) {
		2528	.file_read_streaming => .{
		2529	.file_read_streaming = switch (completion.errno()) {
		2530	.SUCCESS => @as(u32, @bitCast(completion.result)),
		2531	.INTR => 0,
		2532	.CANCELED => break :result null,
		2533	.INVAL => \|err\| errnoBug(err),
		2534	.FAULT => \|err\| errnoBug(err),
		2535	.AGAIN => error.WouldBlock,
		2536	.BADF => \|err\| errnoBug(err), // File descriptor used after closed
		2537	.IO => error.InputOutput,
		2538	.ISDIR => error.IsDir,
		2539	.NOBUFS => error.SystemResources,
		2540	.NOMEM => error.SystemResources,
		2541	.NOTCONN => error.SocketUnconnected,
		2542	.CONNRESET => error.ConnectionResetByPeer,
		2543	else => \|err\| unexpectedErrno(err),
		2544	},
		2545	},
		2546	.file_write_streaming => .{
		2547	.file_write_streaming = switch (completion.errno()) {
		2548	.SUCCESS => @as(u32, @bitCast(completion.result)),
		2549	.INTR => 0,
		2550	.CANCELED => break :result null,
		2551	.INVAL => \|err\| errnoBug(err),
		2552	.FAULT => \|err\| errnoBug(err),
		2553	.AGAIN => error.WouldBlock,
		2554	.BADF => error.NotOpenForWriting, // Can be a race condition.
		2555	.DESTADDRREQ => \|err\| errnoBug(err), // `connect` was never called.
		2556	.DQUOT => error.DiskQuota,
		2557	.FBIG => error.FileTooBig,
		2558	.IO => error.InputOutput,
		2559	.NOSPC => error.NoSpaceLeft,
		2560	.PERM => error.PermissionDenied,
		2561	.PIPE => error.BrokenPipe,
		2562	.CONNRESET => \|err\| errnoBug(err), // Not a socket handle.
		2563	.BUSY => error.DeviceBusy,
		2564	else => \|err\| unexpectedErrno(err),
		2565	},
		2566	},
		2567	.device_io_control => unreachable,
		2568	})) \|result\| {
		2569	switch (batch.completed.tail) {
		2570	.none => batch.completed.head = index,
		2571	else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next =
		2572	index,
		2573	}
		2574	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
		2575	batch.completed.tail = index;
		2576	} else {
		2577	switch (batch.unused.tail) {
		2578	.none => batch.unused.head = index,
		2579	else => \|tail_index\| batch.storage[tail_index.toIndex()].unused.next = index,
		2580	}
		2581	storage.* = .{ .unused = .{ .prev = batch.unused.tail, .next = .none } };
		2582	batch.unused.tail = index;
		2583	}
		2584	}
1412	}	2585	}
1413	}	2586	}
1414		2587
1415	fn mutexLock(userdata: ?anyopaque, prev_state: Io.Mutex.State, mutex: Io.Mutex) error{Canceled}!void {	2588	fn batchCancel(userdata: ?anyopaque, batch: Io.Batch) void {
1416	const el: *EventLoop = @ptrCast(@alignCast(userdata));	2589	const ev: *Evented = @ptrCast(@alignCast(userdata));
1417	el.yield(null, .{ .mutex_lock = .{ .prev_state = prev_state, .mutex = mutex } });	2590	_ = ev;
		2591	batchDrainReady(batch) catch \|err\| switch (err) {
		2592	error.Timeout => unreachable, // no timeout
		2593	};
		2594	var index = batch.pending.head;
		2595	if (index == .none) return;
		2596	var cancel_region: CancelRegion = .initBlocked();
		2597	defer cancel_region.deinit();
		2598	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
		2599	error.Canceled => unreachable, // blocked
		2600	};
		2601	while (index != .none) {
		2602	const pending = &batch.storage[index.toIndex()].pending;
		2603	thread.enqueue().* = .{
		2604	.opcode = .ASYNC_CANCEL,
		2605	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
		2606	.ioprio = 0,
		2607	.fd = 0,
		2608	.off = 0,
		2609	.addr = @intFromPtr(&pending.context) \| 0b10,
		2610	.len = 0,
		2611	.rw_flags = 0,
		2612	.user_data = @intFromEnum(Completion.UserData.wakeup),
		2613	.buf_index = 0,
		2614	.personality = 0,
		2615	.splice_fd_in = 0,
		2616	.addr3 = 0,
		2617	.resv = 0,
		2618	};
		2619	index = pending.node.next;
		2620	}
		2621	while (batch.pending.head != .none) batchDrainReady(batch) catch \|err\| switch (err) {
		2622	error.Timeout => unreachable, // no timeout
		2623	};
1418	}	2624	}
1419	fn mutexUnlock(userdata: ?anyopaque, prev_state: Io.Mutex.State, mutex: Io.Mutex) void {	2625
1420	var maybe_waiting_fiber: ?*Fiber = @ptrFromInt(@intFromEnum(prev_state));	2626	fn dirCreateDir(
1421	while (if (maybe_waiting_fiber) \|waiting_fiber\| @cmpxchgWeak(	2627	userdata: ?*anyopaque,
1422	Io.Mutex.State,	2628	dir: Dir,
1423	&mutex.state,	2629	sub_path: []const u8,
1424	@enumFromInt(@intFromPtr(waiting_fiber)),	2630	permissions: Dir.Permissions,
1425	@enumFromInt(@intFromPtr(waiting_fiber.queue_next)),	2631	) Dir.CreateDirError!void {
1426	.release,	2632	const ev: *Evented = @ptrCast(@alignCast(userdata));
1427	.acquire,	2633
1428	) else @cmpxchgWeak(	2634	var path_buffer: [PATH_MAX]u8 = undefined;
1429	Io.Mutex.State,	2635	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
1430	&mutex.state,	2636
1431	.locked_once,	2637	var cancel_region: CancelRegion = .init();
1432	.unlocked,	2638	defer cancel_region.deinit();
1433	.release,	2639	while (true) {
1434	.acquire,	2640	const thread = try cancel_region.awaitIoUring();
1435	) orelse return) \|next_state\| maybe_waiting_fiber = @ptrFromInt(@intFromEnum(next_state));	2641	thread.enqueue().* = .{
1436	maybe_waiting_fiber.?.queue_next = null;	2642	.opcode = .MKDIRAT,
1437	const el: *EventLoop = @ptrCast(@alignCast(userdata));	2643	.flags = 0,
1438	el.yield(maybe_waiting_fiber.?, .reschedule);	2644	.ioprio = 0,
		2645	.fd = dir.handle,
		2646	.off = 0,
		2647	.addr = @intFromPtr(sub_path_posix.ptr),
		2648	.len = permissions.toMode(),
		2649	.rw_flags = 0,
		2650	.user_data = @intFromPtr(cancel_region.fiber),
		2651	.buf_index = 0,
		2652	.personality = 0,
		2653	.splice_fd_in = 0,
		2654	.addr3 = 0,
		2655	.resv = 0,
		2656	};
		2657	ev.yield(null, .nothing);
		2658	switch (cancel_region.errno()) {
		2659	.SUCCESS => return,
		2660	.INTR, .CANCELED => continue,
		2661	.ACCES => return error.AccessDenied,
		2662	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
		2663	.PERM => return error.PermissionDenied,
		2664	.DQUOT => return error.DiskQuota,
		2665	.EXIST => return error.PathAlreadyExists,
		2666	.FAULT => \|err\| return errnoBug(err),
		2667	.LOOP => return error.SymLinkLoop,
		2668	.MLINK => return error.LinkQuotaExceeded,
		2669	.NAMETOOLONG => return error.NameTooLong,
		2670	.NOENT => return error.FileNotFound,
		2671	.NOMEM => return error.SystemResources,
		2672	.NOSPC => return error.NoSpaceLeft,
		2673	.NOTDIR => return error.NotDir,
		2674	.ROFS => return error.ReadOnlyFileSystem,
		2675	// dragonfly: when dir_fd is unlinked from filesystem
		2676	.NOTCONN => return error.FileNotFound,
		2677	.ILSEQ => return error.BadPathName,
		2678	else => \|err\| return unexpectedErrno(err),
		2679	}
		2680	}
1439	}	2681	}
1440		2682
1441	const ConditionImpl = struct {	2683	fn dirCreateDirPath(
1442	tail: *Fiber,	2684	userdata: ?*anyopaque,
1443	event: union(enum) {	2685	dir: Dir,
1444	queued,	2686	sub_path: []const u8,
1445	wake: Io.Condition.Wake,	2687	permissions: Dir.Permissions,
1446	},	2688	) Dir.CreateDirPathError!Dir.CreatePathStatus {
1447	};	2689	const ev: *Evented = @ptrCast(@alignCast(userdata));
1448		2690
1449	fn conditionWait(userdata: ?anyopaque, cond: Io.Condition, mutex: *Io.Mutex) Io.Cancelable!void {	2691	var it = Dir.path.componentIterator(sub_path);
1450	const el: *EventLoop = @ptrCast(@alignCast(userdata));	2692	var status: Dir.CreatePathStatus = .existed;
1451	el.yield(null, .{ .condition_wait = .{ .cond = cond, .mutex = mutex } });	2693	var component = it.last() orelse return error.BadPathName;
1452	const thread = Thread.current();	2694	while (true) {
1453	const fiber = thread.currentFiber();	2695	if (dirCreateDir(ev, dir, component.path, permissions)) \|_\| {
1454	const cond_impl = fiber.resultPointer(ConditionImpl);	2696	status = .created;
1455	try mutex.lock(el.io());	2697	} else \|err\| switch (err) {
1456	switch (cond_impl.event) {	2698	error.PathAlreadyExists => {
1457	.queued => {},	2699	// stat the file and return an error if it's not a directory
1458	.wake => \|wake\| if (fiber.queue_next) \|next_fiber\| switch (wake) {	2700	// this is important because otherwise a dangling symlink
1459	.one => if (@cmpxchgStrong(	2701	// could cause an infinite loop
1460	?*Fiber,	2702	const fstat = try dirStatFile(ev, dir, component.path, .{});
1461	@as(?Fiber, @ptrCast(&cond.state)),	2703	if (fstat.kind != .directory) return error.NotDir;
1462	null,	2704	},
1463	next_fiber,	2705	error.FileNotFound => \|e\| {
1464	.release,	2706	component = it.previous() orelse return e;
1465	.acquire,	2707	continue;
1466	)) \|old_fiber\| {
1467	const old_cond_impl = old_fiber.?.resultPointer(ConditionImpl);
1468	assert(old_cond_impl.tail.queue_next == null);
1469	old_cond_impl.tail.queue_next = next_fiber;
1470	old_cond_impl.tail = cond_impl.tail;
1471	},	2708	},
1472	.all => el.schedule(thread, .{ .head = next_fiber, .tail = cond_impl.tail }),	2709	else => \|e\| return e,
		2710	}
		2711	component = it.next() orelse return status;
		2712	}
		2713	}
		2714
		2715	fn dirCreateDirPathOpen(
		2716	userdata: ?*anyopaque,
		2717	dir: Dir,
		2718	sub_path: []const u8,
		2719	permissions: Dir.Permissions,
		2720	options: Dir.OpenOptions,
		2721	) Dir.CreateDirPathOpenError!Dir {
		2722	const ev: *Evented = @ptrCast(@alignCast(userdata));
		2723	return dirOpenDir(ev, dir, sub_path, options) catch \|err\| switch (err) {
		2724	error.FileNotFound => {
		2725	_ = try dirCreateDirPath(ev, dir, sub_path, permissions);
		2726	return dirOpenDir(ev, dir, sub_path, options);
		2727	},
		2728	else => \|e\| return e,
		2729	};
		2730	}
		2731
		2732	fn dirOpenDir(
		2733	userdata: ?*anyopaque,
		2734	dir: Dir,
		2735	sub_path: []const u8,
		2736	options: Dir.OpenOptions,
		2737	) Dir.OpenError!Dir {
		2738	const ev: *Evented = @ptrCast(@alignCast(userdata));
		2739
		2740	var path_buffer: [PATH_MAX]u8 = undefined;
		2741	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
		2742
		2743	var cancel_region: CancelRegion = .init();
		2744	defer cancel_region.deinit();
		2745	return .{
		2746	.handle = ev.openat(&cancel_region, dir.handle, sub_path_posix, .{
		2747	.ACCMODE = .RDONLY,
		2748	.DIRECTORY = true,
		2749	.NOFOLLOW = !options.follow_symlinks,
		2750	.CLOEXEC = true,
		2751	.PATH = !options.iterate,
		2752	}, 0) catch \|err\| switch (err) {
		2753	error.IsDir => return errnoBug(.ISDIR),
		2754	error.WouldBlock => return errnoBug(.AGAIN),
		2755	error.FileTooBig => return errnoBug(.FBIG),
		2756	error.NoSpaceLeft => return errnoBug(.NOSPC),
		2757	error.DeviceBusy => return errnoBug(.BUSY), // O_EXCL not passed
		2758	error.FileBusy => return errnoBug(.TXTBSY),
		2759	error.PathAlreadyExists => return errnoBug(.EXIST), // Not creating.
		2760	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
		2761	error.AntivirusInterference => unreachable, // Windows-only
		2762	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
		2763	else => \|e\| return e,
1473	},	2764	},
		2765	};
		2766	}
		2767
		2768	fn dirStat(userdata: ?*anyopaque, dir: Dir) Dir.StatError!Dir.Stat {
		2769	const ev: *Evented = @ptrCast(@alignCast(userdata));
		2770	var cancel_region: CancelRegion = .init();
		2771	defer cancel_region.deinit();
		2772	return ev.stat(&cancel_region, dir.handle);
		2773	}
		2774
		2775	fn dirStatFile(
		2776	userdata: ?*anyopaque,
		2777	dir: Dir,
		2778	sub_path: []const u8,
		2779	options: Dir.StatFileOptions,
		2780	) Dir.StatFileError!File.Stat {
		2781	const ev: *Evented = @ptrCast(@alignCast(userdata));
		2782	var path_buffer: [PATH_MAX]u8 = undefined;
		2783	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
		2784	var cancel_region: CancelRegion = .init();
		2785	defer cancel_region.deinit();
		2786	return ev.statx(&cancel_region, dir.handle, sub_path_posix, linux.AT.NO_AUTOMOUNT \|
		2787	@as(u32, if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW));
		2788	}
		2789
		2790	fn dirAccess(
		2791	userdata: ?*anyopaque,
		2792	dir: Dir,
		2793	sub_path: []const u8,
		2794	options: Dir.AccessOptions,
		2795	) Dir.AccessError!void {
		2796	const ev: *Evented = @ptrCast(@alignCast(userdata));
		2797	_ = ev;
		2798
		2799	var path_buffer: [PATH_MAX]u8 = undefined;
		2800	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
		2801
		2802	const mode: u32 =
		2803	@as(u32, if (options.read) linux.R_OK else 0) \|
		2804	@as(u32, if (options.write) linux.W_OK else 0) \|
		2805	@as(u32, if (options.execute) linux.X_OK else 0);
		2806	const flags: u32 = if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW;
		2807
		2808	var cancel_region: CancelRegion = .init();
		2809	defer cancel_region.deinit();
		2810	while (true) {
		2811	try cancel_region.await(.nothing);
		2812	switch (linux.errno(linux.faccessat(dir.handle, sub_path_posix, mode, flags))) {
		2813	.SUCCESS => return,
		2814	.INTR => continue,
		2815	.ACCES => return error.AccessDenied,
		2816	.PERM => return error.PermissionDenied,
		2817	.ROFS => return error.ReadOnlyFileSystem,
		2818	.LOOP => return error.SymLinkLoop,
		2819	.TXTBSY => return error.FileBusy,
		2820	.NOTDIR => return error.FileNotFound,
		2821	.NOENT => return error.FileNotFound,
		2822	.NAMETOOLONG => return error.NameTooLong,
		2823	.INVAL => \|err\| return errnoBug(err),
		2824	.FAULT => \|err\| return errnoBug(err),
		2825	.IO => return error.InputOutput,
		2826	.NOMEM => return error.SystemResources,
		2827	.ILSEQ => return error.BadPathName,
		2828	else => \|err\| return unexpectedErrno(err),
		2829	}
		2830	}
		2831	}
		2832
		2833	fn dirCreateFile(
		2834	userdata: ?*anyopaque,
		2835	dir: Dir,
		2836	sub_path: []const u8,
		2837	flags: File.CreateFlags,
		2838	) File.OpenError!File {
		2839	const ev: *Evented = @ptrCast(@alignCast(userdata));
		2840
		2841	var path_buffer: [PATH_MAX]u8 = undefined;
		2842	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
		2843
		2844	var cancel_region: CancelRegion = .init();
		2845	defer cancel_region.deinit();
		2846	const fd = try ev.openat(&cancel_region, dir.handle, sub_path_posix, .{
		2847	.ACCMODE = if (flags.read) .RDWR else .WRONLY,
		2848	.CREAT = true,
		2849	.TRUNC = flags.truncate,
		2850	.EXCL = flags.exclusive,
		2851	.CLOEXEC = true,
		2852	}, flags.permissions.toMode());
		2853	errdefer ev.close(fd);
		2854
		2855	switch (flags.lock) {
		2856	.none => {},
		2857	.shared, .exclusive => try ev.flock(
		2858	&cancel_region,
		2859	fd,
		2860	flags.lock,
		2861	if (flags.lock_nonblocking) .nonblocking else .blocking,
		2862	),
1474	}	2863	}
1475	fiber.queue_next = null;	2864
		2865	return .{ .handle = fd, .flags = .{ .nonblocking = false } };
1476	}	2866	}
1477		2867
1478	fn conditionWake(userdata: ?anyopaque, cond: Io.Condition, wake: Io.Condition.Wake) void {	2868	fn dirCreateFileAtomic(
1479	const el: *EventLoop = @ptrCast(@alignCast(userdata));	2869	userdata: ?*anyopaque,
1480	const waiting_fiber = @atomicRmw(?Fiber, @as(?*Fiber, @ptrCast(&cond.state)), .Xchg, null, .acquire) orelse return;	2870	dir: Dir,
1481	waiting_fiber.resultPointer(ConditionImpl).event = .{ .wake = wake };	2871	dest_path: []const u8,
1482	el.yield(waiting_fiber, .reschedule);	2872	options: Dir.CreateFileAtomicOptions,
		2873	) Dir.CreateFileAtomicError!File.Atomic {
		2874	const ev: *Evented = @ptrCast(@alignCast(userdata));
		2875	// Linux has O_TMPFILE, but linkat() does not support AT_REPLACE, so it's
		2876	// useless when we have to make up a bogus path name to do the rename()
		2877	// anyway.
		2878	if (!options.replace) tmpfile: {
		2879	const flags: linux.O = if (@hasField(linux.O, "TMPFILE")) .{
		2880	.ACCMODE = .RDWR,
		2881	.TMPFILE = true,
		2882	.DIRECTORY = true,
		2883	.CLOEXEC = true,
		2884	} else if (@hasField(linux.O, "TMPFILE0") and !@hasField(linux.O, "TMPFILE2")) .{
		2885	.ACCMODE = .RDWR,
		2886	.TMPFILE0 = true,
		2887	.TMPFILE1 = true,
		2888	.DIRECTORY = true,
		2889	.CLOEXEC = true,
		2890	} else break :tmpfile;
		2891
		2892	const dest_dirname = Dir.path.dirname(dest_path);
		2893	if (dest_dirname) \|dirname\| {
		2894	// This has a nice side effect of preemptively triggering EISDIR or
		2895	// ENOENT, avoiding the ambiguity below.
		2896	_ = dirCreateDirPath(ev, dir, dirname, .default_dir) catch \|err\| switch (err) {
		2897	// None of these make sense in this context.
		2898	error.IsDir,
		2899	error.Streaming,
		2900	error.DiskQuota,
		2901	error.PathAlreadyExists,
		2902	error.LinkQuotaExceeded,
		2903	error.PipeBusy,
		2904	error.FileTooBig,
		2905	error.DeviceBusy,
		2906	error.FileLocksUnsupported,
		2907	error.FileBusy,
		2908	=> return error.Unexpected,
		2909
		2910	else => \|e\| return e,
		2911	};
		2912	}
		2913
		2914	var path_buffer: [PATH_MAX]u8 = undefined;
		2915	const sub_path_posix = try pathToPosix(dest_dirname orelse ".", &path_buffer);
		2916
		2917	var cancel_region: CancelRegion = .init();
		2918	defer cancel_region.deinit();
		2919	return .{
		2920	.file = .{
		2921	.handle = ev.openat(
		2922	&cancel_region,
		2923	dir.handle,
		2924	sub_path_posix,
		2925	flags,
		2926	options.permissions.toMode(),
		2927	) catch \|err\| switch (err) {
		2928	error.IsDir, error.FileNotFound => {
		2929	// Ambiguous error code. It might mean the file system
		2930	// does not support O_TMPFILE. Therefore, we must fall
		2931	// back to not using O_TMPFILE.
		2932	break :tmpfile;
		2933	},
		2934	error.FileTooBig => return errnoBug(.FBIG),
		2935	error.DeviceBusy => return errnoBug(.BUSY), // O_EXCL not passed
		2936	error.PathAlreadyExists => return errnoBug(.EXIST), // Not creating.
		2937	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
		2938	error.AntivirusInterference => unreachable, // Windows-only
		2939	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
		2940	else => \|e\| return e,
		2941	},
		2942	.flags = .{ .nonblocking = false },
		2943	},
		2944	.file_basename_hex = 0,
		2945	.dest_sub_path = dest_path,
		2946	.file_open = true,
		2947	.file_exists = false,
		2948	.close_dir_on_deinit = false,
		2949	.dir = dir,
		2950	};
		2951	}
		2952
		2953	if (Dir.path.dirname(dest_path)) \|dirname\| {
		2954	const new_dir = if (options.make_path)
		2955	dirCreateDirPathOpen(ev, dir, dirname, .default_dir, .{}) catch \|err\| switch (err) {
		2956	// None of these make sense in this context.
		2957	error.IsDir,
		2958	error.Streaming,
		2959	error.DiskQuota,
		2960	error.PathAlreadyExists,
		2961	error.LinkQuotaExceeded,
		2962	error.PipeBusy,
		2963	error.FileTooBig,
		2964	error.FileLocksUnsupported,
		2965	error.DeviceBusy,
		2966	=> return error.Unexpected,
		2967
		2968	else => \|e\| return e,
		2969	}
		2970	else
		2971	try dirOpenDir(ev, dir, dirname, .{});
		2972
		2973	return ev.atomicFileInit(Dir.path.basename(dest_path), options.permissions, new_dir, true);
		2974	}
		2975
		2976	return ev.atomicFileInit(dest_path, options.permissions, dir, false);
1483	}	2977	}
1484		2978
1485	fn errno(signed: i32) std.os.linux.E {	2979	fn atomicFileInit(
1486	return .init(@bitCast(@as(isize, signed)));	2980	ev: *Evented,
		2981	dest_basename: []const u8,
		2982	permissions: File.Permissions,
		2983	dir: Dir,
		2984	close_dir_on_deinit: bool,
		2985	) Dir.CreateFileAtomicError!File.Atomic {
		2986	while (true) {
		2987	var random_integer: u64 = undefined;
		2988	random(ev, @ptrCast(&random_integer));
		2989	const tmp_sub_path = std.fmt.hex(random_integer);
		2990	const file = dirCreateFile(ev, dir, &tmp_sub_path, .{
		2991	.permissions = permissions,
		2992	.exclusive = true,
		2993	}) catch \|err\| switch (err) {
		2994	error.PathAlreadyExists => continue,
		2995	error.DeviceBusy => continue,
		2996	error.FileBusy => continue,
		2997
		2998	error.IsDir => return error.Unexpected, // No path components.
		2999	error.FileTooBig => return error.Unexpected, // Creating, not opening.
		3000	error.FileLocksUnsupported => return error.Unexpected, // Not asking for locks.
		3001	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
		3002
		3003	else => \|e\| return e,
		3004	};
		3005	return .{
		3006	.file = file,
		3007	.file_basename_hex = random_integer,
		3008	.dest_sub_path = dest_basename,
		3009	.file_open = true,
		3010	.file_exists = true,
		3011	.close_dir_on_deinit = close_dir_on_deinit,
		3012	.dir = dir,
		3013	};
		3014	}
1487	}	3015	}
1488		3016
1489	fn getSqe(iou: IoUring) std.os.linux.io_uring_sqe {	3017	fn dirOpenFile(
1490	while (true) return iou.get_sqe() catch {	3018	userdata: ?*anyopaque,
1491	_ = iou.submit_and_wait(0) catch \|err\| switch (err) {	3019	dir: Dir,
1492	error.SignalInterrupt => std.log.warn("submit_and_wait failed with SignalInterrupt", .{}),	3020	sub_path: []const u8,
1493	else => \|e\| @panic(@errorName(e)),	3021	flags: File.OpenFlags,
		3022	) File.OpenError!File {
		3023	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3024
		3025	var path_buffer: [PATH_MAX]u8 = undefined;
		3026	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
		3027
		3028	var cancel_region: CancelRegion = .init();
		3029	defer cancel_region.deinit();
		3030	const fd = try ev.openat(&cancel_region, dir.handle, sub_path_posix, .{
		3031	.ACCMODE = switch (flags.mode) {
		3032	.read_only => .RDONLY,
		3033	.write_only => .WRONLY,
		3034	.read_write => .RDWR,
		3035	},
		3036	.NOCTTY = !flags.allow_ctty,
		3037	.NOFOLLOW = !flags.follow_symlinks,
		3038	.CLOEXEC = true,
		3039	.PATH = flags.path_only,
		3040	}, 0);
		3041	errdefer ev.close(fd);
		3042
		3043	if (!flags.allow_directory) {
		3044	const is_dir = is_dir: {
		3045	const s = ev.stat(&cancel_region, fd) catch \|err\| switch (err) {
		3046	// The directory-ness is either unknown or unknowable
		3047	error.Streaming => break :is_dir false,
		3048	else => \|e\| return e,
		3049	};
		3050	break :is_dir s.kind == .directory;
		3051	};
		3052	if (is_dir) return error.IsDir;
		3053	}
		3054
		3055	switch (flags.lock) {
		3056	.none => {},
		3057	.shared, .exclusive => try ev.flock(
		3058	&cancel_region,
		3059	fd,
		3060	flags.lock,
		3061	if (flags.lock_nonblocking) .nonblocking else .blocking,
		3062	),
		3063	}
		3064
		3065	return .{ .handle = fd, .flags = .{ .nonblocking = false } };
		3066	}
		3067
		3068	fn dirClose(userdata: ?*anyopaque, dirs: []const Dir) void {
		3069	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3070	for (dirs) \|dir\| ev.close(dir.handle);
		3071	}
		3072
		3073	fn dirRead(userdata: ?anyopaque, dr: Dir.Reader, buffer: []Dir.Entry) Dir.Reader.Error!usize {
		3074	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3075	var buffer_index: usize = 0;
		3076	while (buffer.len - buffer_index != 0) {
		3077	if (dr.end - dr.index == 0) {
		3078	// Refill the buffer, unless we've already created references to
		3079	// buffered data.
		3080	if (buffer_index != 0) break;
		3081	var cancel_region: CancelRegion = .init();
		3082	defer cancel_region.deinit();
		3083	if (dr.state == .reset) {
		3084	ev.lseek(&cancel_region, dr.dir.handle, 0, linux.SEEK.SET) catch \|err\| switch (err) {
		3085	error.Unseekable => return error.Unexpected,
		3086	else => \|e\| return e,
		3087	};
		3088	dr.state = .reading;
		3089	}
		3090	const n = while (true) {
		3091	try cancel_region.await(.nothing);
		3092	const rc = linux.getdents64(dr.dir.handle, dr.buffer.ptr, dr.buffer.len);
		3093	switch (linux.errno(rc)) {
		3094	.SUCCESS => break rc,
		3095	.INTR => continue,
		3096	.BADF => \|err\| return errnoBug(err), // Dir is invalid or was opened without iteration ability.
		3097	.FAULT => \|err\| return errnoBug(err),
		3098	.NOTDIR => \|err\| return errnoBug(err),
		3099	// To be consistent across platforms, iteration
		3100	// ends if the directory being iterated is deleted
		3101	// during iteration. This matches the behavior of
		3102	// non-Linux, non-WASI UNIX platforms.
		3103	.NOENT => {
		3104	dr.state = .finished;
		3105	return 0;
		3106	},
		3107	// This can occur when reading /proc/$PID/net, or
		3108	// if the provided buffer is too small. Neither
		3109	// scenario is intended to be handled by this API.
		3110	.INVAL => return error.Unexpected,
		3111	.ACCES => return error.AccessDenied, // Lacking permission to iterate this directory.
		3112	else => \|err\| return unexpectedErrno(err),
		3113	}
		3114	};
		3115	if (n == 0) {
		3116	dr.state = .finished;
		3117	return 0;
		3118	}
		3119	dr.index = 0;
		3120	dr.end = n;
		3121	}
		3122	// Linux aligns the header by padding after the null byte of the name
		3123	// to align the next entry. This means we can find the end of the name
		3124	// by looking at only the 8 bytes before the next record. However since
		3125	// file names are usually short it's better to keep the machine code
		3126	// simpler.
		3127	//
		3128	// Furthermore, I observed qemu user mode to not align this struct, so
		3129	// this code makes the conservative choice to not assume alignment.
		3130	const linux_entry: *align(1) linux.dirent64 = @ptrCast(&dr.buffer[dr.index]);
		3131	const next_index = dr.index + linux_entry.reclen;
		3132	dr.index = next_index;
		3133	const name_ptr: [*]u8 = &linux_entry.name;
		3134	const padded_name = name_ptr[0 .. linux_entry.reclen - @offsetOf(linux.dirent64, "name")];
		3135	const name_len = std.mem.findScalar(u8, padded_name, 0).?;
		3136	const name = name_ptr[0..name_len :0];
		3137
		3138	if (std.mem.eql(u8, name, ".") or std.mem.eql(u8, name, "..")) continue;
		3139
		3140	const entry_kind: File.Kind = switch (linux_entry.type) {
		3141	linux.DT.BLK => .block_device,
		3142	linux.DT.CHR => .character_device,
		3143	linux.DT.DIR => .directory,
		3144	linux.DT.FIFO => .named_pipe,
		3145	linux.DT.LNK => .sym_link,
		3146	linux.DT.REG => .file,
		3147	linux.DT.SOCK => .unix_domain_socket,
		3148	else => .unknown,
		3149	};
		3150	buffer[buffer_index] = .{
		3151	.name = name,
		3152	.kind = entry_kind,
		3153	.inode = linux_entry.ino,
1494	};	3154	};
1495	continue;	3155	buffer_index += 1;
		3156	}
		3157	return buffer_index;
		3158	}
		3159
		3160	fn dirRealPath(userdata: ?*anyopaque, dir: Dir, out_buffer: []u8) Dir.RealPathError!usize {
		3161	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3162	var cancel_region: CancelRegion = .init();
		3163	defer cancel_region.deinit();
		3164	return ev.realPath(&cancel_region, dir.handle, out_buffer);
		3165	}
		3166
		3167	fn dirRealPathFile(
		3168	userdata: ?*anyopaque,
		3169	dir: Dir,
		3170	sub_path: []const u8,
		3171	out_buffer: []u8,
		3172	) Dir.RealPathFileError!usize {
		3173	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3174
		3175	var path_buffer: [PATH_MAX]u8 = undefined;
		3176	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
		3177
		3178	var cancel_region: CancelRegion = .init();
		3179	defer cancel_region.deinit();
		3180	const fd = ev.openat(&cancel_region, dir.handle, sub_path_posix, .{
		3181	.CLOEXEC = true,
		3182	.PATH = true,
		3183	}, 0) catch \|err\| switch (err) {
		3184	error.WouldBlock => return errnoBug(.AGAIN),
		3185	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
		3186	else => \|e\| return e,
1496	};	3187	};
		3188	defer ev.close(fd);
		3189	return ev.realPath(&cancel_region, fd, out_buffer);
		3190	}
		3191
		3192	fn dirDeleteFile(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteFileError!void {
		3193	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3194
		3195	var path_buffer: [PATH_MAX]u8 = undefined;
		3196	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
		3197
		3198	var cancel_region: CancelRegion = .init();
		3199	defer cancel_region.deinit();
		3200	while (true) {
		3201	const thread = try cancel_region.awaitIoUring();
		3202	thread.enqueue().* = .{
		3203	.opcode = .UNLINKAT,
		3204	.flags = 0,
		3205	.ioprio = 0,
		3206	.fd = dir.handle,
		3207	.off = 0,
		3208	.addr = @intFromPtr(sub_path_posix.ptr),
		3209	.len = 0,
		3210	.rw_flags = 0,
		3211	.user_data = @intFromPtr(cancel_region.fiber),
		3212	.buf_index = 0,
		3213	.personality = 0,
		3214	.splice_fd_in = 0,
		3215	.addr3 = 0,
		3216	.resv = 0,
		3217	};
		3218	ev.yield(null, .nothing);
		3219	switch (cancel_region.errno()) {
		3220	.SUCCESS => return,
		3221	.INTR, .CANCELED => continue,
		3222	.PERM => return error.PermissionDenied,
		3223	.ACCES => return error.AccessDenied,
		3224	.BUSY => return error.FileBusy,
		3225	.FAULT => \|err\| return errnoBug(err),
		3226	.IO => return error.FileSystem,
		3227	.ISDIR => return error.IsDir,
		3228	.LOOP => return error.SymLinkLoop,
		3229	.NAMETOOLONG => return error.NameTooLong,
		3230	.NOENT => return error.FileNotFound,
		3231	.NOTDIR => return error.NotDir,
		3232	.NOMEM => return error.SystemResources,
		3233	.ROFS => return error.ReadOnlyFileSystem,
		3234	.EXIST => \|err\| return errnoBug(err),
		3235	.NOTEMPTY => \|err\| return errnoBug(err), // Not passing AT.REMOVEDIR
		3236	.ILSEQ => return error.BadPathName,
		3237	.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
		3238	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
		3239	else => \|err\| return unexpectedErrno(err),
		3240	}
		3241	}
		3242	}
		3243
		3244	fn dirDeleteDir(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteDirError!void {
		3245	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3246
		3247	var path_buffer: [PATH_MAX]u8 = undefined;
		3248	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
		3249
		3250	var cancel_region: CancelRegion = .init();
		3251	defer cancel_region.deinit();
		3252	while (true) {
		3253	const thread = try cancel_region.awaitIoUring();
		3254	thread.enqueue().* = .{
		3255	.opcode = .UNLINKAT,
		3256	.flags = 0,
		3257	.ioprio = 0,
		3258	.fd = dir.handle,
		3259	.off = 0,
		3260	.addr = @intFromPtr(sub_path_posix.ptr),
		3261	.len = 0,
		3262	.rw_flags = linux.AT.REMOVEDIR,
		3263	.user_data = @intFromPtr(cancel_region.fiber),
		3264	.buf_index = 0,
		3265	.personality = 0,
		3266	.splice_fd_in = 0,
		3267	.addr3 = 0,
		3268	.resv = 0,
		3269	};
		3270	ev.yield(null, .nothing);
		3271	switch (cancel_region.errno()) {
		3272	.SUCCESS => return,
		3273	.INTR, .CANCELED => continue,
		3274	.ACCES => return error.AccessDenied,
		3275	.PERM => return error.PermissionDenied,
		3276	.BUSY => return error.FileBusy,
		3277	.FAULT => \|err\| return errnoBug(err),
		3278	.IO => return error.FileSystem,
		3279	.ISDIR => \|err\| return errnoBug(err),
		3280	.LOOP => return error.SymLinkLoop,
		3281	.NAMETOOLONG => return error.NameTooLong,
		3282	.NOENT => return error.FileNotFound,
		3283	.NOTDIR => return error.NotDir,
		3284	.NOMEM => return error.SystemResources,
		3285	.ROFS => return error.ReadOnlyFileSystem,
		3286	.EXIST => \|err\| return errnoBug(err),
		3287	.NOTEMPTY => return error.DirNotEmpty,
		3288	.ILSEQ => return error.BadPathName,
		3289	.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
		3290	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
		3291	else => \|err\| return unexpectedErrno(err),
		3292	}
		3293	}
		3294	}
		3295
		3296	fn dirRename(
		3297	userdata: ?*anyopaque,
		3298	old_dir: Dir,
		3299	old_sub_path: []const u8,
		3300	new_dir: Dir,
		3301	new_sub_path: []const u8,
		3302	) Dir.RenameError!void {
		3303	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3304
		3305	var old_path_buffer: [PATH_MAX]u8 = undefined;
		3306	var new_path_buffer: [PATH_MAX]u8 = undefined;
		3307
		3308	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
		3309	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
		3310
		3311	var cancel_region: CancelRegion = .init();
		3312	defer cancel_region.deinit();
		3313	return ev.renameat(
		3314	&cancel_region,
		3315	old_dir.handle,
		3316	old_sub_path_posix,
		3317	new_dir.handle,
		3318	new_sub_path_posix,
		3319	.{},
		3320	);
		3321	}
		3322
		3323	fn dirRenamePreserve(
		3324	userdata: ?*anyopaque,
		3325	old_dir: Dir,
		3326	old_sub_path: []const u8,
		3327	new_dir: Dir,
		3328	new_sub_path: []const u8,
		3329	) Dir.RenamePreserveError!void {
		3330	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3331
		3332	var old_path_buffer: [PATH_MAX]u8 = undefined;
		3333	var new_path_buffer: [PATH_MAX]u8 = undefined;
		3334
		3335	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
		3336	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
		3337
		3338	var cancel_region: CancelRegion = .init();
		3339	defer cancel_region.deinit();
		3340	return ev.renameat(
		3341	&cancel_region,
		3342	old_dir.handle,
		3343	old_sub_path_posix,
		3344	new_dir.handle,
		3345	new_sub_path_posix,
		3346	.{ .NOREPLACE = true },
		3347	);
		3348	}
		3349
		3350	fn dirSymLink(
		3351	userdata: ?*anyopaque,
		3352	dir: Dir,
		3353	target_path: []const u8,
		3354	sym_link_path: []const u8,
		3355	flags: Dir.SymLinkFlags,
		3356	) Dir.SymLinkError!void {
		3357	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3358	_ = flags;
		3359
		3360	var target_path_buffer: [PATH_MAX]u8 = undefined;
		3361	var sym_link_path_buffer: [PATH_MAX]u8 = undefined;
		3362
		3363	const target_path_posix = try pathToPosix(target_path, &target_path_buffer);
		3364	const sym_link_path_posix = try pathToPosix(sym_link_path, &sym_link_path_buffer);
		3365
		3366	var cancel_region: CancelRegion = .init();
		3367	defer cancel_region.deinit();
		3368	while (true) {
		3369	const thread = try cancel_region.awaitIoUring();
		3370	thread.enqueue().* = .{
		3371	.opcode = .SYMLINKAT,
		3372	.flags = 0,
		3373	.ioprio = 0,
		3374	.fd = dir.handle,
		3375	.off = @intFromPtr(sym_link_path_posix.ptr),
		3376	.addr = @intFromPtr(target_path_posix.ptr),
		3377	.len = 0,
		3378	.rw_flags = 0,
		3379	.user_data = @intFromPtr(cancel_region.fiber),
		3380	.buf_index = 0,
		3381	.personality = 0,
		3382	.splice_fd_in = 0,
		3383	.addr3 = 0,
		3384	.resv = 0,
		3385	};
		3386	ev.yield(null, .nothing);
		3387	switch (cancel_region.errno()) {
		3388	.SUCCESS => return,
		3389	.INTR, .CANCELED => continue,
		3390	.FAULT => \|err\| return errnoBug(err),
		3391	.INVAL => \|err\| return errnoBug(err),
		3392	.ACCES => return error.AccessDenied,
		3393	.PERM => return error.PermissionDenied,
		3394	.DQUOT => return error.DiskQuota,
		3395	.EXIST => return error.PathAlreadyExists,
		3396	.IO => return error.FileSystem,
		3397	.LOOP => return error.SymLinkLoop,
		3398	.NAMETOOLONG => return error.NameTooLong,
		3399	.NOENT => return error.FileNotFound,
		3400	.NOTDIR => return error.NotDir,
		3401	.NOMEM => return error.SystemResources,
		3402	.NOSPC => return error.NoSpaceLeft,
		3403	.ROFS => return error.ReadOnlyFileSystem,
		3404	.ILSEQ => return error.BadPathName,
		3405	else => \|err\| return unexpectedErrno(err),
		3406	}
		3407	}
		3408	}
		3409
		3410	fn dirReadLink(
		3411	userdata: ?*anyopaque,
		3412	dir: Dir,
		3413	sub_path: []const u8,
		3414	buffer: []u8,
		3415	) Dir.ReadLinkError!usize {
		3416	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3417	_ = ev;
		3418
		3419	var sub_path_buffer: [PATH_MAX]u8 = undefined;
		3420	const sub_path_posix = try pathToPosix(sub_path, &sub_path_buffer);
		3421
		3422	var cancel_region: CancelRegion = .init();
		3423	defer cancel_region.deinit();
		3424	while (true) {
		3425	try cancel_region.await(.nothing);
		3426	const rc = linux.readlinkat(dir.handle, sub_path_posix, buffer.ptr, buffer.len);
		3427	switch (linux.errno(rc)) {
		3428	.SUCCESS => {
		3429	const len: usize = @bitCast(rc);
		3430	return len;
		3431	},
		3432	.INTR => continue,
		3433	.ACCES => return error.AccessDenied,
		3434	.FAULT => \|err\| return errnoBug(err),
		3435	.INVAL => return error.NotLink,
		3436	.IO => return error.FileSystem,
		3437	.LOOP => return error.SymLinkLoop,
		3438	.NAMETOOLONG => return error.NameTooLong,
		3439	.NOENT => return error.FileNotFound,
		3440	.NOMEM => return error.SystemResources,
		3441	.NOTDIR => return error.NotDir,
		3442	.ILSEQ => return error.BadPathName,
		3443	else => \|err\| return unexpectedErrno(err),
		3444	}
		3445	}
		3446	}
		3447
		3448	fn dirSetOwner(
		3449	userdata: ?*anyopaque,
		3450	dir: Dir,
		3451	owner: ?File.Uid,
		3452	group: ?File.Gid,
		3453	) Dir.SetOwnerError!void {
		3454	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3455	var cancel_region: CancelRegion = .init();
		3456	defer cancel_region.deinit();
		3457	try ev.fchownat(
		3458	&cancel_region,
		3459	dir.handle,
		3460	"",
		3461	owner orelse std.math.maxInt(linux.uid_t),
		3462	group orelse std.math.maxInt(linux.gid_t),
		3463	linux.AT.EMPTY_PATH,
		3464	);
		3465	}
		3466
		3467	fn dirSetFileOwner(
		3468	userdata: ?*anyopaque,
		3469	dir: Dir,
		3470	sub_path: []const u8,
		3471	owner: ?File.Uid,
		3472	group: ?File.Gid,
		3473	options: Dir.SetFileOwnerOptions,
		3474	) Dir.SetFileOwnerError!void {
		3475	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3476	var path_buffer: [PATH_MAX]u8 = undefined;
		3477	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
		3478	var cancel_region: CancelRegion = .init();
		3479	defer cancel_region.deinit();
		3480	try ev.fchownat(
		3481	&cancel_region,
		3482	dir.handle,
		3483	sub_path_posix,
		3484	owner orelse std.math.maxInt(linux.uid_t),
		3485	group orelse std.math.maxInt(linux.gid_t),
		3486	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
		3487	);
		3488	}
		3489
		3490	fn dirSetPermissions(
		3491	userdata: ?*anyopaque,
		3492	dir: Dir,
		3493	permissions: Dir.Permissions,
		3494	) Dir.SetPermissionsError!void {
		3495	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3496	var cancel_region: CancelRegion = .init();
		3497	defer cancel_region.deinit();
		3498	ev.fchmodat(
		3499	&cancel_region,
		3500	dir.handle,
		3501	"",
		3502	permissions.toMode(),
		3503	linux.AT.EMPTY_PATH,
		3504	) catch \|err\| switch (err) {
		3505	error.NameTooLong => return errnoBug(.NAMETOOLONG),
		3506	error.BadPathName => return errnoBug(.ILSEQ),
		3507	error.ProcessFdQuotaExceeded => return errnoBug(.MFILE),
		3508	error.SystemFdQuotaExceeded => return errnoBug(.NFILE),
		3509	error.OperationUnsupported => return errnoBug(.OPNOTSUPP),
		3510	else => \|e\| return e,
		3511	};
		3512	}
		3513
		3514	fn dirSetFilePermissions(
		3515	userdata: ?*anyopaque,
		3516	dir: Dir,
		3517	sub_path: []const u8,
		3518	permissions: Dir.Permissions,
		3519	options: Dir.SetFilePermissionsOptions,
		3520	) Dir.SetFilePermissionsError!void {
		3521	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3522	var path_buffer: [PATH_MAX]u8 = undefined;
		3523	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
		3524	var cancel_region: CancelRegion = .init();
		3525	defer cancel_region.deinit();
		3526	try ev.fchmodat(
		3527	&cancel_region,
		3528	dir.handle,
		3529	sub_path_posix,
		3530	permissions.toMode(),
		3531	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
		3532	);
		3533	}
		3534
		3535	fn dirSetTimestamps(
		3536	userdata: ?*anyopaque,
		3537	dir: Dir,
		3538	sub_path: []const u8,
		3539	options: Dir.SetTimestampsOptions,
		3540	) Dir.SetTimestampsError!void {
		3541	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3542	var path_buffer: [PATH_MAX]u8 = undefined;
		3543	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
		3544	var cancel_region: CancelRegion = .init();
		3545	defer cancel_region.deinit();
		3546	try ev.utimensat(
		3547	&cancel_region,
		3548	dir.handle,
		3549	sub_path_posix,
		3550	if (options.modify_timestamp != .now or options.access_timestamp != .now) &.{
		3551	setTimestampToPosix(options.access_timestamp),
		3552	setTimestampToPosix(options.modify_timestamp),
		3553	} else null,
		3554	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
		3555	);
		3556	}
		3557
		3558	fn dirHardLink(
		3559	userdata: ?*anyopaque,
		3560	old_dir: Dir,
		3561	old_sub_path: []const u8,
		3562	new_dir: Dir,
		3563	new_sub_path: []const u8,
		3564	options: Dir.HardLinkOptions,
		3565	) Dir.HardLinkError!void {
		3566	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3567
		3568	var old_path_buffer: [PATH_MAX]u8 = undefined;
		3569	var new_path_buffer: [PATH_MAX]u8 = undefined;
		3570
		3571	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
		3572	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
		3573
		3574	var cancel_region: CancelRegion = .init();
		3575	defer cancel_region.deinit();
		3576	return ev.linkat(
		3577	&cancel_region,
		3578	old_dir.handle,
		3579	old_sub_path_posix,
		3580	new_dir.handle,
		3581	new_sub_path_posix,
		3582	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
		3583	);
		3584	}
		3585
		3586	fn fileStat(userdata: ?*anyopaque, file: File) File.StatError!File.Stat {
		3587	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3588	var cancel_region: CancelRegion = .init();
		3589	defer cancel_region.deinit();
		3590	return ev.stat(&cancel_region, file.handle);
		3591	}
		3592
		3593	fn fileLength(userdata: ?*anyopaque, file: File) File.LengthError!u64 {
		3594	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3595	var cancel_region: CancelRegion = .init();
		3596	defer cancel_region.deinit();
		3597	while (true) {
		3598	var statx_buf = std.mem.zeroes(linux.Statx);
		3599	const thread = try cancel_region.awaitIoUring();
		3600	thread.enqueue().* = .{
		3601	.opcode = .STATX,
		3602	.flags = 0,
		3603	.ioprio = 0,
		3604	.fd = file.handle,
		3605	.off = @intFromPtr(&statx_buf),
		3606	.addr = @intFromPtr(""),
		3607	.len = @bitCast(linux.STATX{ .SIZE = true }),
		3608	.rw_flags = linux.AT.EMPTY_PATH,
		3609	.user_data = @intFromPtr(cancel_region.fiber),
		3610	.buf_index = 0,
		3611	.personality = 0,
		3612	.splice_fd_in = 0,
		3613	.addr3 = 0,
		3614	.resv = 0,
		3615	};
		3616	ev.yield(null, .nothing);
		3617	switch (cancel_region.errno()) {
		3618	.SUCCESS => {
		3619	if (!statx_buf.mask.SIZE) return error.Unexpected;
		3620	return statx_buf.size;
		3621	},
		3622	.INTR, .CANCELED => continue,
		3623	.ACCES => \|err\| return errnoBug(err),
		3624	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
		3625	.FAULT => \|err\| return errnoBug(err),
		3626	.INVAL => \|err\| return errnoBug(err),
		3627	.LOOP => \|err\| return errnoBug(err),
		3628	.NAMETOOLONG => \|err\| return errnoBug(err),
		3629	.NOENT => \|err\| return errnoBug(err),
		3630	.NOMEM => return error.SystemResources,
		3631	.NOTDIR => \|err\| return errnoBug(err),
		3632	else => \|err\| return unexpectedErrno(err),
		3633	}
		3634	}
		3635	}
		3636
		3637	fn fileClose(userdata: ?*anyopaque, files: []const File) void {
		3638	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3639	for (files) \|file\| ev.close(file.handle);
		3640	}
		3641
		3642	fn fileWritePositional(
		3643	userdata: ?*anyopaque,
		3644	file: File,
		3645	header: []const u8,
		3646	data: []const []const u8,
		3647	splat: usize,
		3648	offset: u64,
		3649	) File.WritePositionalError!usize {
		3650	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3651
		3652	var iovecs: [max_iovecs_len]iovec_const = undefined;
		3653	var iovlen: iovlen_t = 0;
		3654	addBuf(&iovecs, &iovlen, header);
		3655	for (data[0 .. data.len - 1]) \|bytes\| addBuf(&iovecs, &iovlen, bytes);
		3656	const pattern = data[data.len - 1];
		3657	if (iovecs.len - iovlen != 0) switch (splat) {
		3658	0 => {},
		3659	1 => addBuf(&iovecs, &iovlen, pattern),
		3660	else => switch (pattern.len) {
		3661	0 => {},
		3662	1 => {
		3663	var backup_buffer: [splat_buffer_size]u8 = undefined;
		3664	const splat_buffer = &backup_buffer;
		3665	const memset_len = @min(splat_buffer.len, splat);
		3666	const buf = splat_buffer[0..memset_len];
		3667	@memset(buf, pattern[0]);
		3668	addBuf(&iovecs, &iovlen, buf);
		3669	var remaining_splat = splat - buf.len;
		3670	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0) {
		3671	assert(buf.len == splat_buffer.len);
		3672	addBuf(&iovecs, &iovlen, splat_buffer);
		3673	remaining_splat -= splat_buffer.len;
		3674	}
		3675	addBuf(&iovecs, &iovlen, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
		3676	},
		3677	else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
		3678	addBuf(&iovecs, &iovlen, pattern);
		3679	},
		3680	},
		3681	};
		3682
		3683	var cancel_region: CancelRegion = .init();
		3684	defer cancel_region.deinit();
		3685	return ev.pwritev(&cancel_region, file.handle, iovecs[0..iovlen], offset);
		3686	}
		3687
		3688	/// This is either usize or u32. Since, either is fine, let's use the same
		3689	/// `addBuf` function for both writing to a file and sending network messages.
		3690	const iovlen_t = @FieldType(linux.msghdr_const, "iovlen");
		3691
		3692	fn addBuf(v: []iovec_const, i: *iovlen_t, bytes: []const u8) void {
		3693	// OS checks ptr addr before length so zero length vectors must be omitted.
		3694	if (bytes.len == 0) return;
		3695	if (v.len - i.* == 0) return;
		3696	v[i.*] = .{ .base = bytes.ptr, .len = bytes.len };
		3697	i.* += 1;
		3698	}
		3699
		3700	fn fileWriteFileStreaming(
		3701	userdata: ?*anyopaque,
		3702	file: File,
		3703	header: []const u8,
		3704	file_reader: *File.Reader,
		3705	limit: Io.Limit,
		3706	) File.Writer.WriteFileError!usize {
		3707	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3708	_ = ev;
		3709	_ = file;
		3710	_ = header;
		3711	_ = file_reader;
		3712	_ = limit;
		3713	return error.Unimplemented;
		3714	}
		3715
		3716	fn fileWriteFilePositional(
		3717	userdata: ?*anyopaque,
		3718	file: File,
		3719	header: []const u8,
		3720	file_reader: *File.Reader,
		3721	limit: Io.Limit,
		3722	offset: u64,
		3723	) File.WriteFilePositionalError!usize {
		3724	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3725	_ = ev;
		3726	_ = file;
		3727	_ = header;
		3728	_ = file_reader;
		3729	_ = limit;
		3730	_ = offset;
		3731	return error.Unimplemented;
		3732	}
		3733
		3734	fn fileReadPositional(
		3735	userdata: ?*anyopaque,
		3736	file: File,
		3737	data: []const []u8,
		3738	offset: u64,
		3739	) File.ReadPositionalError!usize {
		3740	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3741
		3742	var iovecs_buffer: [max_iovecs_len]iovec = undefined;
		3743	var i: usize = 0;
		3744	for (data) \|buf\| {
		3745	if (iovecs_buffer.len - i == 0) break;
		3746	if (buf.len != 0) {
		3747	iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len };
		3748	i += 1;
		3749	}
		3750	}
		3751	if (i == 0) return 0;
		3752	const dest = iovecs_buffer[0..i];
		3753	assert(dest[0].len > 0);
		3754
		3755	var cancel_region: CancelRegion = .init();
		3756	defer cancel_region.deinit();
		3757	return ev.preadv(&cancel_region, file.handle, dest, offset) catch \|err\| switch (err) {
		3758	error.SocketUnconnected => errnoBug(.NOTCONN), // not a socket
		3759	error.ConnectionResetByPeer => errnoBug(.CONNRESET), // not a socket
		3760	else => \|e\| e,
		3761	};
		3762	}
		3763
		3764	fn fileSeekBy(userdata: ?*anyopaque, file: File, offset: i64) File.SeekError!void {
		3765	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3766	var cancel_region: CancelRegion = .init();
		3767	defer cancel_region.deinit();
		3768	try ev.lseek(&cancel_region, file.handle, @bitCast(offset), linux.SEEK.CUR);
		3769	}
		3770
		3771	fn fileSeekTo(userdata: ?*anyopaque, file: File, offset: u64) File.SeekError!void {
		3772	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3773	var cancel_region: CancelRegion = .init();
		3774	defer cancel_region.deinit();
		3775	try ev.lseek(&cancel_region, file.handle, offset, linux.SEEK.SET);
		3776	}
		3777
		3778	fn fileSync(userdata: ?*anyopaque, file: File) File.SyncError!void {
		3779	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3780	var cancel_region: CancelRegion = .init();
		3781	defer cancel_region.deinit();
		3782	while (true) {
		3783	const thread = try cancel_region.awaitIoUring();
		3784	thread.enqueue().* = .{
		3785	.opcode = .FSYNC,
		3786	.flags = 0,
		3787	.ioprio = 0,
		3788	.fd = file.handle,
		3789	.off = 0,
		3790	.addr = 0,
		3791	.len = 0,
		3792	.rw_flags = 0,
		3793	.user_data = @intFromPtr(cancel_region.fiber),
		3794	.buf_index = 0,
		3795	.personality = 0,
		3796	.splice_fd_in = 0,
		3797	.addr3 = 0,
		3798	.resv = 0,
		3799	};
		3800	ev.yield(null, .nothing);
		3801	switch (cancel_region.errno()) {
		3802	.SUCCESS => return,
		3803	.INTR, .CANCELED => continue,
		3804	.BADF => \|err\| return errnoBug(err),
		3805	.INVAL => \|err\| return errnoBug(err),
		3806	.ROFS => \|err\| return errnoBug(err),
		3807	.IO => return error.InputOutput,
		3808	.NOSPC => return error.NoSpaceLeft,
		3809	.DQUOT => return error.DiskQuota,
		3810	else => \|err\| return unexpectedErrno(err),
		3811	}
		3812	}
		3813	}
		3814
		3815	fn fileIsTty(userdata: ?*anyopaque, file: File) Io.Cancelable!bool {
		3816	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3817	_ = ev;
		3818	var cancel_region: CancelRegion = .init();
		3819	defer cancel_region.deinit();
		3820	while (true) {
		3821	try cancel_region.await(.nothing);
		3822	var wsz: winsize = undefined;
		3823	const fd: usize = @bitCast(@as(isize, file.handle));
		3824	const rc = linux.syscall3(.ioctl, fd, linux.T.IOCGWINSZ, @intFromPtr(&wsz));
		3825	switch (linux.errno(rc)) {
		3826	.SUCCESS => return true,
		3827	.INTR => continue,
		3828	else => return false,
		3829	}
		3830	}
		3831	}
		3832
		3833	fn fileEnableAnsiEscapeCodes(userdata: ?*anyopaque, file: File) File.EnableAnsiEscapeCodesError!void {
		3834	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3835	if (!try fileIsTty(ev, file)) return error.NotTerminalDevice;
		3836	}
		3837
		3838	fn fileSetLength(userdata: ?*anyopaque, file: File, length: u64) File.SetLengthError!void {
		3839	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3840	var cancel_region: CancelRegion = .init();
		3841	defer cancel_region.deinit();
		3842	while (true) {
		3843	const thread = try cancel_region.awaitIoUring();
		3844	thread.enqueue().* = .{
		3845	.opcode = .FTRUNCATE,
		3846	.flags = 0,
		3847	.ioprio = 0,
		3848	.fd = file.handle,
		3849	.off = length,
		3850	.addr = 0,
		3851	.len = 0,
		3852	.rw_flags = 0,
		3853	.user_data = @intFromPtr(cancel_region.fiber),
		3854	.buf_index = 0,
		3855	.personality = 0,
		3856	.splice_fd_in = 0,
		3857	.addr3 = 0,
		3858	.resv = 0,
		3859	};
		3860	ev.yield(null, .nothing);
		3861	switch (cancel_region.errno()) {
		3862	.SUCCESS => return,
		3863	.INTR, .CANCELED => continue,
		3864	.FBIG => return error.FileTooBig,
		3865	.IO => return error.InputOutput,
		3866	.PERM => return error.PermissionDenied,
		3867	.TXTBSY => return error.FileBusy,
		3868	.BADF => \|err\| return errnoBug(err), // Handle not open for writing.
		3869	.INVAL => return error.NonResizable, // This is returned for /dev/null for example.
		3870	else => \|err\| return unexpectedErrno(err),
		3871	}
		3872	}
		3873	}
		3874
		3875	fn fileSetOwner(
		3876	userdata: ?*anyopaque,
		3877	file: File,
		3878	owner: ?File.Uid,
		3879	group: ?File.Gid,
		3880	) File.SetOwnerError!void {
		3881	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3882	var cancel_region: CancelRegion = .init();
		3883	defer cancel_region.deinit();
		3884	try ev.fchownat(
		3885	&cancel_region,
		3886	file.handle,
		3887	"",
		3888	owner orelse std.math.maxInt(linux.uid_t),
		3889	group orelse std.math.maxInt(linux.gid_t),
		3890	linux.AT.EMPTY_PATH,
		3891	);
		3892	}
		3893
		3894	fn fileSetPermissions(
		3895	userdata: ?*anyopaque,
		3896	file: File,
		3897	permissions: File.Permissions,
		3898	) File.SetPermissionsError!void {
		3899	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3900	var cancel_region: CancelRegion = .init();
		3901	defer cancel_region.deinit();
		3902	ev.fchmodat(
		3903	&cancel_region,
		3904	file.handle,
		3905	"",
		3906	permissions.toMode(),
		3907	linux.AT.EMPTY_PATH,
		3908	) catch \|err\| switch (err) {
		3909	error.NameTooLong => return errnoBug(.NAMETOOLONG),
		3910	error.BadPathName => return errnoBug(.ILSEQ),
		3911	error.ProcessFdQuotaExceeded => return errnoBug(.MFILE),
		3912	error.SystemFdQuotaExceeded => return errnoBug(.NFILE),
		3913	error.OperationUnsupported => return errnoBug(.OPNOTSUPP),
		3914	else => \|e\| return e,
		3915	};
		3916	}
		3917
		3918	fn fileSetTimestamps(
		3919	userdata: ?*anyopaque,
		3920	file: File,
		3921	options: File.SetTimestampsOptions,
		3922	) File.SetTimestampsError!void {
		3923	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3924	var cancel_region: CancelRegion = .init();
		3925	defer cancel_region.deinit();
		3926	try ev.utimensat(
		3927	&cancel_region,
		3928	file.handle,
		3929	"",
		3930	if (options.modify_timestamp != .now or options.access_timestamp != .now) &.{
		3931	setTimestampToPosix(options.access_timestamp),
		3932	setTimestampToPosix(options.modify_timestamp),
		3933	} else null,
		3934	linux.AT.EMPTY_PATH,
		3935	);
		3936	}
		3937
		3938	fn fileLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!void {
		3939	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3940	var cancel_region: CancelRegion = .init();
		3941	defer cancel_region.deinit();
		3942	ev.flock(&cancel_region, file.handle, lock, .blocking) catch \|err\| switch (err) {
		3943	error.WouldBlock => unreachable, // blocking
		3944	else => \|e\| return e,
		3945	};
		3946	}
		3947
		3948	fn fileTryLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!bool {
		3949	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3950	var cancel_region: CancelRegion = .init();
		3951	defer cancel_region.deinit();
		3952	ev.flock(&cancel_region, file.handle, lock, switch (lock) {
		3953	.none => .blocking,
		3954	.shared, .exclusive => .nonblocking,
		3955	}) catch \|err\| switch (err) {
		3956	error.WouldBlock => return false,
		3957	else => \|e\| return e,
		3958	};
		3959	return true;
		3960	}
		3961
		3962	fn fileUnlock(userdata: ?*anyopaque, file: File) void {
		3963	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3964	var cancel_region: CancelRegion = .initBlocked();
		3965	defer cancel_region.deinit();
		3966	ev.flock(&cancel_region, file.handle, .none, .blocking) catch \|err\| switch (err) {
		3967	error.Canceled => unreachable, // blocked
		3968	error.WouldBlock => unreachable, // blocking
		3969	error.SystemResources => return recoverableOsBugDetected(), // Resource deallocation.
		3970	error.FileLocksUnsupported => return recoverableOsBugDetected(), // We already got the lock.
		3971	error.Unexpected => return recoverableOsBugDetected(), // Resource deallocation must succeed.
		3972	};
		3973	}
		3974
		3975	fn fileDowngradeLock(userdata: ?*anyopaque, file: File) File.DowngradeLockError!void {
		3976	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3977	var cancel_region: CancelRegion = .init();
		3978	defer cancel_region.deinit();
		3979	ev.flock(&cancel_region, file.handle, .shared, .nonblocking) catch \|err\| switch (err) {
		3980	error.WouldBlock => return errnoBug(.AGAIN), // File was not locked in exclusive mode.
		3981	error.SystemResources => return errnoBug(.NOLCK), // Lock already obtained.
		3982	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Lock already obtained.
		3983	else => \|e\| return e,
		3984	};
		3985	}
		3986
		3987	fn fileRealPath(userdata: ?*anyopaque, file: File, out_buffer: []u8) File.RealPathError!usize {
		3988	const ev: *Evented = @ptrCast(@alignCast(userdata));
		3989	var cancel_region: CancelRegion = .init();
		3990	defer cancel_region.deinit();
		3991	return ev.realPath(&cancel_region, file.handle, out_buffer);
		3992	}
		3993
		3994	fn fileHardLink(
		3995	userdata: ?*anyopaque,
		3996	file: File,
		3997	new_dir: Dir,
		3998	new_sub_path: []const u8,
		3999	options: File.HardLinkOptions,
		4000	) File.HardLinkError!void {
		4001	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4002
		4003	var new_path_buffer: [PATH_MAX]u8 = undefined;
		4004	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
		4005
		4006	var cancel_region: CancelRegion = .init();
		4007	defer cancel_region.deinit();
		4008	return ev.linkat(
		4009	&cancel_region,
		4010	file.handle,
		4011	"",
		4012	new_dir.handle,
		4013	new_sub_path_posix,
		4014	linux.AT.EMPTY_PATH \| @as(u32, if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW),
		4015	);
		4016	}
		4017
		4018	fn fileMemoryMapCreate(
		4019	userdata: ?*anyopaque,
		4020	file: File,
		4021	options: File.MemoryMap.CreateOptions,
		4022	) File.MemoryMap.CreateError!File.MemoryMap {
		4023	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4024	_ = ev;
		4025	const prot: linux.PROT = .{
		4026	.READ = options.protection.read,
		4027	.WRITE = options.protection.write,
		4028	.EXEC = options.protection.execute,
		4029	};
		4030	const flags: linux.MAP = .{
		4031	.TYPE = .SHARED_VALIDATE,
		4032	.POPULATE = options.populate,
		4033	};
		4034
		4035	const page_align = std.heap.page_size_min;
		4036
		4037	var cancel_region: CancelRegion = .init();
		4038	defer cancel_region.deinit();
		4039	const contents = while (true) {
		4040	try cancel_region.await(.nothing);
		4041	const casted_offset = std.math.cast(i64, options.offset) orelse return error.Unseekable;
		4042	const rc = linux.mmap(null, options.len, prot, flags, file.handle, casted_offset);
		4043	switch (linux.errno(rc)) {
		4044	.SUCCESS => break @as([*]align(page_align) u8, @ptrFromInt(rc))[0..options.len],
		4045	.INTR => continue,
		4046	.ACCES => return error.AccessDenied,
		4047	.AGAIN => return error.LockedMemoryLimitExceeded,
		4048	.MFILE => return error.ProcessFdQuotaExceeded,
		4049	.NFILE => return error.SystemFdQuotaExceeded,
		4050	.NOMEM => return error.OutOfMemory,
		4051	.PERM => return error.PermissionDenied,
		4052	.OVERFLOW => return error.Unseekable,
		4053	.BADF => \|err\| return errnoBug(err), // Always a race condition.
		4054	.INVAL => \|err\| return errnoBug(err), // Invalid parameters to mmap()
		4055	.OPNOTSUPP => \|err\| return errnoBug(err), // Bad flags with MAP.SHARED_VALIDATE on Linux.
		4056	else => \|err\| return unexpectedErrno(err),
		4057	}
		4058	};
		4059	return .{
		4060	.file = file,
		4061	.offset = options.offset,
		4062	.memory = contents,
		4063	.section = {},
		4064	};
		4065	}
		4066
		4067	fn fileMemoryMapDestroy(userdata: ?anyopaque, mm: File.MemoryMap) void {
		4068	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4069	_ = ev;
		4070	const memory = mm.memory;
		4071	if (memory.len == 0) return;
		4072	switch (linux.errno(linux.munmap(memory.ptr, memory.len))) {
		4073	.SUCCESS => {},
		4074	else => \|err\| if (builtin.mode == .Debug)
		4075	std.log.err("failed to unmap {d} bytes at {*}: {t}", .{ memory.len, memory.ptr, err }),
		4076	}
		4077	mm.* = undefined;
		4078	}
		4079
		4080	fn processExecutableOpen(
		4081	userdata: ?*anyopaque,
		4082	flags: File.OpenFlags,
		4083	) process.OpenExecutableError!File {
		4084	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4085	return dirOpenFile(ev, .{ .handle = linux.AT.FDCWD }, "/proc/self/exe", flags);
		4086	}
		4087
		4088	fn processExecutablePath(userdata: ?*anyopaque, out_buffer: []u8) process.ExecutablePathError!usize {
		4089	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4090	return dirReadLink(ev, .cwd(), "/proc/self/exe", out_buffer) catch \|err\| switch (err) {
		4091	error.UnsupportedReparsePointType => unreachable, // Windows-only
		4092	error.NetworkNotFound => unreachable, // Windows-only
		4093	error.FileBusy => unreachable, // Windows-only
		4094	else => \|e\| return e,
		4095	};
		4096	}
		4097
		4098	fn lockStderr(userdata: ?*anyopaque, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
		4099	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4100	const ev_io = ev.io();
		4101	ev.stderr_mutex.lockUncancelable(ev_io);
		4102	errdefer ev.stderr_mutex.unlock(ev_io);
		4103	return ev.initLockedStderr(terminal_mode);
		4104	}
		4105
		4106	fn tryLockStderr(
		4107	userdata: ?*anyopaque,
		4108	terminal_mode: ?Io.Terminal.Mode,
		4109	) Io.Cancelable!?Io.LockedStderr {
		4110	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4111	const ev_io = ev.io();
		4112	if (!ev.stderr_mutex.tryLock()) return null;
		4113	errdefer ev.stderr_mutex.unlock(ev_io);
		4114	return try ev.initLockedStderr(terminal_mode);
		4115	}
		4116
		4117	fn initLockedStderr(ev: *Evented, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
		4118	if (!ev.stderr_writer_initialized) {
		4119	const ev_io = ev.io();
		4120	try ev.scanEnviron();
		4121	const NO_COLOR = ev.environ.exist.NO_COLOR;
		4122	const CLICOLOR_FORCE = ev.environ.exist.CLICOLOR_FORCE;
		4123	ev.stderr_mode = terminal_mode orelse
		4124	try .detect(ev_io, ev.stderr_writer.file, NO_COLOR, CLICOLOR_FORCE);
		4125	ev.stderr_writer_initialized = true;
		4126	}
		4127	return .{
		4128	.file_writer = &ev.stderr_writer,
		4129	.terminal_mode = terminal_mode orelse ev.stderr_mode,
		4130	};
		4131	}
		4132
		4133	fn unlockStderr(userdata: ?*anyopaque) void {
		4134	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4135	ev.stderr_writer.interface.flush() catch \|err\| switch (err) {
		4136	error.WriteFailed => switch (ev.stderr_writer.err.?) {
		4137	error.Canceled => recancel(ev),
		4138	else => {},
		4139	},
		4140	};
		4141	ev.stderr_writer.interface.end = 0;
		4142	ev.stderr_writer.interface.buffer = &.{};
		4143	ev.stderr_mutex.unlock(ev.io());
		4144	}
		4145
		4146	fn processCurrentPath(userdata: ?*anyopaque, buffer: []u8) process.CurrentPathError!usize {
		4147	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4148	_ = ev;
		4149	var cancel_region: CancelRegion = .init();
		4150	defer cancel_region.deinit();
		4151	while (true) {
		4152	try cancel_region.await(.nothing);
		4153	switch (linux.errno(linux.getcwd(buffer.ptr, buffer.len))) {
		4154	.SUCCESS => return std.mem.findScalar(u8, buffer, 0).?,
		4155	.INTR => continue,
		4156	.NOENT => return error.CurrentDirUnlinked,
		4157	.RANGE => return error.NameTooLong,
		4158	.FAULT => \|err\| return errnoBug(err),
		4159	.INVAL => \|err\| return errnoBug(err),
		4160	else => \|err\| return unexpectedErrno(err),
		4161	}
		4162	}
		4163	}
		4164
		4165	fn processSetCurrentDir(userdata: ?*anyopaque, dir: Dir) process.SetCurrentDirError!void {
		4166	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4167	_ = ev;
		4168	if (dir.handle == linux.AT.FDCWD) return;
		4169	var cancel_region: CancelRegion = .init();
		4170	defer cancel_region.deinit();
		4171	while (true) {
		4172	try cancel_region.await(.nothing);
		4173	switch (linux.errno(linux.fchdir(dir.handle))) {
		4174	.SUCCESS => return,
		4175	.INTR => continue,
		4176	.ACCES => return error.AccessDenied,
		4177	.NOTDIR => return error.NotDir,
		4178	.IO => return error.FileSystem,
		4179	.BADF => \|err\| return errnoBug(err),
		4180	else => \|err\| return unexpectedErrno(err),
		4181	}
		4182	}
		4183	}
		4184
		4185	fn processSetCurrentPath(userdata: ?*anyopaque, dir_path: []const u8) ChdirError!void {
		4186	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4187	_ = ev;
		4188	var path_buffer: [PATH_MAX]u8 = undefined;
		4189	const dir_path_posix = try pathToPosix(dir_path, &path_buffer);
		4190	var cancel_region: CancelRegion = .init();
		4191	defer cancel_region.deinit();
		4192	while (true) {
		4193	try cancel_region.await(.nothing);
		4194	switch (linux.errno(linux.chdir(dir_path_posix))) {
		4195	.SUCCESS => return,
		4196	.INTR => continue,
		4197	.ACCES => return error.AccessDenied,
		4198	.IO => return error.FileSystem,
		4199	.LOOP => return error.SymLinkLoop,
		4200	.NAMETOOLONG => return error.NameTooLong,
		4201	.NOENT => return error.FileNotFound,
		4202	.NOMEM => return error.SystemResources,
		4203	.NOTDIR => return error.NotDir,
		4204	.ILSEQ => return error.BadPathName,
		4205	.FAULT => \|err\| return errnoBug(err),
		4206	else => \|err\| return unexpectedErrno(err),
		4207	}
		4208	}
		4209	}
		4210
		4211	fn processReplace(userdata: ?*anyopaque, options: process.ReplaceOptions) process.ReplaceError {
		4212	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4213
		4214	try ev.scanEnviron(); // for PATH
		4215	const PATH = ev.environ.string.PATH orelse default_PATH;
		4216
		4217	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
		4218	defer arena_allocator.deinit();
		4219	const arena = arena_allocator.allocator();
		4220
		4221	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
		4222	for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
		4223
		4224	const env_block = env_block: {
		4225	const prog_fd: i32 = -1;
		4226	if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
		4227	.zig_progress_fd = prog_fd,
		4228	});
		4229	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
		4230	.zig_progress_fd = prog_fd,
		4231	});
		4232	};
		4233
		4234	return execv(options.expand_arg0, argv_buf.ptr[0].?, argv_buf.ptr, env_block, PATH);
		4235	}
		4236
		4237	fn processReplacePath(
		4238	userdata: ?*anyopaque,
		4239	dir: Dir,
		4240	options: process.ReplaceOptions,
		4241	) process.ReplaceError {
		4242	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4243	_ = ev;
		4244	_ = dir;
		4245	_ = options;
		4246	@panic("TODO processReplacePath");
		4247	}
		4248
		4249	fn processSpawn(userdata: ?*anyopaque, options: process.SpawnOptions) process.SpawnError!process.Child {
		4250	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4251	const spawned = try ev.spawn(options);
		4252	defer ev.close(spawned.err_fd);
		4253
		4254	// Wait for the child to report any errors in or before `execvpe`.
		4255	var child_err: ForkBailError = undefined;
		4256	var cancel_region: CancelRegion = .initBlocked();
		4257	defer cancel_region.deinit();
		4258	ev.readAll(&cancel_region, spawned.err_fd, @ptrCast(&child_err)) catch \|read_err\| {
		4259	switch (read_err) {
		4260	error.Canceled => unreachable, // blocked
		4261	error.EndOfStream => {
		4262	// Write end closed by CLOEXEC at the time of the `execvpe` call,
		4263	// indicating success.
		4264	},
		4265	else => {
		4266	// Problem reading the error from the error reporting pipe. We
		4267	// don't know if the child is alive or dead. Better to assume it is
		4268	// alive so the resource does not risk being leaked.
		4269	},
		4270	}
		4271	return .{
		4272	.id = spawned.pid,
		4273	.thread_handle = {},
		4274	.stdin = spawned.stdin,
		4275	.stdout = spawned.stdout,
		4276	.stderr = spawned.stderr,
		4277	.request_resource_usage_statistics = options.request_resource_usage_statistics,
		4278	};
		4279	};
		4280	return child_err;
		4281	}
		4282
		4283	fn processSpawnPath(
		4284	userdata: ?*anyopaque,
		4285	dir: Dir,
		4286	options: process.SpawnOptions,
		4287	) process.SpawnError!process.Child {
		4288	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4289	_ = ev;
		4290	_ = dir;
		4291	_ = options;
		4292	@panic("TODO processSpawnPath");
		4293	}
		4294
		4295	const Spawned = struct {
		4296	pid: pid_t,
		4297	err_fd: fd_t,
		4298	stdin: ?File,
		4299	stdout: ?File,
		4300	stderr: ?File,
		4301	};
		4302	fn spawn(ev: *Evented, options: process.SpawnOptions) process.SpawnError!Spawned {
		4303	// The child process does need to access (one end of) these pipes. However,
		4304	// we must initially set CLOEXEC to avoid a race condition. If another thread
		4305	// is racing to spawn a different child process, we don't want it to inherit
		4306	// these FDs in any scenario; that would mean that, for instance, calls to
		4307	// `poll` from the parent would not report the child's stdout as closing when
		4308	// expected, since the other child may retain a reference to the write end of
		4309	// the pipe. So, we create the pipes with CLOEXEC initially. After fork, we
		4310	// need to do something in the new child to make sure we preserve the reference
		4311	// we want. We could use `fcntl` to remove CLOEXEC from the FD, but as it
		4312	// turns out, we `dup2` everything anyway, so there's no need!
		4313	const pipe_flags: linux.O = .{ .CLOEXEC = true };
		4314
		4315	const stdin_pipe = if (options.stdin == .pipe) try pipe2(pipe_flags) else undefined;
		4316	errdefer if (options.stdin == .pipe) {
		4317	ev.destroyPipe(stdin_pipe);
		4318	};
		4319
		4320	const stdout_pipe = if (options.stdout == .pipe) try pipe2(pipe_flags) else undefined;
		4321	errdefer if (options.stdout == .pipe) {
		4322	ev.destroyPipe(stdout_pipe);
		4323	};
		4324
		4325	const stderr_pipe = if (options.stderr == .pipe) try pipe2(pipe_flags) else undefined;
		4326	errdefer if (options.stderr == .pipe) {
		4327	ev.destroyPipe(stderr_pipe);
		4328	};
		4329
		4330	const any_ignore =
		4331	options.stdin == .ignore or options.stdout == .ignore or options.stderr == .ignore;
		4332	const dev_null_fd = if (any_ignore) dev_null_fd: {
		4333	var cancel_region: CancelRegion = .init();
		4334	defer cancel_region.deinit();
		4335	break :dev_null_fd try ev.null_fd.open(ev, &cancel_region, "/dev/null", .{
		4336	.ACCMODE = .RDWR,
		4337	});
		4338	} else undefined;
		4339
		4340	const prog_pipe: [2]fd_t = if (options.progress_node.index != .none) pipe: {
		4341	// We use CLOEXEC for the same reason as in `pipe_flags`.
		4342	const pipe = try pipe2(.{ .NONBLOCK = true, .CLOEXEC = true });
		4343	_ = linux.fcntl(pipe[0], linux.F.SETPIPE_SZ, @as(u32, std.Progress.max_packet_len * 2));
		4344	break :pipe pipe;
		4345	} else .{ -1, -1 };
		4346	errdefer ev.destroyPipe(prog_pipe);
		4347
		4348	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
		4349	defer arena_allocator.deinit();
		4350	const arena = arena_allocator.allocator();
		4351
		4352	// The POSIX standard does not allow malloc() between fork() and execve(),
		4353	// and this allocator may be a libc allocator.
		4354	// I have personally observed the child process deadlocking when it tries
		4355	// to call malloc() due to a heap allocation between fork() and execve(),
		4356	// in musl v1.1.24.
		4357	// Additionally, we want to reduce the number of possible ways things
		4358	// can fail between fork() and execve().
		4359	// Therefore, we do all the allocation for the execve() before the fork().
		4360	// This means we must do the null-termination of argv and env vars here.
		4361	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
		4362	for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
		4363
		4364	const prog_fileno = 3;
		4365	comptime assert(@max(linux.STDIN_FILENO, linux.STDOUT_FILENO, linux.STDERR_FILENO) + 1 == prog_fileno);
		4366
		4367	const env_block = env_block: {
		4368	const prog_fd: i32 = if (prog_pipe[1] == -1) -1 else prog_fileno;
		4369	if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
		4370	.zig_progress_fd = prog_fd,
		4371	});
		4372	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
		4373	.zig_progress_fd = prog_fd,
		4374	});
		4375	};
		4376
		4377	// This pipe communicates to the parent errors in the child between `fork` and `execvpe`.
		4378	// It is closed by the child (via CLOEXEC) without writing if `execvpe` succeeds.
		4379	const err_pipe: [2]fd_t = try pipe2(.{ .CLOEXEC = true });
		4380	errdefer ev.destroyPipe(err_pipe);
		4381
		4382	try ev.scanEnviron(); // for PATH
		4383	const PATH = ev.environ.string.PATH orelse default_PATH;
		4384
		4385	const pid_result: pid_t = fork: {
		4386	const rc = linux.fork();
		4387	switch (linux.errno(rc)) {
		4388	.SUCCESS => break :fork @intCast(rc),
		4389	.AGAIN => return error.SystemResources,
		4390	.NOMEM => return error.SystemResources,
		4391	.NOSYS => return error.OperationUnsupported,
		4392	else => \|err\| return unexpectedErrno(err),
		4393	}
		4394	};
		4395
		4396	if (pid_result == 0) {
		4397	defer comptime unreachable; // We are the child.
		4398	_ = swapCancelProtection(ev, .blocked);
		4399	const ep1 = err_pipe[1];
		4400
		4401	ev.setUpChildIo(options.stdin, stdin_pipe[0], linux.STDIN_FILENO, dev_null_fd) catch \|err\|
		4402	ev.forkBail(ep1, err);
		4403	ev.setUpChildIo(options.stdout, stdout_pipe[1], linux.STDOUT_FILENO, dev_null_fd) catch \|err\|
		4404	ev.forkBail(ep1, err);
		4405	ev.setUpChildIo(options.stderr, stderr_pipe[1], linux.STDERR_FILENO, dev_null_fd) catch \|err\|
		4406	ev.forkBail(ep1, err);
		4407
		4408	switch (options.cwd) {
		4409	.inherit => {},
		4410	.dir => \|cwd\| processSetCurrentDir(ev, cwd) catch \|err\| ev.forkBail(ep1, err),
		4411	.path => \|cwd\| processSetCurrentPath(ev, cwd) catch \|err\| ev.forkBail(ep1, err),
		4412	}
		4413
		4414	// Must happen after fchdir above, the cwd file descriptor might be
		4415	// equal to prog_fileno and be clobbered by this dup2 call.
		4416	if (prog_pipe[1] != -1) dup2(prog_pipe[1], prog_fileno) catch \|err\| ev.forkBail(ep1, err);
		4417
		4418	if (options.gid) \|gid\| {
		4419	switch (linux.errno(linux.setregid(gid, gid))) {
		4420	.SUCCESS => {},
		4421	.AGAIN => ev.forkBail(ep1, error.ResourceLimitReached),
		4422	.INVAL => ev.forkBail(ep1, error.InvalidUserId),
		4423	.PERM => ev.forkBail(ep1, error.PermissionDenied),
		4424	else => ev.forkBail(ep1, error.Unexpected),
		4425	}
		4426	}
		4427
		4428	if (options.uid) \|uid\| {
		4429	switch (linux.errno(linux.setreuid(uid, uid))) {
		4430	.SUCCESS => {},
		4431	.AGAIN => ev.forkBail(ep1, error.ResourceLimitReached),
		4432	.INVAL => ev.forkBail(ep1, error.InvalidUserId),
		4433	.PERM => ev.forkBail(ep1, error.PermissionDenied),
		4434	else => ev.forkBail(ep1, error.Unexpected),
		4435	}
		4436	}
		4437
		4438	if (options.pgid) \|pid\| {
		4439	switch (linux.errno(linux.setpgid(0, pid))) {
		4440	.SUCCESS => {},
		4441	.ACCES => ev.forkBail(ep1, error.ProcessAlreadyExec),
		4442	.INVAL => ev.forkBail(ep1, error.InvalidProcessGroupId),
		4443	.PERM => ev.forkBail(ep1, error.PermissionDenied),
		4444	else => ev.forkBail(ep1, error.Unexpected),
		4445	}
		4446	}
		4447
		4448	if (options.start_suspended) {
		4449	switch (linux.errno(linux.kill(linux.getpid(), .STOP))) {
		4450	.SUCCESS => {},
		4451	.PERM => ev.forkBail(ep1, error.PermissionDenied),
		4452	else => ev.forkBail(ep1, error.Unexpected),
		4453	}
		4454	}
		4455
		4456	const err = execv(options.expand_arg0, argv_buf.ptr[0].?, argv_buf.ptr, env_block, PATH);
		4457	ev.forkBail(ep1, err);
		4458	}
		4459
		4460	const pid: pid_t = @intCast(pid_result); // We are the parent.
		4461	errdefer comptime unreachable; // The child is forked; we must not error from now on
		4462
		4463	ev.close(err_pipe[1]); // make sure only the child holds the write end open
		4464
		4465	if (options.stdin == .pipe) ev.close(stdin_pipe[0]);
		4466	if (options.stdout == .pipe) ev.close(stdout_pipe[1]);
		4467	if (options.stderr == .pipe) ev.close(stderr_pipe[1]);
		4468
		4469	if (prog_pipe[1] != -1) ev.close(prog_pipe[1]);
		4470
		4471	options.progress_node.setIpcFile(ev, .{ .handle = prog_pipe[0], .flags = .{ .nonblocking = true } });
		4472
		4473	return .{
		4474	.pid = pid,
		4475	.err_fd = err_pipe[0],
		4476	.stdin = switch (options.stdin) {
		4477	.pipe => .{ .handle = stdin_pipe[1], .flags = .{ .nonblocking = false } },
		4478	else => null,
		4479	},
		4480	.stdout = switch (options.stdout) {
		4481	.pipe => .{ .handle = stdout_pipe[0], .flags = .{ .nonblocking = false } },
		4482	else => null,
		4483	},
		4484	.stderr = switch (options.stderr) {
		4485	.pipe => .{ .handle = stderr_pipe[0], .flags = .{ .nonblocking = false } },
		4486	else => null,
		4487	},
		4488	};
		4489	}
		4490
		4491	pub const PipeError = error{
		4492	SystemFdQuotaExceeded,
		4493	ProcessFdQuotaExceeded,
		4494	} \|\| Io.UnexpectedError;
		4495	pub fn pipe2(flags: linux.O) PipeError![2]fd_t {
		4496	var fds: [2]fd_t = undefined;
		4497	switch (linux.errno(linux.pipe2(&fds, flags))) {
		4498	.SUCCESS => return fds,
		4499	.INVAL => \|err\| return errnoBug(err), // Invalid flags
		4500	.NFILE => return error.SystemFdQuotaExceeded,
		4501	.MFILE => return error.ProcessFdQuotaExceeded,
		4502	else => \|err\| return unexpectedErrno(err),
		4503	}
		4504	}
		4505	fn destroyPipe(ev: *Evented, pipe: [2]fd_t) void {
		4506	if (pipe[0] != -1) ev.close(pipe[0]);
		4507	if (pipe[0] != pipe[1]) ev.close(pipe[1]);
		4508	}
		4509
		4510	fn setUpChildIo(
		4511	ev: *Evented,
		4512	stdio: process.SpawnOptions.StdIo,
		4513	pipe_fd: fd_t,
		4514	std_fileno: i32,
		4515	dev_null_fd: fd_t,
		4516	) !void {
		4517	switch (stdio) {
		4518	.pipe => try dup2(pipe_fd, std_fileno),
		4519	.close => ev.close(std_fileno),
		4520	.inherit => {},
		4521	.ignore => try dup2(dev_null_fd, std_fileno),
		4522	.file => \|file\| try dup2(file.handle, std_fileno),
		4523	}
		4524	}
		4525
		4526	pub const DupError = error{
		4527	ProcessFdQuotaExceeded,
		4528	SystemResources,
		4529	} \|\| Io.UnexpectedError \|\| Io.Cancelable;
		4530	pub fn dup2(old_fd: fd_t, new_fd: fd_t) DupError!void {
		4531	var cancel_region: CancelRegion = .init();
		4532	defer cancel_region.deinit();
		4533	while (true) {
		4534	try cancel_region.await(.nothing);
		4535	switch (linux.errno(linux.dup2(old_fd, new_fd))) {
		4536	.SUCCESS => {},
		4537	.BUSY, .INTR => continue,
		4538	.INVAL => \|err\| return errnoBug(err), // invalid parameters
		4539	.BADF => \|err\| return errnoBug(err), // use after free
		4540	.MFILE => return error.ProcessFdQuotaExceeded,
		4541	.NOMEM => return error.SystemResources,
		4542	else => \|err\| return unexpectedErrno(err),
		4543	}
		4544	}
		4545	}
		4546
		4547	/// Errors that can occur between fork() and execv()
		4548	const ForkBailError = process.SetCurrentDirError \|\| ChdirError \|\|
		4549	process.SpawnError \|\| process.ReplaceError;
		4550	/// Child of fork calls this to report an error to the fork parent. Then the
		4551	/// child exits.
		4552	fn forkBail(ev: *Evented, fd: fd_t, err: ForkBailError) noreturn {
		4553	var cancel_region: CancelRegion = .initBlocked();
		4554	defer cancel_region.deinit();
		4555	ev.writeAll(&cancel_region, fd, @ptrCast(&err)) catch {};
		4556	const exit = if (builtin.single_threaded) linux.exit else linux.exit_group;
		4557	exit(1);
		4558	}
		4559
		4560	fn execv(
		4561	arg0_expand: process.ArgExpansion,
		4562	file: [*:0]const u8,
		4563	child_argv: [:null]?[:0]const u8,
		4564	env_block: process.Environ.PosixBlock,
		4565	PATH: []const u8,
		4566	) process.ReplaceError {
		4567	const file_slice = std.mem.sliceTo(file, 0);
		4568	if (std.mem.findScalar(u8, file_slice, '/') != null) return execvPath(file, child_argv, env_block);
		4569
		4570	// Use of PATH_MAX here is valid as the path_buf will be passed
		4571	// directly to the operating system in posixExecvPath.
		4572	var path_buf: [PATH_MAX]u8 = undefined;
		4573	var it = std.mem.tokenizeScalar(u8, PATH, ':');
		4574	var seen_eacces = false;
		4575	var err: process.ReplaceError = error.FileNotFound;
		4576
		4577	// In case of expanding arg0 we must put it back if we return with an error.
		4578	const prev_arg0 = child_argv[0];
		4579	defer switch (arg0_expand) {
		4580	.expand => child_argv[0] = prev_arg0,
		4581	.no_expand => {},
		4582	};
		4583
		4584	while (it.next()) \|search_path\| {
		4585	const path_len = search_path.len + file_slice.len + 1;
		4586	if (path_buf.len < path_len + 1) return error.NameTooLong;
		4587	@memcpy(path_buf[0..search_path.len], search_path);
		4588	path_buf[search_path.len] = '/';
		4589	@memcpy(path_buf[search_path.len + 1 ..][0..file_slice.len], file_slice);
		4590	path_buf[path_len] = 0;
		4591	const full_path = path_buf[0..path_len :0].ptr;
		4592	switch (arg0_expand) {
		4593	.expand => child_argv[0] = full_path,
		4594	.no_expand => {},
		4595	}
		4596	err = execvPath(full_path, child_argv, env_block);
		4597	switch (err) {
		4598	error.AccessDenied => seen_eacces = true,
		4599	error.FileNotFound, error.NotDir => {},
		4600	else => \|e\| return e,
		4601	}
		4602	}
		4603	if (seen_eacces) return error.AccessDenied;
		4604	return err;
		4605	}
		4606	/// This function ignores PATH environment variable.
		4607	pub fn execvPath(
		4608	path: [*:0]const u8,
		4609	child_argv: [:null]const ?[:0]const u8,
		4610	env_block: process.Environ.PosixBlock,
		4611	) process.ReplaceError {
		4612	var cancel_region: CancelRegion = .init();
		4613	defer cancel_region.deinit();
		4614	try cancel_region.await(.nothing);
		4615	switch (linux.errno(linux.execve(path, child_argv, env_block.slice.ptr))) {
		4616	.FAULT => \|err\| return errnoBug(err), // Bad pointer parameter.
		4617	.@"2BIG" => return error.SystemResources,
		4618	.MFILE => return error.ProcessFdQuotaExceeded,
		4619	.NAMETOOLONG => return error.NameTooLong,
		4620	.NFILE => return error.SystemFdQuotaExceeded,
		4621	.NOMEM => return error.SystemResources,
		4622	.ACCES => return error.AccessDenied,
		4623	.PERM => return error.PermissionDenied,
		4624	.INVAL => return error.InvalidExe,
		4625	.NOEXEC => return error.InvalidExe,
		4626	.IO => return error.FileSystem,
		4627	.LOOP => return error.FileSystem,
		4628	.ISDIR => return error.IsDir,
		4629	.NOENT => return error.FileNotFound,
		4630	.NOTDIR => return error.NotDir,
		4631	.TXTBSY => return error.FileBusy,
		4632	.LIBBAD => return error.InvalidExe,
		4633	else => \|err\| return unexpectedErrno(err),
		4634	}
		4635	}
		4636
		4637	fn childWait(userdata: ?anyopaque, child: process.Child) process.Child.WaitError!process.Child.Term {
		4638	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4639	defer ev.childCleanup(child);
		4640
		4641	const pid = child.id.?;
		4642	var info: linux.siginfo_t = undefined;
		4643	var cancel_region: CancelRegion = .init();
		4644	defer cancel_region.deinit();
		4645	while (true) {
		4646	const thread = try cancel_region.awaitIoUring();
		4647	thread.enqueue().* = .{
		4648	.opcode = .WAITID,
		4649	.flags = 0,
		4650	.ioprio = 0,
		4651	.fd = pid,
		4652	.off = @intFromPtr(&info),
		4653	.addr = 0,
		4654	.len = @intFromEnum(linux.P.PID),
		4655	.rw_flags = 0,
		4656	.user_data = @intFromPtr(cancel_region.fiber),
		4657	.buf_index = 0,
		4658	.personality = 0,
		4659	.splice_fd_in = linux.W.EXITED \|
		4660	@as(i32, if (child.request_resource_usage_statistics) linux.W.NOWAIT else 0),
		4661	.addr3 = 0,
		4662	.resv = 0,
		4663	};
		4664	ev.yield(null, .nothing);
		4665	switch (cancel_region.errno()) {
		4666	.SUCCESS => {
		4667	if (child.request_resource_usage_statistics) while (true) {
		4668	try cancel_region.await(.nothing);
		4669	var rusage: linux.rusage = undefined;
		4670	switch (linux.errno(linux.waitid(
		4671	.PID,
		4672	pid,
		4673	&info,
		4674	linux.W.EXITED \| linux.W.NOHANG,
		4675	&rusage,
		4676	))) {
		4677	.SUCCESS => {
		4678	child.resource_usage_statistics.rusage = rusage;
		4679	break;
		4680	},
		4681	.INTR, .CANCELED => continue,
		4682	.CHILD => \|err\| return errnoBug(err), // Double-free.
		4683	else => \|err\| return unexpectedErrno(err),
		4684	}
		4685	};
		4686	const status: u32 = @bitCast(info.fields.common.second.sigchld.status);
		4687	const code: linux.CLD = @enumFromInt(info.code);
		4688	return switch (code) {
		4689	.EXITED => .{ .exited = @truncate(status) },
		4690	.KILLED, .DUMPED => .{ .signal = @enumFromInt(status) },
		4691	.TRAPPED, .STOPPED => .{ .stopped = status },
		4692	_, .CONTINUED => .{ .unknown = status },
		4693	};
		4694	},
		4695	.INTR, .CANCELED => continue,
		4696	.CHILD => \|err\| return errnoBug(err), // Double-free.
		4697	else => \|err\| return unexpectedErrno(err),
		4698	}
		4699	}
		4700	}
		4701
		4702	fn childKill(userdata: ?anyopaque, child: process.Child) void {
		4703	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4704	defer ev.childCleanup(child);
		4705
		4706	const pid = child.id.?;
		4707	var cancel_region: CancelRegion = .initBlocked();
		4708	defer cancel_region.deinit();
		4709	while (true) switch (linux.errno(linux.kill(pid, .TERM))) {
		4710	.SUCCESS => break,
		4711	.INTR => continue,
		4712	.PERM => return,
		4713	.INVAL => \|err\| return errnoBug(err) catch {},
		4714	.SRCH => \|err\| return errnoBug(err) catch {},
		4715	else => \|err\| return unexpectedErrno(err) catch {},
		4716	};
		4717
		4718	var info: linux.siginfo_t = undefined;
		4719	while (true) {
		4720	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
		4721	error.Canceled => unreachable, // blocked
		4722	};
		4723	thread.enqueue().* = .{
		4724	.opcode = .WAITID,
		4725	.flags = 0,
		4726	.ioprio = 0,
		4727	.fd = pid,
		4728	.off = @intFromPtr(&info),
		4729	.addr = 0,
		4730	.len = @intFromEnum(linux.P.PID),
		4731	.rw_flags = 0,
		4732	.user_data = @intFromPtr(cancel_region.fiber),
		4733	.buf_index = 0,
		4734	.personality = 0,
		4735	.splice_fd_in = linux.W.EXITED,
		4736	.addr3 = 0,
		4737	.resv = 0,
		4738	};
		4739	ev.yield(null, .nothing);
		4740	switch (cancel_region.errno()) {
		4741	.SUCCESS => return,
		4742	.INTR, .CANCELED => continue,
		4743	.CHILD => \|err\| return errnoBug(err) catch {}, // Double-free.
		4744	else => \|err\| return unexpectedErrno(err) catch {},
		4745	}
		4746	}
		4747	}
		4748
		4749	fn childCleanup(ev: Evented, child: process.Child) void {
		4750	if (child.stdin) \|*stdin\| {
		4751	ev.close(stdin.handle);
		4752	child.stdin = null;
		4753	}
		4754	if (child.stdout) \|*stdout\| {
		4755	ev.close(stdout.handle);
		4756	child.stdout = null;
		4757	}
		4758	if (child.stderr) \|*stderr\| {
		4759	ev.close(stderr.handle);
		4760	child.stderr = null;
		4761	}
		4762	child.id = null;
		4763	}
		4764
		4765	fn progressParentFile(userdata: ?*anyopaque) std.Progress.ParentFileError!File {
		4766	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4767	const cancel_protection = swapCancelProtection(ev, .blocked);
		4768	defer assert(swapCancelProtection(ev, cancel_protection) == .blocked);
		4769	ev.scanEnviron() catch \|err\| switch (err) {
		4770	error.Canceled => unreachable, // blocked
		4771	};
		4772	return ev.environ.zig_progress_file;
		4773	}
		4774
		4775	fn scanEnviron(ev: *Evented) Io.Cancelable!void {
		4776	const ev_io = ev.io();
		4777	try ev.environ_mutex.lock(ev_io);
		4778	defer ev.environ_mutex.unlock(ev_io);
		4779	ev.environ.scan(ev.allocator());
		4780	}
		4781
		4782	fn clockResolution(userdata: ?*anyopaque, clock: Io.Clock) Io.Clock.ResolutionError!Io.Duration {
		4783	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4784	_ = ev;
		4785	const clock_id = clockToPosix(clock);
		4786	var timespec: linux.timespec = undefined;
		4787	return switch (linux.errno(linux.clock_getres(clock_id, &timespec))) {
		4788	.SUCCESS => .fromNanoseconds(nanosecondsFromPosix(&timespec)),
		4789	.INVAL => return error.ClockUnavailable,
		4790	else => \|err\| return unexpectedErrno(err),
		4791	};
		4792	}
		4793
		4794	fn now(userdata: ?*anyopaque, clock: Io.Clock) Io.Timestamp {
		4795	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4796	_ = ev;
		4797	var tp: linux.timespec = undefined;
		4798	switch (linux.errno(linux.clock_gettime(clockToPosix(clock), &tp))) {
		4799	.SUCCESS => return timestampFromPosix(&tp),
		4800	else => return .zero,
		4801	}
		4802	}
		4803
		4804	fn sleep(userdata: ?*anyopaque, timeout: Io.Timeout) Io.Cancelable!void {
		4805	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4806
		4807	const timespec: linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = timespec: switch (timeout) {
		4808	.none => .{
		4809	.{
		4810	.sec = std.math.maxInt(i64),
		4811	.nsec = std.time.ns_per_s - 1,
		4812	},
		4813	.awake,
		4814	linux.IORING_TIMEOUT_ABS,
		4815	},
		4816	.duration => \|duration\| {
		4817	const ns = duration.raw.toNanoseconds();
		4818	break :timespec .{
		4819	.{
		4820	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
		4821	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
		4822	},
		4823	duration.clock,
		4824	0,
		4825	};
		4826	},
		4827	.deadline => \|deadline\| {
		4828	const ns = deadline.raw.toNanoseconds();
		4829	break :timespec .{
		4830	.{
		4831	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
		4832	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
		4833	},
		4834	deadline.clock,
		4835	linux.IORING_TIMEOUT_ABS,
		4836	};
		4837	},
		4838	};
		4839	var cancel_region: CancelRegion = .init();
		4840	defer cancel_region.deinit();
		4841	const thread = try cancel_region.awaitIoUring();
		4842	thread.enqueue().* = .{
		4843	.opcode = .TIMEOUT,
		4844	.flags = 0,
		4845	.ioprio = 0,
		4846	.fd = 0,
		4847	.off = 0,
		4848	.addr = @intFromPtr(&timespec),
		4849	.len = 1,
		4850	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
		4851	.real => linux.IORING_TIMEOUT_REALTIME,
		4852	else => 0,
		4853	.boot => linux.IORING_TIMEOUT_BOOTTIME,
		4854	}),
		4855	.user_data = @intFromPtr(cancel_region.fiber),
		4856	.buf_index = 0,
		4857	.personality = 0,
		4858	.splice_fd_in = 0,
		4859	.addr3 = 0,
		4860	.resv = 0,
		4861	};
		4862	ev.yield(null, .nothing);
		4863	switch (cancel_region.errno()) {
		4864	// Handles SUCCESS as well as clock not available and unexpected
		4865	// errors. The user had a chance to check clock resolution before
		4866	// getting here, which would have reported 0, making this a legal
		4867	// amount of time to sleep.
		4868	else => return,
		4869	.INTR, .CANCELED => return error.Canceled,
		4870	}
		4871	}
		4872
		4873	fn random(userdata: ?*anyopaque, buffer: []u8) void {
		4874	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4875	var thread: *Thread = .current();
		4876	if (!thread.csprng.isInitialized()) {
		4877	@branchHint(.unlikely);
		4878	var seed: [Csprng.seed_len]u8 = undefined;
		4879	{
		4880	const ev_io = ev.io();
		4881	ev.csprng_mutex.lockUncancelable(ev_io);
		4882	defer ev.csprng_mutex.unlock(ev_io);
		4883	if (!ev.csprng.isInitialized()) {
		4884	@branchHint(.unlikely);
		4885	var cancel_region: CancelRegion = .initBlocked();
		4886	defer cancel_region.deinit();
		4887	ev.urandomReadAll(&cancel_region, &seed) catch \|err\| switch (err) {
		4888	error.Canceled => unreachable, // blocked
		4889	else => fallbackSeed(ev, &seed),
		4890	};
		4891	ev.csprng.rng = .init(seed);
		4892	thread = .current();
		4893	}
		4894	ev.csprng.rng.fill(&seed);
		4895	}
		4896	if (!thread.csprng.isInitialized()) {
		4897	@branchHint(.likely);
		4898	thread.csprng.rng = .init(seed);
		4899	} else thread.csprng.rng.addEntropy(&seed);
		4900	}
		4901	thread.csprng.rng.fill(buffer);
		4902	}
		4903
		4904	fn randomSecure(userdata: ?*anyopaque, buffer: []u8) Io.RandomSecureError!void {
		4905	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4906	if (buffer.len == 0) return;
		4907	var cancel_region: CancelRegion = .init();
		4908	defer cancel_region.deinit();
		4909	ev.urandomReadAll(&cancel_region, buffer) catch \|err\| switch (err) {
		4910	error.Canceled => return error.Canceled,
		4911	else => return error.EntropyUnavailable,
		4912	};
		4913	}
		4914
		4915	fn netListenIpUnavailable(
		4916	userdata: ?*anyopaque,
		4917	address: net.IpAddress,
		4918	options: net.IpAddress.ListenOptions,
		4919	) net.IpAddress.ListenError!net.Server {
		4920	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4921	_ = ev;
		4922	_ = address;
		4923	_ = options;
		4924	return error.NetworkDown;
		4925	}
		4926
		4927	fn netAcceptUnavailable(
		4928	userdata: ?*anyopaque,
		4929	listen_handle: net.Socket.Handle,
		4930	) net.Server.AcceptError!net.Stream {
		4931	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4932	_ = ev;
		4933	_ = listen_handle;
		4934	return error.NetworkDown;
		4935	}
		4936
		4937	fn netBindIp(
		4938	userdata: ?*anyopaque,
		4939	address: *const net.IpAddress,
		4940	options: net.IpAddress.BindOptions,
		4941	) net.IpAddress.BindError!net.Socket {
		4942	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4943	const family = posixAddressFamily(address);
		4944	var cancel_region: CancelRegion = .init();
		4945	defer cancel_region.deinit();
		4946	const socket_fd = try ev.socket(&cancel_region, family, options);
		4947	errdefer ev.close(socket_fd);
		4948	var storage: PosixAddress = undefined;
		4949	var addr_len = addressToPosix(address, &storage);
		4950	try ev.bind(&cancel_region, socket_fd, &storage.any, addr_len);
		4951	try ev.getsockname(&cancel_region, socket_fd, &storage.any, &addr_len);
		4952	return .{
		4953	.handle = socket_fd,
		4954	.address = addressFromPosix(&storage),
		4955	};
		4956	}
		4957
		4958	fn netBindIpUnavailable(
		4959	userdata: ?*anyopaque,
		4960	address: *const net.IpAddress,
		4961	options: net.IpAddress.BindOptions,
		4962	) net.IpAddress.BindError!net.Socket {
		4963	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4964	_ = ev;
		4965	_ = address;
		4966	_ = options;
		4967	return error.NetworkDown;
		4968	}
		4969
		4970	fn netConnectIpUnavailable(
		4971	userdata: ?*anyopaque,
		4972	address: *const net.IpAddress,
		4973	options: net.IpAddress.ConnectOptions,
		4974	) net.IpAddress.ConnectError!net.Stream {
		4975	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4976	_ = ev;
		4977	_ = address;
		4978	_ = options;
		4979	return error.NetworkDown;
		4980	}
		4981
		4982	fn netListenUnixUnavailable(
		4983	userdata: ?*anyopaque,
		4984	address: *const net.UnixAddress,
		4985	options: net.UnixAddress.ListenOptions,
		4986	) net.UnixAddress.ListenError!net.Socket.Handle {
		4987	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4988	_ = ev;
		4989	_ = address;
		4990	_ = options;
		4991	return error.AddressFamilyUnsupported;
		4992	}
		4993
		4994	fn netConnectUnixUnavailable(
		4995	userdata: ?*anyopaque,
		4996	address: *const net.UnixAddress,
		4997	) net.UnixAddress.ConnectError!net.Socket.Handle {
		4998	const ev: *Evented = @ptrCast(@alignCast(userdata));
		4999	_ = ev;
		5000	_ = address;
		5001	return error.AddressFamilyUnsupported;
		5002	}
		5003
		5004	fn netSocketCreatePairUnavailable(
		5005	userdata: ?*anyopaque,
		5006	options: net.Socket.CreatePairOptions,
		5007	) net.Socket.CreatePairError![2]net.Socket {
		5008	_ = userdata;
		5009	_ = options;
		5010	return error.OperationUnsupported;
		5011	}
		5012
		5013	fn netSendUnavailable(
		5014	userdata: ?*anyopaque,
		5015	handle: net.Socket.Handle,
		5016	messages: []net.OutgoingMessage,
		5017	flags: net.SendFlags,
		5018	) struct { ?net.Socket.SendError, usize } {
		5019	const ev: *Evented = @ptrCast(@alignCast(userdata));
		5020	_ = ev;
		5021	_ = handle;
		5022	_ = messages;
		5023	_ = flags;
		5024	return .{ error.NetworkDown, 0 };
		5025	}
		5026
		5027	fn netReceive(
		5028	userdata: ?*anyopaque,
		5029	handle: net.Socket.Handle,
		5030	message_buffer: []net.IncomingMessage,
		5031	data_buffer: []u8,
		5032	flags: net.ReceiveFlags,
		5033	timeout: Io.Timeout,
		5034	) struct { ?net.Socket.ReceiveTimeoutError, usize } {
		5035	const ev: *Evented = @ptrCast(@alignCast(userdata));
		5036	const ev_io = ev.io();
		5037
		5038	var message_i: usize = 0;
		5039	var data_i: usize = 0;
		5040
		5041	const deadline: ?struct {
		5042	raw: Io.Timestamp,
		5043	timespec: linux.kernel_timespec,
		5044	clock: Io.Clock,
		5045	} = if (timeout.toTimestamp(ev_io)) \|deadline\| deadline: {
		5046	const ns = deadline.raw.toNanoseconds();
		5047	break :deadline .{
		5048	.raw = deadline.raw,
		5049	.timespec = .{
		5050	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
		5051	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
		5052	},
		5053	.clock = deadline.clock,
		5054	};
		5055	} else null;
		5056
		5057	var cancel_region: CancelRegion = .init();
		5058	defer cancel_region.deinit();
		5059	while (true) {
		5060	if (message_buffer.len - message_i == 0) return .{ null, message_i };
		5061	const message = &message_buffer[message_i];
		5062	const remaining_data_buffer = data_buffer[data_i..];
		5063	var storage: PosixAddress = undefined;
		5064	var iov: iovec = .{ .base = remaining_data_buffer.ptr, .len = remaining_data_buffer.len };
		5065	var msg: linux.msghdr = .{
		5066	.name = &storage.any,
		5067	.namelen = @sizeOf(PosixAddress),
		5068	.iov = (&iov)[0..1],
		5069	.iovlen = 1,
		5070	.control = message.control.ptr,
		5071	.controllen = @intCast(message.control.len),
		5072	.flags = undefined,
		5073	};
		5074
		5075	const thread = cancel_region.awaitIoUring() catch \|err\| return .{ err, message_i };
		5076	thread.enqueue().* = .{
		5077	.opcode = .RECVMSG,
		5078	.flags = if (deadline) \|_\| linux.IOSQE_IO_LINK else 0,
		5079	.ioprio = 0,
		5080	.fd = handle,
		5081	.off = 0,
		5082	.addr = @intFromPtr(&msg),
		5083	.len = 0,
		5084	.rw_flags = linux.MSG.NOSIGNAL \|
		5085	@as(u32, if (flags.oob) linux.MSG.OOB else 0) \|
		5086	@as(u32, if (flags.peek) linux.MSG.PEEK else 0) \|
		5087	@as(u32, if (flags.trunc) linux.MSG.TRUNC else 0),
		5088	.user_data = @intFromPtr(cancel_region.fiber),
		5089	.buf_index = 0,
		5090	.personality = 0,
		5091	.splice_fd_in = 0,
		5092	.addr3 = 0,
		5093	.resv = 0,
		5094	};
		5095	if (deadline) \|deadline_ptr\| thread.enqueue(). = .{
		5096	.opcode = .LINK_TIMEOUT,
		5097	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
		5098	.ioprio = 0,
		5099	.fd = 0,
		5100	.off = 0,
		5101	.addr = @intFromPtr(&deadline_ptr.timespec),
		5102	.len = 1,
		5103	.rw_flags = linux.IORING_TIMEOUT_ABS \| @as(u32, switch (deadline_ptr.clock) {
		5104	.real => linux.IORING_TIMEOUT_REALTIME,
		5105	else => 0,
		5106	.boot => linux.IORING_TIMEOUT_BOOTTIME,
		5107	}),
		5108	.user_data = @intFromEnum(Completion.UserData.wakeup),
		5109	.buf_index = 0,
		5110	.personality = 0,
		5111	.splice_fd_in = 0,
		5112	.addr3 = 0,
		5113	.resv = 0,
		5114	};
		5115	ev.yield(null, .nothing);
		5116	const completion = cancel_region.completion();
		5117	switch (completion.errno()) {
		5118	.SUCCESS => {
		5119	const data = remaining_data_buffer[0..@intCast(completion.result)];
		5120	data_i += data.len;
		5121	message.* = .{
		5122	.from = addressFromPosix(&storage),
		5123	.data = data,
		5124	.control = if (msg.control) \|ptr\| @as([*]u8, @ptrCast(ptr))[0..msg.controllen] else message.control,
		5125	.flags = .{
		5126	.eor = (msg.flags & linux.MSG.EOR) != 0,
		5127	.trunc = (msg.flags & linux.MSG.TRUNC) != 0,
		5128	.ctrunc = (msg.flags & linux.MSG.CTRUNC) != 0,
		5129	.oob = (msg.flags & linux.MSG.OOB) != 0,
		5130	.errqueue = if (@hasDecl(linux.MSG, "ERRQUEUE")) (msg.flags & linux.MSG.ERRQUEUE) != 0 else false,
		5131	},
		5132	};
		5133	message_i += 1;
		5134	continue;
		5135	},
		5136	.AGAIN => unreachable,
		5137	.INTR, .CANCELED => {
		5138	if (deadline) \|d\| {
		5139	if (now(ev, d.clock).nanoseconds >= d.raw.nanoseconds) return .{ error.Timeout, message_i };
		5140	}
		5141	continue;
		5142	},
		5143
		5144	.BADF => \|err\| return .{ errnoBug(err), message_i },
		5145	.NFILE => return .{ error.SystemFdQuotaExceeded, message_i },
		5146	.MFILE => return .{ error.ProcessFdQuotaExceeded, message_i },
		5147	.FAULT => \|err\| return .{ errnoBug(err), message_i },
		5148	.INVAL => \|err\| return .{ errnoBug(err), message_i },
		5149	.NOBUFS => return .{ error.SystemResources, message_i },
		5150	.NOMEM => return .{ error.SystemResources, message_i },
		5151	.NOTCONN => return .{ error.SocketUnconnected, message_i },
		5152	.NOTSOCK => \|err\| return .{ errnoBug(err), message_i },
		5153	.MSGSIZE => return .{ error.MessageOversize, message_i },
		5154	.PIPE => return .{ error.SocketUnconnected, message_i },
		5155	.OPNOTSUPP => \|err\| return .{ errnoBug(err), message_i },
		5156	.CONNRESET => return .{ error.ConnectionResetByPeer, message_i },
		5157	.NETDOWN => return .{ error.NetworkDown, message_i },
		5158	else => \|err\| return .{ unexpectedErrno(err), message_i },
		5159	}
		5160	}
		5161	}
		5162
		5163	fn netReceiveUnavailable(
		5164	userdata: ?*anyopaque,
		5165	handle: net.Socket.Handle,
		5166	message_buffer: []net.IncomingMessage,
		5167	data_buffer: []u8,
		5168	flags: net.ReceiveFlags,
		5169	timeout: Io.Timeout,
		5170	) struct { ?net.Socket.ReceiveTimeoutError, usize } {
		5171	const ev: *Evented = @ptrCast(@alignCast(userdata));
		5172	_ = ev;
		5173	_ = handle;
		5174	_ = message_buffer;
		5175	_ = data_buffer;
		5176	_ = flags;
		5177	_ = timeout;
		5178	return .{ error.NetworkDown, 0 };
		5179	}
		5180
		5181	fn netReadUnavailable(
		5182	userdata: ?*anyopaque,
		5183	fd: net.Socket.Handle,
		5184	data: [][]u8,
		5185	) net.Stream.Reader.Error!usize {
		5186	const ev: *Evented = @ptrCast(@alignCast(userdata));
		5187	_ = ev;
		5188	_ = fd;
		5189	_ = data;
		5190	return error.NetworkDown;
		5191	}
		5192
		5193	fn netWriteUnavailable(
		5194	userdata: ?*anyopaque,
		5195	handle: net.Socket.Handle,
		5196	header: []const u8,
		5197	data: []const []const u8,
		5198	splat: usize,
		5199	) net.Stream.Writer.Error!usize {
		5200	const ev: *Evented = @ptrCast(@alignCast(userdata));
		5201	_ = ev;
		5202	_ = handle;
		5203	_ = header;
		5204	_ = data;
		5205	_ = splat;
		5206	return error.NetworkDown;
		5207	}
		5208
		5209	fn netWriteFileUnavailable(
		5210	userdata: ?*anyopaque,
		5211	socket_handle: net.Socket.Handle,
		5212	header: []const u8,
		5213	file_reader: *File.Reader,
		5214	limit: Io.Limit,
		5215	) net.Stream.Writer.WriteFileError!usize {
		5216	const ev: *Evented = @ptrCast(@alignCast(userdata));
		5217	_ = ev;
		5218	_ = socket_handle;
		5219	_ = header;
		5220	_ = file_reader;
		5221	_ = limit;
		5222	return error.NetworkDown;
		5223	}
		5224
		5225	fn netClose(userdata: ?*anyopaque, handles: []const net.Socket.Handle) void {
		5226	const ev: *Evented = @ptrCast(@alignCast(userdata));
		5227	for (handles) \|handle\| ev.close(handle);
		5228	}
		5229
		5230	fn netCloseUnavailable(userdata: ?*anyopaque, handles: []const net.Socket.Handle) void {
		5231	const ev: *Evented = @ptrCast(@alignCast(userdata));
		5232	_ = ev;
		5233	_ = handles;
		5234	unreachable; // How you gonna close something that was impossible to open?
		5235	}
		5236
		5237	fn netShutdown(
		5238	userdata: ?*anyopaque,
		5239	handle: net.Socket.Handle,
		5240	how: net.ShutdownHow,
		5241	) net.ShutdownError!void {
		5242	const ev: *Evented = @ptrCast(@alignCast(userdata));
		5243	var cancel_region: CancelRegion = .init();
		5244	defer cancel_region.deinit();
		5245	while (true) {
		5246	const thread = try cancel_region.awaitIoUring();
		5247	thread.enqueue().* = .{
		5248	.opcode = .SHUTDOWN,
		5249	.flags = 0,
		5250	.ioprio = 0,
		5251	.fd = handle,
		5252	.off = 0,
		5253	.addr = 0,
		5254	.len = switch (how) {
		5255	.recv => linux.SHUT.RD,
		5256	.send => linux.SHUT.WR,
		5257	.both => linux.SHUT.RDWR,
		5258	},
		5259	.rw_flags = 0,
		5260	.user_data = @intFromPtr(cancel_region.fiber),
		5261	.buf_index = 0,
		5262	.personality = 0,
		5263	.splice_fd_in = 0,
		5264	.addr3 = 0,
		5265	.resv = 0,
		5266	};
		5267	ev.yield(null, .nothing);
		5268	switch (cancel_region.errno()) {
		5269	.SUCCESS => return,
		5270	.INTR, .CANCELED => continue,
		5271	.BADF, .NOTSOCK, .INVAL => \|err\| return errnoBug(err),
		5272	.NOTCONN => return error.SocketUnconnected,
		5273	.NOBUFS => return error.SystemResources,
		5274	else => \|err\| return unexpectedErrno(err),
		5275	}
		5276	}
		5277	}
		5278
		5279	fn netShutdownUnavailable(
		5280	userdata: ?*anyopaque,
		5281	handle: net.Socket.Handle,
		5282	how: net.ShutdownHow,
		5283	) net.ShutdownError!void {
		5284	const ev: *Evented = @ptrCast(@alignCast(userdata));
		5285	_ = ev;
		5286	_ = handle;
		5287	_ = how;
		5288	unreachable; // How you gonna shutdown something that was impossible to open?
		5289	}
		5290
		5291	fn netInterfaceNameResolveUnavailable(
		5292	userdata: ?*anyopaque,
		5293	name: *const net.Interface.Name,
		5294	) net.Interface.Name.ResolveError!net.Interface {
		5295	const ev: *Evented = @ptrCast(@alignCast(userdata));
		5296	_ = ev;
		5297	_ = name;
		5298	return error.InterfaceNotFound;
		5299	}
		5300
		5301	fn netInterfaceNameUnavailable(
		5302	userdata: ?*anyopaque,
		5303	interface: net.Interface,
		5304	) net.Interface.NameError!net.Interface.Name {
		5305	const ev: *Evented = @ptrCast(@alignCast(userdata));
		5306	_ = ev;
		5307	_ = interface;
		5308	return error.Unexpected;
		5309	}
		5310
		5311	fn netLookupUnavailable(
		5312	userdata: ?*anyopaque,
		5313	host_name: net.HostName,
		5314	resolved: *Io.Queue(net.HostName.LookupResult),
		5315	options: net.HostName.LookupOptions,
		5316	) net.HostName.LookupError!void {
		5317	const ev: *Evented = @ptrCast(@alignCast(userdata));
		5318	_ = host_name;
		5319	_ = options;
		5320	resolved.close(ev.io());
		5321	return error.NetworkDown;
		5322	}
		5323
		5324	fn bind(
		5325	ev: *Evented,
		5326	cancel_region: *CancelRegion,
		5327	socket_fd: fd_t,
		5328	addr: *const linux.sockaddr,
		5329	addr_len: linux.socklen_t,
		5330	) !void {
		5331	while (true) {
		5332	const thread = try cancel_region.awaitIoUring();
		5333	thread.enqueue().* = .{
		5334	.opcode = .BIND,
		5335	.flags = 0,
		5336	.ioprio = 0,
		5337	.fd = socket_fd,
		5338	.off = addr_len,
		5339	.addr = @intFromPtr(addr),
		5340	.len = 0,
		5341	.rw_flags = 0,
		5342	.user_data = @intFromPtr(cancel_region.fiber),
		5343	.buf_index = 0,
		5344	.personality = 0,
		5345	.splice_fd_in = 0,
		5346	.addr3 = 0,
		5347	.resv = 0,
		5348	};
		5349	ev.yield(null, .nothing);
		5350	switch (cancel_region.errno()) {
		5351	.SUCCESS => return,
		5352	.INTR, .CANCELED => continue,
		5353	.ADDRINUSE => return error.AddressInUse,
		5354	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
		5355	.INVAL => \|err\| return errnoBug(err), // invalid parameters
		5356	.NOTSOCK => \|err\| return errnoBug(err), // invalid `sockfd`
		5357	.AFNOSUPPORT => return error.AddressFamilyUnsupported,
		5358	.ADDRNOTAVAIL => return error.AddressUnavailable,
		5359	.FAULT => \|err\| return errnoBug(err), // invalid `addr` pointer
		5360	.NOMEM => return error.SystemResources,
		5361	else => \|err\| return unexpectedErrno(err),
		5362	}
		5363	}
		5364	}
		5365
		5366	fn close(ev: *Evented, fd: fd_t) void {
		5367	var cancel_region: CancelRegion = .initBlocked();
		5368	defer cancel_region.deinit();
		5369	while (true) {
		5370	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
		5371	error.Canceled => unreachable, // blocked
		5372	};
		5373	thread.enqueue().* = .{
		5374	.opcode = .CLOSE,
		5375	.flags = 0,
		5376	.ioprio = 0,
		5377	.fd = fd,
		5378	.off = 0,
		5379	.addr = 0,
		5380	.len = 0,
		5381	.rw_flags = 0,
		5382	.user_data = @intFromPtr(cancel_region.fiber),
		5383	.buf_index = 0,
		5384	.personality = 0,
		5385	.splice_fd_in = 0,
		5386	.addr3 = 0,
		5387	.resv = 0,
		5388	};
		5389	ev.yield(null, .nothing);
		5390	switch (cancel_region.errno()) {
		5391	.SUCCESS => return,
		5392	.INTR, .CANCELED => continue,
		5393	.BADF => unreachable, // Always a race condition.
		5394	else => break,
		5395	}
		5396	}
		5397	}
		5398
		5399	fn fchmodat(
		5400	ev: *Evented,
		5401	cancel_region: *CancelRegion,
		5402	dir: fd_t,
		5403	path: [*:0]const u8,
		5404	mode: linux.mode_t,
		5405	flags: u32,
		5406	) Dir.SetFilePermissionsError!void {
		5407	_ = ev;
		5408	while (true) {
		5409	try cancel_region.await(.nothing);
		5410	switch (linux.errno(linux.fchmodat2(dir, path, mode, flags))) {
		5411	.SUCCESS => return,
		5412	.INTR => continue,
		5413	.BADF => \|err\| return errnoBug(err),
		5414	.FAULT => \|err\| return errnoBug(err),
		5415	.INVAL => \|err\| return errnoBug(err),
		5416	.ACCES => return error.AccessDenied,
		5417	.IO => return error.InputOutput,
		5418	.LOOP => return error.SymLinkLoop,
		5419	.NOENT => return error.FileNotFound,
		5420	.NOMEM => return error.SystemResources,
		5421	.NOTDIR => return error.FileNotFound,
		5422	.OPNOTSUPP => return error.OperationUnsupported,
		5423	.PERM => return error.PermissionDenied,
		5424	.ROFS => return error.ReadOnlyFileSystem,
		5425	else => \|err\| return unexpectedErrno(err),
		5426	}
		5427	}
		5428	}
		5429
		5430	fn fchownat(
		5431	ev: *Evented,
		5432	cancel_region: *CancelRegion,
		5433	dir: fd_t,
		5434	path: [*:0]const u8,
		5435	owner: linux.uid_t,
		5436	group: linux.gid_t,
		5437	flags: u32,
		5438	) File.SetOwnerError!void {
		5439	_ = ev;
		5440	while (true) {
		5441	try cancel_region.await(.nothing);
		5442	switch (linux.errno(linux.fchownat(dir, path, owner, group, flags))) {
		5443	.SUCCESS => return,
		5444	.INTR => continue,
		5445	.BADF => \|err\| return errnoBug(err), // likely fd refers to directory opened without `Dir.OpenOptions.iterate`
		5446	.FAULT => \|err\| return errnoBug(err),
		5447	.INVAL => \|err\| return errnoBug(err),
		5448	.ACCES => return error.AccessDenied,
		5449	.IO => return error.InputOutput,
		5450	.LOOP => return error.SymLinkLoop,
		5451	.NOENT => return error.FileNotFound,
		5452	.NOMEM => return error.SystemResources,
		5453	.NOTDIR => return error.FileNotFound,
		5454	.PERM => return error.PermissionDenied,
		5455	.ROFS => return error.ReadOnlyFileSystem,
		5456	else => \|err\| return unexpectedErrno(err),
		5457	}
		5458	}
		5459	}
		5460
		5461	fn flock(
		5462	ev: *Evented,
		5463	cancel_region: *CancelRegion,
		5464	fd: fd_t,
		5465	op: File.Lock,
		5466	blocking: enum { blocking, nonblocking },
		5467	) (File.LockError \|\| error{WouldBlock})!void {
		5468	while (true) {
		5469	try cancel_region.await(.nothing);
		5470	switch (linux.errno(linux.flock(fd, LOCK.NB \| @as(i32, switch (op) {
		5471	.none => LOCK.UN,
		5472	.shared => LOCK.SH,
		5473	.exclusive => LOCK.EX,
		5474	})))) {
		5475	.SUCCESS => return,
		5476	.INTR => continue,
		5477	.BADF => \|err\| return errnoBug(err),
		5478	.INVAL => \|err\| return errnoBug(err), // invalid parameters
		5479	.NOLCK => return error.SystemResources,
		5480	.AGAIN => {
		5481	const thread = try cancel_region.awaitIoUring();
		5482	thread.enqueue().* = .{
		5483	.opcode = .NOP,
		5484	.flags = 0,
		5485	.ioprio = 0,
		5486	.fd = 0,
		5487	.off = 0,
		5488	.addr = 0,
		5489	.len = 0,
		5490	.rw_flags = 0,
		5491	.user_data = @intFromPtr(cancel_region.fiber),
		5492	.buf_index = 0,
		5493	.personality = 0,
		5494	.splice_fd_in = 0,
		5495	.addr3 = 0,
		5496	.resv = 0,
		5497	};
		5498	ev.yield(null, .nothing);
		5499	switch (cancel_region.errno()) {
		5500	.SUCCESS, .INTR, .CANCELED => {},
		5501	else => unreachable,
		5502	}
		5503	switch (blocking) {
		5504	.blocking => continue,
		5505	.nonblocking => return error.WouldBlock,
		5506	}
		5507	},
		5508	.OPNOTSUPP => return error.FileLocksUnsupported,
		5509	else => \|err\| return unexpectedErrno(err),
		5510	}
		5511	}
		5512	}
		5513
		5514	fn getsockname(
		5515	ev: *Evented,
		5516	cancel_region: *CancelRegion,
		5517	socket_fd: fd_t,
		5518	addr: *linux.sockaddr,
		5519	addr_len: *linux.socklen_t,
		5520	) !void {
		5521	_ = ev;
		5522	while (true) {
		5523	try cancel_region.await(.nothing);
		5524	switch (linux.errno(linux.getsockname(socket_fd, addr, addr_len))) {
		5525	.SUCCESS => return,
		5526	.INTR => continue,
		5527	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
		5528	.FAULT => \|err\| return errnoBug(err),
		5529	.INVAL => \|err\| return errnoBug(err), // invalid parameters
		5530	.NOTSOCK => \|err\| return errnoBug(err), // always a race condition
		5531	.NOBUFS => return error.SystemResources,
		5532	else => \|err\| return unexpectedErrno(err),
		5533	}
		5534	}
		5535	}
		5536
		5537	fn linkat(
		5538	ev: *Evented,
		5539	cancel_region: *CancelRegion,
		5540	old_dir: fd_t,
		5541	old_path: [*:0]const u8,
		5542	new_dir: fd_t,
		5543	new_path: [*:0]const u8,
		5544	flags: u32,
		5545	) File.HardLinkError!void {
		5546	while (true) {
		5547	const thread = try cancel_region.awaitIoUring();
		5548	thread.enqueue().* = .{
		5549	.opcode = .LINKAT,
		5550	.flags = 0,
		5551	.ioprio = 0,
		5552	.fd = old_dir,
		5553	.off = @intFromPtr(new_path),
		5554	.addr = @intFromPtr(old_path),
		5555	.len = @bitCast(new_dir),
		5556	.rw_flags = flags,
		5557	.user_data = @intFromPtr(cancel_region.fiber),
		5558	.buf_index = 0,
		5559	.personality = 0,
		5560	.splice_fd_in = 0,
		5561	.addr3 = 0,
		5562	.resv = 0,
		5563	};
		5564	ev.yield(null, .nothing);
		5565	switch (cancel_region.errno()) {
		5566	.SUCCESS => return,
		5567	.INTR, .CANCELED => continue,
		5568	.ACCES => return error.AccessDenied,
		5569	.DQUOT => return error.DiskQuota,
		5570	.EXIST => return error.PathAlreadyExists,
		5571	.IO => return error.HardwareFailure,
		5572	.LOOP => return error.SymLinkLoop,
		5573	.MLINK => return error.LinkQuotaExceeded,
		5574	.NAMETOOLONG => return error.NameTooLong,
		5575	.NOENT => return error.FileNotFound,
		5576	.NOMEM => return error.SystemResources,
		5577	.NOSPC => return error.NoSpaceLeft,
		5578	.NOTDIR => return error.NotDir,
		5579	.PERM => return error.PermissionDenied,
		5580	.ROFS => return error.ReadOnlyFileSystem,
		5581	.XDEV => return error.CrossDevice,
		5582	.ILSEQ => return error.BadPathName,
		5583	.FAULT => \|err\| return errnoBug(err),
		5584	.INVAL => \|err\| return errnoBug(err),
		5585	else => \|err\| return unexpectedErrno(err),
		5586	}
		5587	}
		5588	}
		5589
		5590	fn lseek(
		5591	ev: *Evented,
		5592	cancel_region: *CancelRegion,
		5593	fd: fd_t,
		5594	offset: u64,
		5595	whence: u32,
		5596	) File.SeekError!void {
		5597	_ = ev;
		5598	while (true) {
		5599	try cancel_region.await(.nothing);
		5600	var result: u64 = undefined;
		5601	switch (linux.errno(switch (@sizeOf(usize)) {
		5602	else => comptime unreachable,
		5603	4 => linux.llseek(fd, offset, &result, whence),
		5604	8 => linux.lseek(fd, @bitCast(offset), whence),
		5605	})) {
		5606	.SUCCESS => return,
		5607	.INTR => continue,
		5608	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
		5609	.INVAL => return error.Unseekable,
		5610	.OVERFLOW => return error.Unseekable,
		5611	.SPIPE => return error.Unseekable,
		5612	.NXIO => return error.Unseekable,
		5613	else => \|err\| return unexpectedErrno(err),
		5614	}
		5615	}
		5616	}
		5617
		5618	fn openat(
		5619	ev: *Evented,
		5620	cancel_region: *CancelRegion,
		5621	dir: fd_t,
		5622	path: [*:0]const u8,
		5623	flags: linux.O,
		5624	mode: linux.mode_t,
		5625	) File.OpenError!fd_t {
		5626	var mut_flags = flags;
		5627	if (@hasField(linux.O, "LARGEFILE")) mut_flags.LARGEFILE = true;
		5628	while (true) {
		5629	const thread = try cancel_region.awaitIoUring();
		5630	thread.enqueue().* = .{
		5631	.opcode = .OPENAT,
		5632	.flags = 0,
		5633	.ioprio = 0,
		5634	.fd = dir,
		5635	.off = 0,
		5636	.addr = @intFromPtr(path),
		5637	.len = mode,
		5638	.rw_flags = @bitCast(mut_flags),
		5639	.user_data = @intFromPtr(cancel_region.fiber),
		5640	.buf_index = 0,
		5641	.personality = 0,
		5642	.splice_fd_in = 0,
		5643	.addr3 = 0,
		5644	.resv = 0,
		5645	};
		5646	ev.yield(null, .nothing);
		5647	const completion = cancel_region.completion();
		5648	switch (completion.errno()) {
		5649	.SUCCESS => return completion.result,
		5650	.INTR, .CANCELED => continue,
		5651	.FAULT => \|err\| return errnoBug(err),
		5652	.INVAL => return error.BadPathName,
		5653	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
		5654	.ACCES => return error.AccessDenied,
		5655	.FBIG => return error.FileTooBig,
		5656	.OVERFLOW => return error.FileTooBig,
		5657	.ISDIR => return error.IsDir,
		5658	.LOOP => return error.SymLinkLoop,
		5659	.MFILE => return error.ProcessFdQuotaExceeded,
		5660	.NAMETOOLONG => return error.NameTooLong,
		5661	.NFILE => return error.SystemFdQuotaExceeded,
		5662	.NODEV => return error.NoDevice,
		5663	.NOENT => return error.FileNotFound,
		5664	.SRCH => return error.FileNotFound, // Linux when opening procfs files.
		5665	.NOMEM => return error.SystemResources,
		5666	.NOSPC => return error.NoSpaceLeft,
		5667	.NOTDIR => return error.NotDir,
		5668	.PERM => return error.PermissionDenied,
		5669	.EXIST => return error.PathAlreadyExists,
		5670	.BUSY => return error.DeviceBusy,
		5671	.OPNOTSUPP => return error.FileLocksUnsupported,
		5672	.AGAIN => return error.WouldBlock,
		5673	.TXTBSY => return error.FileBusy,
		5674	.NXIO => return error.NoDevice,
		5675	.ILSEQ => return error.BadPathName,
		5676	else => \|err\| return unexpectedErrno(err),
		5677	}
		5678	}
		5679	}
		5680
		5681	fn preadv(
		5682	ev: *Evented,
		5683	cancel_region: *CancelRegion,
		5684	fd: fd_t,
		5685	iov: []const iovec,
		5686	offset: ?u64,
		5687	) File.Reader.Error!usize {
		5688	if (iov.len == 0) return 0;
		5689	const gather = iov.len > 1 or iov[0].len > 0xfffff000;
		5690	while (true) {
		5691	const thread = try cancel_region.awaitIoUring();
		5692	thread.enqueue().* = .{
		5693	.opcode = if (gather) .READV else .READ,
		5694	.flags = 0,
		5695	.ioprio = 0,
		5696	.fd = fd,
		5697	.off = offset orelse std.math.maxInt(u64),
		5698	.addr = if (gather) @intFromPtr(iov.ptr) else @intFromPtr(iov[0].base),
		5699	.len = @intCast(if (gather) iov.len else iov[0].len),
		5700	.rw_flags = 0,
		5701	.user_data = @intFromPtr(cancel_region.fiber),
		5702	.buf_index = 0,
		5703	.personality = 0,
		5704	.splice_fd_in = 0,
		5705	.addr3 = 0,
		5706	.resv = 0,
		5707	};
		5708	ev.yield(null, .nothing);
		5709	const completion = cancel_region.completion();
		5710	switch (completion.errno()) {
		5711	.SUCCESS => return @as(u32, @bitCast(completion.result)),
		5712	.INTR, .CANCELED => continue,
		5713	.INVAL => \|err\| return errnoBug(err),
		5714	.FAULT => \|err\| return errnoBug(err),
		5715	.AGAIN => return error.WouldBlock,
		5716	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed
		5717	.IO => return error.InputOutput,
		5718	.ISDIR => return error.IsDir,
		5719	.NOBUFS => return error.SystemResources,
		5720	.NOMEM => return error.SystemResources,
		5721	.NOTCONN => return error.SocketUnconnected,
		5722	.CONNRESET => return error.ConnectionResetByPeer,
		5723	else => \|err\| return unexpectedErrno(err),
		5724	}
		5725	}
		5726	}
		5727
		5728	fn pwritev(
		5729	ev: *Evented,
		5730	cancel_region: *CancelRegion,
		5731	fd: fd_t,
		5732	iov: []const iovec_const,
		5733	offset: ?u64,
		5734	) File.Writer.Error!usize {
		5735	if (iov.len == 0) return 0;
		5736	const scatter = iov.len > 1 or iov[0].len > 0xfffff000;
		5737	while (true) {
		5738	const thread = try cancel_region.awaitIoUring();
		5739	thread.enqueue().* = .{
		5740	.opcode = if (scatter) .WRITEV else .WRITE,
		5741	.flags = 0,
		5742	.ioprio = 0,
		5743	.fd = fd,
		5744	.off = offset orelse std.math.maxInt(u64),
		5745	.addr = if (scatter) @intFromPtr(iov.ptr) else @intFromPtr(iov[0].base),
		5746	.len = @intCast(if (scatter) iov.len else iov[0].len),
		5747	.rw_flags = 0,
		5748	.user_data = @intFromPtr(cancel_region.fiber),
		5749	.buf_index = 0,
		5750	.personality = 0,
		5751	.splice_fd_in = 0,
		5752	.addr3 = 0,
		5753	.resv = 0,
		5754	};
		5755	ev.yield(null, .nothing);
		5756	const completion = cancel_region.completion();
		5757	switch (completion.errno()) {
		5758	.SUCCESS => return @as(u32, @bitCast(completion.result)),
		5759	.INTR, .CANCELED => continue,
		5760	.INVAL => \|err\| return errnoBug(err),
		5761	.FAULT => \|err\| return errnoBug(err),
		5762	.AGAIN => return error.WouldBlock,
		5763	.BADF => return error.NotOpenForWriting, // Can be a race condition.
		5764	.DESTADDRREQ => \|err\| return errnoBug(err), // `connect` was never called.
		5765	.DQUOT => return error.DiskQuota,
		5766	.FBIG => return error.FileTooBig,
		5767	.IO => return error.InputOutput,
		5768	.NOSPC => return error.NoSpaceLeft,
		5769	.PERM => return error.PermissionDenied,
		5770	.PIPE => return error.BrokenPipe,
		5771	.CONNRESET => \|err\| return errnoBug(err), // Not a socket handle.
		5772	.BUSY => return error.DeviceBusy,
		5773	else => \|err\| return unexpectedErrno(err),
		5774	}
		5775	}
		5776	}
		5777
		5778	fn readAll(
		5779	ev: *Evented,
		5780	cancel_region: *CancelRegion,
		5781	fd: fd_t,
		5782	buffer: []u8,
		5783	) (File.Reader.Error \|\| error{EndOfStream})!void {
		5784	var index: usize = 0;
		5785	while (buffer.len - index != 0) {
		5786	const len = try ev.preadv(cancel_region, fd, &.{
		5787	.{ .base = buffer[index..].ptr, .len = buffer.len - index },
		5788	}, null);
		5789	if (len == 0) return error.EndOfStream;
		5790	index += len;
		5791	}
		5792	}
		5793
		5794	fn realPath(
		5795	ev: *Evented,
		5796	cancel_region: *CancelRegion,
		5797	fd: fd_t,
		5798	out_buffer: []u8,
		5799	) File.RealPathError!usize {
		5800	_ = ev;
		5801	var procfs_buf: [std.fmt.count("/proc/self/fd/{d}\x00", .{std.math.minInt(fd_t)})]u8 = undefined;
		5802	const proc_path = std.fmt.bufPrintSentinel(&procfs_buf, "/proc/self/fd/{d}", .{fd}, 0) catch
		5803	unreachable;
		5804	while (true) {
		5805	try cancel_region.await(.nothing);
		5806	const rc = linux.readlink(proc_path, out_buffer.ptr, out_buffer.len);
		5807	switch (linux.errno(rc)) {
		5808	.SUCCESS => return rc,
		5809	.INTR => continue,
		5810	.ACCES => return error.AccessDenied,
		5811	.FAULT => \|err\| return errnoBug(err),
		5812	.IO => return error.FileSystem,
		5813	.LOOP => return error.SymLinkLoop,
		5814	.NAMETOOLONG => return error.NameTooLong,
		5815	.NOENT => return error.FileNotFound,
		5816	.NOMEM => return error.SystemResources,
		5817	.NOTDIR => return error.NotDir,
		5818	.ILSEQ => \|err\| return errnoBug(err),
		5819	else => \|err\| return unexpectedErrno(err),
		5820	}
		5821	}
		5822	}
		5823
		5824	fn renameat(
		5825	ev: *Evented,
		5826	cancel_region: *CancelRegion,
		5827	old_dir: fd_t,
		5828	old_path: [*:0]const u8,
		5829	new_dir: fd_t,
		5830	new_path: [*:0]const u8,
		5831	flags: linux.RENAME,
		5832	) Dir.RenameError!void {
		5833	while (true) {
		5834	const thread = try cancel_region.awaitIoUring();
		5835	thread.enqueue().* = .{
		5836	.opcode = .RENAMEAT,
		5837	.flags = 0,
		5838	.ioprio = 0,
		5839	.fd = old_dir,
		5840	.off = @intFromPtr(new_path),
		5841	.addr = @intFromPtr(old_path),
		5842	.len = @bitCast(new_dir),
		5843	.rw_flags = @bitCast(flags),
		5844	.user_data = @intFromPtr(cancel_region.fiber),
		5845	.buf_index = 0,
		5846	.personality = 0,
		5847	.splice_fd_in = 0,
		5848	.addr3 = 0,
		5849	.resv = 0,
		5850	};
		5851	ev.yield(null, .nothing);
		5852	switch (cancel_region.errno()) {
		5853	.SUCCESS => return,
		5854	.INTR, .CANCELED => continue,
		5855	.ACCES => return error.AccessDenied,
		5856	.PERM => return error.PermissionDenied,
		5857	.BUSY => return error.FileBusy,
		5858	.DQUOT => return error.DiskQuota,
		5859	.ISDIR => return error.IsDir,
		5860	.IO => return error.HardwareFailure,
		5861	.LOOP => return error.SymLinkLoop,
		5862	.MLINK => return error.LinkQuotaExceeded,
		5863	.NAMETOOLONG => return error.NameTooLong,
		5864	.NOENT => return error.FileNotFound,
		5865	.NOTDIR => return error.NotDir,
		5866	.NOMEM => return error.SystemResources,
		5867	.NOSPC => return error.NoSpaceLeft,
		5868	.EXIST => return error.DirNotEmpty,
		5869	.NOTEMPTY => return error.DirNotEmpty,
		5870	.ROFS => return error.ReadOnlyFileSystem,
		5871	.XDEV => return error.CrossDevice,
		5872	.ILSEQ => return error.BadPathName,
		5873	.FAULT => \|err\| return errnoBug(err),
		5874	.INVAL => \|err\| return errnoBug(err),
		5875	else => \|err\| return unexpectedErrno(err),
		5876	}
		5877	}
		5878	}
		5879
		5880	fn setsockopt(
		5881	ev: *Evented,
		5882	cancel_region: *CancelRegion,
		5883	fd: fd_t,
		5884	level: i32,
		5885	opt_name: u32,
		5886	option: u32,
		5887	) !void {
		5888	const o: []const u8 = @ptrCast(&option);
		5889	while (true) {
		5890	const off: extern struct {
		5891	cmd_op: linux.IO_URING_SOCKET_OP,
		5892	pad: u32,
		5893	} align(@alignOf(u64)) = .{
		5894	.cmd_op = .SETSOCKOPT,
		5895	.pad = 0,
		5896	};
		5897	const addr: extern struct { level: i32, opt_name: u32 } align(@alignOf(u64)) = .{
		5898	.level = level,
		5899	.opt_name = opt_name,
		5900	};
		5901	const thread = try cancel_region.awaitIoUring();
		5902	thread.enqueue().* = .{
		5903	.opcode = .URING_CMD,
		5904	.flags = 0,
		5905	.ioprio = 0,
		5906	.fd = fd,
		5907	.off = @as(const u64, @ptrCast(&off)).,
		5908	.addr = @as(const u64, @ptrCast(&addr)).,
		5909	.len = 0,
		5910	.rw_flags = 0,
		5911	.user_data = @intFromPtr(cancel_region.fiber),
		5912	.buf_index = 0,
		5913	.personality = 0,
		5914	.splice_fd_in = @intCast(o.len),
		5915	.addr3 = @intFromPtr(o.ptr),
		5916	.resv = 0,
		5917	};
		5918	ev.yield(null, .nothing);
		5919	switch (cancel_region.errno()) {
		5920	.SUCCESS => return,
		5921	.INTR, .CANCELED => continue,
		5922	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
		5923	.NOTSOCK => \|err\| return errnoBug(err),
		5924	.INVAL => \|err\| return errnoBug(err),
		5925	.FAULT => \|err\| return errnoBug(err),
		5926	else => \|err\| return unexpectedErrno(err),
		5927	}
		5928	}
		5929	}
		5930
		5931	fn socket(
		5932	ev: *Evented,
		5933	cancel_region: *CancelRegion,
		5934	family: linux.sa_family_t,
		5935	options: net.IpAddress.BindOptions,
		5936	) error{
		5937	AddressFamilyUnsupported,
		5938	ProtocolUnsupportedBySystem,
		5939	ProcessFdQuotaExceeded,
		5940	SystemFdQuotaExceeded,
		5941	SystemResources,
		5942	ProtocolUnsupportedByAddressFamily,
		5943	SocketModeUnsupported,
		5944	OptionUnsupported,
		5945	Unexpected,
		5946	Canceled,
		5947	}!fd_t {
		5948	const mode = posixSocketMode(options.mode);
		5949	const protocol = posixProtocol(options.protocol);
		5950	const socket_fd = while (true) {
		5951	const thread = try cancel_region.awaitIoUring();
		5952	thread.enqueue().* = .{
		5953	.opcode = .SOCKET,
		5954	.flags = 0,
		5955	.ioprio = 0,
		5956	.fd = family,
		5957	.off = mode \| linux.SOCK.CLOEXEC,
		5958	.addr = 0,
		5959	.len = protocol,
		5960	.rw_flags = 0,
		5961	.user_data = @intFromPtr(cancel_region.fiber),
		5962	.buf_index = 0,
		5963	.personality = 0,
		5964	.splice_fd_in = 0,
		5965	.addr3 = 0,
		5966	.resv = 0,
		5967	};
		5968	ev.yield(null, .nothing);
		5969	const completion = cancel_region.completion();
		5970	switch (completion.errno()) {
		5971	.SUCCESS => break completion.result,
		5972	.INTR, .CANCELED => continue,
		5973	.AFNOSUPPORT => return error.AddressFamilyUnsupported,
		5974	.INVAL => return error.ProtocolUnsupportedBySystem,
		5975	.MFILE => return error.ProcessFdQuotaExceeded,
		5976	.NFILE => return error.SystemFdQuotaExceeded,
		5977	.NOBUFS => return error.SystemResources,
		5978	.NOMEM => return error.SystemResources,
		5979	.PROTONOSUPPORT => return error.ProtocolUnsupportedByAddressFamily,
		5980	.PROTOTYPE => return error.SocketModeUnsupported,
		5981	else => \|err\| return unexpectedErrno(err),
		5982	}
		5983	};
		5984	errdefer ev.close(socket_fd);
		5985
		5986	if (options.ip6_only) {
		5987	if (linux.IPV6 == void) return error.OptionUnsupported;
		5988	try ev.setsockopt(cancel_region, socket_fd, linux.IPPROTO.IPV6, linux.IPV6.V6ONLY, 0);
		5989	}
		5990
		5991	return socket_fd;
		5992	}
		5993
		5994	fn stat(ev: Evented, cancel_region: CancelRegion, fd: fd_t) Dir.StatError!Dir.Stat {
		5995	return ev.statx(cancel_region, fd, "", linux.AT.EMPTY_PATH) catch \|err\| switch (err) {
		5996	error.BadPathName, error.NameTooLong => unreachable, // path is empty
		5997	error.AccessDenied => return errnoBug(.ACCES),
		5998	error.SymLinkLoop => return errnoBug(.LOOP),
		5999	error.FileNotFound => return errnoBug(.NOENT),
		6000	error.NotDir => return errnoBug(.NOTDIR),
		6001	else => \|e\| return e,
		6002	};
		6003	}
		6004
		6005	fn statx(
		6006	ev: *Evented,
		6007	cancel_region: *CancelRegion,
		6008	dir: fd_t,
		6009	path: [*:0]const u8,
		6010	flags: u32,
		6011	) (Dir.StatError \|\| Dir.PathNameError \|\| error{ FileNotFound, NotDir, SymLinkLoop })!Dir.Stat {
		6012	while (true) {
		6013	var statx_buf = std.mem.zeroes(linux.Statx);
		6014	const thread = try cancel_region.awaitIoUring();
		6015	thread.enqueue().* = .{
		6016	.opcode = .STATX,
		6017	.flags = 0,
		6018	.ioprio = 0,
		6019	.fd = dir,
		6020	.off = @intFromPtr(&statx_buf),
		6021	.addr = @intFromPtr(path),
		6022	.len = @bitCast(linux_statx_request),
		6023	.rw_flags = flags,
		6024	.user_data = @intFromPtr(cancel_region.fiber),
		6025	.buf_index = 0,
		6026	.personality = 0,
		6027	.splice_fd_in = 0,
		6028	.addr3 = 0,
		6029	.resv = 0,
		6030	};
		6031	ev.yield(null, .nothing);
		6032	switch (cancel_region.errno()) {
		6033	.SUCCESS => return statFromLinux(&statx_buf),
		6034	.INTR, .CANCELED => continue,
		6035	.ACCES => return error.AccessDenied,
		6036	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
		6037	.FAULT => \|err\| return errnoBug(err),
		6038	.INVAL => \|err\| return errnoBug(err),
		6039	.LOOP => return error.SymLinkLoop,
		6040	.NAMETOOLONG => \|err\| return errnoBug(err),
		6041	.NOENT => return error.FileNotFound,
		6042	.NOTDIR => return error.NotDir,
		6043	.NOMEM => return error.SystemResources,
		6044	else => \|err\| return unexpectedErrno(err),
		6045	}
		6046	}
		6047	}
		6048
		6049	fn urandomReadAll(
		6050	ev: *Evented,
		6051	cancel_region: *CancelRegion,
		6052	buffer: []u8,
		6053	) (File.OpenError \|\| File.Reader.Error \|\| error{EndOfStream})!void {
		6054	return ev.readAll(cancel_region, try ev.random_fd.open(ev, cancel_region, "/dev/urandom", .{
		6055	.ACCMODE = .RDONLY,
		6056	.CLOEXEC = true,
		6057	}), buffer);
		6058	}
		6059
		6060	fn utimensat(
		6061	ev: *Evented,
		6062	cancel_region: *CancelRegion,
		6063	dir: fd_t,
		6064	path: [*:0]const u8,
		6065	times: ?*const [2]linux.timespec,
		6066	flags: u32,
		6067	) File.SetTimestampsError!void {
		6068	_ = ev;
		6069	while (true) {
		6070	try cancel_region.await(.nothing);
		6071	switch (linux.errno(linux.utimensat(dir, path, times, flags))) {
		6072	.SUCCESS => return,
		6073	.INTR => continue,
		6074	.BADF => \|err\| return errnoBug(err), // always a race condition
		6075	.FAULT => \|err\| return errnoBug(err),
		6076	.INVAL => \|err\| return errnoBug(err),
		6077	.ACCES => return error.AccessDenied,
		6078	.PERM => return error.PermissionDenied,
		6079	.ROFS => return error.ReadOnlyFileSystem,
		6080	else => \|err\| return unexpectedErrno(err),
		6081	}
		6082	}
		6083	}
		6084
		6085	fn writeAll(
		6086	ev: *Evented,
		6087	cancel_region: *CancelRegion,
		6088	fd: fd_t,
		6089	buffer: []const u8,
		6090	) (File.Writer.Error \|\| error{EndOfStream})!void {
		6091	var index: usize = 0;
		6092	while (buffer.len - index != 0) {
		6093	const len = try ev.pwritev(cancel_region, fd, &.{
		6094	.{ .base = buffer[index..].ptr, .len = buffer.len - index },
		6095	}, null);
		6096	if (len == 0) return error.EndOfStream;
		6097	index += len;
		6098	}
		6099	}
		6100
		6101	test {
		6102	_ = Fiber.CancelProtection;
1497	}	6103	}

lib/std/Io/Threaded.zig+221-229

...	@@ -78,7 +78,7 @@ null_file: NullFile = .{},	...	@@ -78,7 +78,7 @@ null_file: NullFile = .{},
78	random_file: RandomFile = .{},	78	random_file: RandomFile = .{},
79	pipe_file: PipeFile = .{},	79	pipe_file: PipeFile = .{},
80		80
81	csprng: Csprng = .{},	81	csprng: Csprng = .uninitialized,
82		82
83	system_basic_information: SystemBasicInformation = .{},	83	system_basic_information: SystemBasicInformation = .{},
84		84
...	@@ -88,10 +88,12 @@ const SystemBasicInformation = if (!is_windows) struct {} else struct {	...	@@ -88,10 +88,12 @@ const SystemBasicInformation = if (!is_windows) struct {} else struct {
88	};	88	};
89		89
90	pub const Csprng = struct {	90	pub const Csprng = struct {
91	rng: std.Random.DefaultCsprng = .{	91	rng: std.Random.DefaultCsprng,
		92
		93	pub const uninitialized: Csprng = .{ .rng = .{
92	.state = undefined,	94	.state = undefined,
93	.offset = std.math.maxInt(usize),	95	.offset = std.math.maxInt(usize),
94	},	96	} };
95		97
96	pub const seed_len = std.Random.DefaultCsprng.secret_seed_length;	98	pub const seed_len = std.Random.DefaultCsprng.secret_seed_length;
97		99
...	@@ -120,7 +122,7 @@ pub const Argv0 = switch (native_os) {	...	@@ -120,7 +122,7 @@ pub const Argv0 = switch (native_os) {
120	},	122	},
121	};	123	};
122		124
123	const Environ = struct {	125	pub const Environ = struct {
124	/// Unmodified data directly from the OS.	126	/// Unmodified data directly from the OS.
125	process_environ: process.Environ,	127	process_environ: process.Environ,
126	/// Protected by `mutex`. Determines whether the other fields have been	128	/// Protected by `mutex`. Determines whether the other fields have been
...	@@ -157,6 +159,127 @@ const Environ = struct {	...	@@ -157,6 +159,127 @@ const Environ = struct {
157	HOME: ?[:0]const u8 = null,	159	HOME: ?[:0]const u8 = null,
158	},	160	},
159	};	161	};
		162
		163	pub fn scan(environ: *Environ, allocator: std.mem.Allocator) void {
		164	if (environ.initialized) return;
		165	environ.initialized = true;
		166
		167	if (is_windows) {
		168	// This value expires with any call that modifies the environment,
		169	// which is outside of this Io implementation's control, so references
		170	// must be short-lived.
		171	const peb = windows.peb();
		172	assert(windows.ntdll.RtlEnterCriticalSection(peb.FastPebLock) == .SUCCESS);
		173	defer assert(windows.ntdll.RtlLeaveCriticalSection(peb.FastPebLock) == .SUCCESS);
		174	const ptr = peb.ProcessParameters.Environment;
		175
		176	var i: usize = 0;
		177	while (ptr[i] != 0) {
		178	// There are some special environment variables that start with =,
		179	// so we need a special case to not treat = as a key/value separator
		180	// if it's the first character.
		181	// https://devblogs.microsoft.com/oldnewthing/20100506-00/?p=14133
		182	const key_start = i;
		183	if (ptr[i] == '=') i += 1;
		184	while (ptr[i] != 0 and ptr[i] != '=') : (i += 1) {}
		185	const key_w = ptr[key_start..i];
		186
		187	const value_start = i + 1;
		188	while (ptr[i] != 0) : (i += 1) {} // skip over '=' and value
		189	const value_w = ptr[value_start..i];
		190	i += 1; // skip over null byte
		191
		192	if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'N', 'O', '_', 'C', 'O', 'L', 'O', 'R' })) {
		193	environ.exist.NO_COLOR = true;
		194	} else if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'C', 'L', 'I', 'C', 'O', 'L', 'O', 'R', '_', 'F', 'O', 'R', 'C', 'E' })) {
		195	environ.exist.CLICOLOR_FORCE = true;
		196	} else if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'Z', 'I', 'G', '_', 'P', 'R', 'O', 'G', 'R', 'E', 'S', 'S' })) {
		197	environ.zig_progress_file = file: {
		198	var value_buf: [std.fmt.count("{d}", .{std.math.maxInt(usize)})]u8 = undefined;
		199	const len = std.unicode.calcWtf8Len(value_w);
		200	if (len > value_buf.len) break :file error.UnrecognizedFormat;
		201	assert(std.unicode.wtf16LeToWtf8(&value_buf, value_w) == len);
		202	break :file .{
		203	.handle = @ptrFromInt(std.fmt.parseInt(usize, value_buf[0..len], 10) catch
		204	break :file error.UnrecognizedFormat),
		205	.flags = .{ .nonblocking = true },
		206	};
		207	};
		208	}
		209	comptime assert(@sizeOf(String) == 0);
		210	}
		211	} else if (native_os == .wasi and !builtin.link_libc) {
		212	var environ_size: usize = undefined;
		213	var environ_buf_size: usize = undefined;
		214
		215	switch (std.os.wasi.environ_sizes_get(&environ_size, &environ_buf_size)) {
		216	.SUCCESS => {},
		217	else => \|err\| {
		218	environ.err = posix.unexpectedErrno(err);
		219	return;
		220	},
		221	}
		222	if (environ_size == 0) return;
		223
		224	const wasi_environ = allocator.alloc([*:0]u8, environ_size) catch \|err\| {
		225	environ.err = err;
		226	return;
		227	};
		228	defer allocator.free(wasi_environ);
		229	const wasi_environ_buf = allocator.alloc(u8, environ_buf_size) catch \|err\| {
		230	environ.err = err;
		231	return;
		232	};
		233	defer allocator.free(wasi_environ_buf);
		234
		235	switch (std.os.wasi.environ_get(wasi_environ.ptr, wasi_environ_buf.ptr)) {
		236	.SUCCESS => {},
		237	else => \|err\| {
		238	environ.err = posix.unexpectedErrno(err);
		239	return;
		240	},
		241	}
		242
		243	for (wasi_environ) \|env\| {
		244	const pair = std.mem.sliceTo(env, 0);
		245	var parts = std.mem.splitScalar(u8, pair, '=');
		246	const key = parts.first();
		247	if (std.mem.eql(u8, key, "NO_COLOR")) {
		248	environ.exist.NO_COLOR = true;
		249	} else if (std.mem.eql(u8, key, "CLICOLOR_FORCE")) {
		250	environ.exist.CLICOLOR_FORCE = true;
		251	}
		252	comptime assert(@sizeOf(String) == 0);
		253	}
		254	} else {
		255	for (environ.process_environ.block.slice) \|opt_entry\| {
		256	const entry = opt_entry.?;
		257	var entry_i: usize = 0;
		258	while (entry[entry_i] != 0 and entry[entry_i] != '=') : (entry_i += 1) {}
		259	const key = entry[0..entry_i];
		260
		261	var end_i: usize = entry_i;
		262	while (entry[end_i] != 0) : (end_i += 1) {}
		263	const value = entry[entry_i + 1 .. end_i :0];
		264
		265	if (std.mem.eql(u8, key, "NO_COLOR")) {
		266	environ.exist.NO_COLOR = true;
		267	} else if (std.mem.eql(u8, key, "CLICOLOR_FORCE")) {
		268	environ.exist.CLICOLOR_FORCE = true;
		269	} else if (std.mem.eql(u8, key, "ZIG_PROGRESS")) {
		270	environ.zig_progress_file = file: {
		271	break :file .{
		272	.handle = std.fmt.parseInt(u31, value, 10) catch
		273	break :file error.UnrecognizedFormat,
		274	.flags = .{ .nonblocking = true },
		275	};
		276	};
		277	} else inline for (@typeInfo(String).@"struct".fields) \|field\| {
		278	if (std.mem.eql(u8, key, field.name)) @field(environ.string, field.name) = value;
		279	}
		280	}
		281	}
		282	}
160	};	283	};
161		284
162	pub const NullFile = switch (native_os) {	285	pub const NullFile = switch (native_os) {
...	@@ -1397,13 +1520,13 @@ pub fn waitForApcOrAlert() void {	...	@@ -1397,13 +1520,13 @@ pub fn waitForApcOrAlert() void {
1397	_ = windows.ntdll.NtDelayExecution(windows.TRUE, &infinite_timeout);	1520	_ = windows.ntdll.NtDelayExecution(windows.TRUE, &infinite_timeout);
1398	}	1521	}
1399		1522
1400	const max_iovecs_len = 8;	1523	pub const max_iovecs_len = 8;
1401	const splat_buffer_size = 64;	1524	pub const splat_buffer_size = 64;
1402	/// Happens to be the same number that matches maximum number of handles that	1525	/// Happens to be the same number that matches maximum number of handles that
1403	/// NtWaitForMultipleObjects accepts. We use this value also for poll() on	1526	/// NtWaitForMultipleObjects accepts. We use this value also for poll() on
1404	/// posix systems.	1527	/// posix systems.
1405	const poll_buffer_len = 64;	1528	const poll_buffer_len = 64;
1406	const default_PATH = "/usr/local/bin:/bin/:/usr/bin";	1529	pub const default_PATH = "/usr/local/bin:/bin/:/usr/bin";
1407	/// There are multiple kernel bugs being worked around with retries.	1530	/// There are multiple kernel bugs being worked around with retries.
1408	const max_windows_kernel_bug_retries = 13;	1531	const max_windows_kernel_bug_retries = 13;
1409		1532
...	@@ -1588,7 +1711,7 @@ fn worker(t: *Threaded) void {	...	@@ -1588,7 +1711,7 @@ fn worker(t: *Threaded) void {
1588	.cancel_protection = .unblocked,	1711	.cancel_protection = .unblocked,
1589	.futex_waiter = undefined,	1712	.futex_waiter = undefined,
1590	.unpark_flag = unpark_flag_init,	1713	.unpark_flag = unpark_flag_init,
1591	.csprng = .{},	1714	.csprng = .uninitialized,
1592	};	1715	};
1593	Thread.current = &thread;	1716	Thread.current = &thread;
1594		1717
...	@@ -2563,12 +2686,12 @@ fn operate(userdata: ?*anyopaque, operation: Io.Operation) Io.Cancelable!Io.Oper	...	@@ -2563,12 +2686,12 @@ fn operate(userdata: ?*anyopaque, operation: Io.Operation) Io.Cancelable!Io.Oper
2563	fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {	2686	fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2564	const t: *Threaded = @ptrCast(@alignCast(userdata));	2687	const t: *Threaded = @ptrCast(@alignCast(userdata));
2565	if (is_windows) {	2688	if (is_windows) {
2566	batchAwaitWindows(b, false) catch \|err\| switch (err) {	2689	batchDrainSubmittedWindows(b, false) catch \|err\| switch (err) {
2567	error.ConcurrencyUnavailable => unreachable, // passed concurrency=false	2690	error.ConcurrencyUnavailable => unreachable, // passed concurrency=false
2568	else => \|e\| return e,	2691	else => \|e\| return e,
2569	};	2692	};
2570	const alertable_syscall = try AlertableSyscall.start();	2693	const alertable_syscall = try AlertableSyscall.start();
2571	while (b.pending.head != .none and b.completions.head == .none) waitForApcOrAlert();	2694	while (b.pending.head != .none and b.completed.head == .none) waitForApcOrAlert();
2572	alertable_syscall.finish();	2695	alertable_syscall.finish();
2573	return;	2696	return;
2574	}	2697	}
...	@@ -2576,7 +2699,7 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {	...	@@ -2576,7 +2699,7 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2576	var poll_buffer: [poll_buffer_len]posix.pollfd = undefined;	2699	var poll_buffer: [poll_buffer_len]posix.pollfd = undefined;
2577	var poll_len: u32 = 0;	2700	var poll_len: u32 = 0;
2578	{	2701	{
2579	var index = b.submissions.head;	2702	var index = b.submitted.head;
2580	while (index != .none and poll_len < poll_buffer_len) {	2703	while (index != .none and poll_len < poll_buffer_len) {
2581	const submission = &b.storage[index.toIndex()].submission;	2704	const submission = &b.storage[index.toIndex()].submission;
2582	switch (submission.operation) {	2705	switch (submission.operation) {
...	@@ -2605,7 +2728,7 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {	...	@@ -2605,7 +2728,7 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2605	1 => {},	2728	1 => {},
2606	else => while (true) {	2729	else => while (true) {
2607	const timeout_ms: i32 = t: {	2730	const timeout_ms: i32 = t: {
2608	if (b.completions.head != .none) {	2731	if (b.completed.head != .none) {
2609	// It is legal to call batchWait with already completed	2732	// It is legal to call batchWait with already completed
2610	// operations in the ring. In such case, we need to avoid	2733	// operations in the ring. In such case, we need to avoid
2611	// blocking in the poll syscall, but we can still take this	2734	// blocking in the poll syscall, but we can still take this
...	@@ -2620,7 +2743,7 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {	...	@@ -2620,7 +2743,7 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2620	switch (posix.errno(rc)) {	2743	switch (posix.errno(rc)) {
2621	.SUCCESS => {	2744	.SUCCESS => {
2622	if (rc == 0) {	2745	if (rc == 0) {
2623	if (b.completions.head != .none) {	2746	if (b.completed.head != .none) {
2624	// Since there are already completions available in the	2747	// Since there are already completions available in the
2625	// queue, this is neither a timeout nor a case for	2748	// queue, this is neither a timeout nor a case for
2626	// retrying.	2749	// retrying.
...	@@ -2629,7 +2752,7 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {	...	@@ -2629,7 +2752,7 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2629	continue;	2752	continue;
2630	}	2753	}
2631	var prev_index: Io.Operation.OptionalIndex = .none;	2754	var prev_index: Io.Operation.OptionalIndex = .none;
2632	var index = b.submissions.head;	2755	var index = b.submitted.head;
2633	for (poll_buffer[0..poll_len]) \|poll_entry\| {	2756	for (poll_buffer[0..poll_len]) \|poll_entry\| {
2634	const storage = &b.storage[index.toIndex()];	2757	const storage = &b.storage[index.toIndex()];
2635	const submission = &storage.submission;	2758	const submission = &storage.submission;
...	@@ -2638,17 +2761,17 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {	...	@@ -2638,17 +2761,17 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2638	const result = try operate(t, submission.operation);	2761	const result = try operate(t, submission.operation);
2639		2762
2640	switch (prev_index) {	2763	switch (prev_index) {
2641	.none => b.submissions.head = next_index,	2764	.none => b.submitted.head = next_index,
2642	else => b.storage[prev_index.toIndex()].submission.node.next = next_index,	2765	else => b.storage[prev_index.toIndex()].submission.node.next = next_index,
2643	}	2766	}
2644	if (next_index == .none) b.submissions.tail = prev_index;	2767	if (next_index == .none) b.submitted.tail = prev_index;
2645		2768
2646	switch (b.completions.tail) {	2769	switch (b.completed.tail) {
2647	.none => b.completions.head = index,	2770	.none => b.completed.head = index,
2648	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next = index,	2771	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next = index,
2649	}	2772	}
2650	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };	2773	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2651	b.completions.tail = index;	2774	b.completed.tail = index;
2652	} else prev_index = index;	2775	} else prev_index = index;
2653	index = next_index;	2776	index = next_index;
2654	}	2777	}
...	@@ -2662,10 +2785,10 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {	...	@@ -2662,10 +2785,10 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2662	}	2785	}
2663	}	2786	}
2664		2787
2665	var tail_index = b.completions.tail;	2788	var tail_index = b.completed.tail;
2666	defer b.completions.tail = tail_index;	2789	defer b.completed.tail = tail_index;
2667	var index = b.submissions.head;	2790	var index = b.submitted.head;
2668	errdefer b.submissions.head = index;	2791	errdefer b.submitted.head = index;
2669	while (index != .none) {	2792	while (index != .none) {
2670	const storage = &b.storage[index.toIndex()];	2793	const storage = &b.storage[index.toIndex()];
2671	const submission = &storage.submission;	2794	const submission = &storage.submission;
...	@@ -2673,22 +2796,22 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {	...	@@ -2673,22 +2796,22 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2673	const result = try operate(t, submission.operation);	2796	const result = try operate(t, submission.operation);
2674		2797
2675	switch (tail_index) {	2798	switch (tail_index) {
2676	.none => b.completions.head = index,	2799	.none => b.completed.head = index,
2677	else => b.storage[tail_index.toIndex()].completion.node.next = index,	2800	else => b.storage[tail_index.toIndex()].completion.node.next = index,
2678	}	2801	}
2679	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };	2802	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2680	tail_index = index;	2803	tail_index = index;
2681	index = next_index;	2804	index = next_index;
2682	}	2805	}
2683	b.submissions = .{ .head = .none, .tail = .none };	2806	b.submitted = .{ .head = .none, .tail = .none };
2684	}	2807	}
2685		2808
2686	fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout) Io.Batch.AwaitConcurrentError!void {	2809	fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout) Io.Batch.AwaitConcurrentError!void {
2687	const t: *Threaded = @ptrCast(@alignCast(userdata));	2810	const t: *Threaded = @ptrCast(@alignCast(userdata));
2688	if (is_windows) {	2811	if (is_windows) {
2689	const deadline: ?Io.Clock.Timestamp = timeout.toTimestamp(ioBasic(t));	2812	const deadline: ?Io.Clock.Timestamp = timeout.toTimestamp(ioBasic(t));
2690	try batchAwaitWindows(b, true);	2813	try batchDrainSubmittedWindows(b, true);
2691	while (b.pending.head != .none and b.completions.head == .none) {	2814	while (b.pending.head != .none and b.completed.head == .none) {
2692	var delay_interval: windows.LARGE_INTEGER = interval: {	2815	var delay_interval: windows.LARGE_INTEGER = interval: {
2693	const d = deadline orelse break :interval std.math.minInt(windows.LARGE_INTEGER);	2816	const d = deadline orelse break :interval std.math.minInt(windows.LARGE_INTEGER);
2694	break :interval timeoutToWindowsInterval(.{ .deadline = d }).?;	2817	break :interval timeoutToWindowsInterval(.{ .deadline = d }).?;
...	@@ -2701,7 +2824,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout	...	@@ -2701,7 +2824,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2701	// The thread woke due to the timeout. Although spurious	2824	// The thread woke due to the timeout. Although spurious
2702	// timeouts are OK, when no deadline is passed we must not	2825	// timeouts are OK, when no deadline is passed we must not
2703	// return `error.Timeout`.	2826	// return `error.Timeout`.
2704	if (timeout != .none and b.completions.head == .none) return error.Timeout;	2827	if (timeout != .none and b.completed.head == .none) return error.Timeout;
2705	},	2828	},
2706	else => {},	2829	else => {},
2707	}	2830	}
...	@@ -2743,7 +2866,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout	...	@@ -2743,7 +2866,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2743	}	2866	}
2744	} = .{ .gpa = t.allocator, .b = b, .slice = &poll_buffer, .len = 0 };	2867	} = .{ .gpa = t.allocator, .b = b, .slice = &poll_buffer, .len = 0 };
2745	{	2868	{
2746	var index = b.submissions.head;	2869	var index = b.submitted.head;
2747	while (index != .none) {	2870	while (index != .none) {
2748	const submission = &b.storage[index.toIndex()].submission;	2871	const submission = &b.storage[index.toIndex()].submission;
2749	switch (submission.operation) {	2872	switch (submission.operation) {
...	@@ -2757,18 +2880,18 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout	...	@@ -2757,18 +2880,18 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2757	switch (poll_storage.len) {	2880	switch (poll_storage.len) {
2758	0 => return,	2881	0 => return,
2759	1 => if (timeout == .none) {	2882	1 => if (timeout == .none) {
2760	const index = b.submissions.head;	2883	const index = b.submitted.head;
2761	const storage = &b.storage[index.toIndex()];	2884	const storage = &b.storage[index.toIndex()];
2762	const result = try operate(t, storage.submission.operation);	2885	const result = try operate(t, storage.submission.operation);
2763		2886
2764	b.submissions = .{ .head = .none, .tail = .none };	2887	b.submitted = .{ .head = .none, .tail = .none };
2765		2888
2766	switch (b.completions.tail) {	2889	switch (b.completed.tail) {
2767	.none => b.completions.head = index,	2890	.none => b.completed.head = index,
2768	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next = index,	2891	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next = index,
2769	}	2892	}
2770	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };	2893	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2771	b.completions.tail = index;	2894	b.completed.tail = index;
2772	return;	2895	return;
2773	},	2896	},
2774	else => {},	2897	else => {},
...	@@ -2777,7 +2900,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout	...	@@ -2777,7 +2900,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2777	const deadline = timeout.toTimestamp(t_io);	2900	const deadline = timeout.toTimestamp(t_io);
2778	while (true) {	2901	while (true) {
2779	const timeout_ms: i32 = t: {	2902	const timeout_ms: i32 = t: {
2780	if (b.completions.head != .none) {	2903	if (b.completed.head != .none) {
2781	// It is legal to call batchWait with already completed	2904	// It is legal to call batchWait with already completed
2782	// operations in the ring. In such case, we need to avoid	2905	// operations in the ring. In such case, we need to avoid
2783	// blocking in the poll syscall, but we can still take this	2906	// blocking in the poll syscall, but we can still take this
...	@@ -2794,7 +2917,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout	...	@@ -2794,7 +2917,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2794	switch (posix.errno(rc)) {	2917	switch (posix.errno(rc)) {
2795	.SUCCESS => {	2918	.SUCCESS => {
2796	if (rc == 0) {	2919	if (rc == 0) {
2797	if (b.completions.head != .none) {	2920	if (b.completed.head != .none) {
2798	// Since there are already completions available in the	2921	// Since there are already completions available in the
2799	// queue, this is neither a timeout nor a case for	2922	// queue, this is neither a timeout nor a case for
2800	// retrying.	2923	// retrying.
...	@@ -2806,7 +2929,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout	...	@@ -2806,7 +2929,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2806	return error.Timeout;	2929	return error.Timeout;
2807	}	2930	}
2808	var prev_index: Io.Operation.OptionalIndex = .none;	2931	var prev_index: Io.Operation.OptionalIndex = .none;
2809	var index = b.submissions.head;	2932	var index = b.submitted.head;
2810	for (poll_storage.slice[0..poll_storage.len]) \|poll_entry\| {	2933	for (poll_storage.slice[0..poll_storage.len]) \|poll_entry\| {
2811	const submission = &b.storage[index.toIndex()].submission;	2934	const submission = &b.storage[index.toIndex()].submission;
2812	const next_index = submission.node.next;	2935	const next_index = submission.node.next;
...	@@ -2814,17 +2937,20 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout	...	@@ -2814,17 +2937,20 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2814	const result = try operate(t, submission.operation);	2937	const result = try operate(t, submission.operation);
2815		2938
2816	switch (prev_index) {	2939	switch (prev_index) {
2817	.none => b.submissions.head = next_index,	2940	.none => b.submitted.head = next_index,
2818	else => b.storage[prev_index.toIndex()].submission.node.next = next_index,	2941	else => b.storage[prev_index.toIndex()].submission.node.next = next_index,
2819	}	2942	}
2820	if (next_index == .none) b.submissions.tail = prev_index;	2943	if (next_index == .none) b.submitted.tail = prev_index;
2821		2944
2822	switch (b.completions.tail) {	2945	switch (b.completed.tail) {
2823	.none => b.completions.head = index,	2946	.none => b.completed.head = index,
2824	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next = index,	2947	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next = index,
2825	}	2948	}
2826	b.completions.tail = index;	2949	b.completed.tail = index;
2827	b.storage[index.toIndex()] = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };	2950	b.storage[index.toIndex()] = .{ .completion = .{
		2951	.node = .{ .next = .none },
		2952	.result = result,
		2953	} };
2828	} else prev_index = index;	2954	} else prev_index = index;
2829	index = next_index;	2955	index = next_index;
2830	}	2956	}
...	@@ -2841,7 +2967,7 @@ const WindowsBatchPendingOperationContext = extern struct {	...	@@ -2841,7 +2967,7 @@ const WindowsBatchPendingOperationContext = extern struct {
2841	file: windows.HANDLE,	2967	file: windows.HANDLE,
2842	iosb: windows.IO_STATUS_BLOCK,	2968	iosb: windows.IO_STATUS_BLOCK,
2843		2969
2844	const Erased = [3]usize;	2970	const Erased = Io.Operation.Storage.Pending.Context;
2845		2971
2846	comptime {	2972	comptime {
2847	assert(@sizeOf(Erased) <= @sizeOf(WindowsBatchPendingOperationContext));	2973	assert(@sizeOf(Erased) <= @sizeOf(WindowsBatchPendingOperationContext));
...	@@ -2858,24 +2984,9 @@ const WindowsBatchPendingOperationContext = extern struct {	...	@@ -2858,24 +2984,9 @@ const WindowsBatchPendingOperationContext = extern struct {
2858		2984
2859	fn batchCancel(userdata: ?anyopaque, b: Io.Batch) void {	2985	fn batchCancel(userdata: ?anyopaque, b: Io.Batch) void {
2860	const t: *Threaded = @ptrCast(@alignCast(userdata));	2986	const t: *Threaded = @ptrCast(@alignCast(userdata));
2861	{
2862	var tail_index = b.unused.tail;
2863	defer b.unused.tail = tail_index;
2864	var index = b.submissions.head;
2865	errdefer b.submissions.head = index;
2866	while (index != .none) {
2867	const next_index = b.storage[index.toIndex()].submission.node.next;
2868	switch (tail_index) {
2869	.none => b.unused.head = index,
2870	else => b.storage[tail_index.toIndex()].unused.next = index,
2871	}
2872	b.storage[index.toIndex()] = .{ .unused = .{ .prev = tail_index, .next = .none } };
2873	tail_index = index;
2874	index = next_index;
2875	}
2876	b.submissions = .{ .head = .none, .tail = .none };
2877	}
2878	if (is_windows) {	2987	if (is_windows) {
		2988	if (b.pending.head == .none) return;
		2989	waitForApcOrAlert();
2879	var index = b.pending.head;	2990	var index = b.pending.head;
2880	while (index != .none) {	2991	while (index != .none) {
2881	const pending = &b.storage[index.toIndex()].pending;	2992	const pending = &b.storage[index.toIndex()].pending;
...	@@ -2889,10 +3000,13 @@ fn batchCancel(userdata: ?anyopaque, b: Io.Batch) void {	...	@@ -2889,10 +3000,13 @@ fn batchCancel(userdata: ?anyopaque, b: Io.Batch) void {
2889	t.allocator.free(@as([*]posix.pollfd, @ptrCast(@alignCast(context)))[0..b.storage.len]);	3000	t.allocator.free(@as([*]posix.pollfd, @ptrCast(@alignCast(context)))[0..b.storage.len]);
2890	b.context = null;	3001	b.context = null;
2891	}	3002	}
2892	assert(b.pending.head == .none);
2893	}	3003	}
2894		3004
2895	fn batchApc(apc_context: ?anyopaque, iosb: windows.IO_STATUS_BLOCK, _: windows.ULONG) callconv(.winapi) void {	3005	fn batchApc(
		3006	apc_context: ?*anyopaque,
		3007	iosb: *windows.IO_STATUS_BLOCK,
		3008	_: windows.ULONG,
		3009	) callconv(.winapi) void {
2896	const b: *Io.Batch = @ptrCast(@alignCast(apc_context));	3010	const b: *Io.Batch = @ptrCast(@alignCast(apc_context));
2897	const context: *WindowsBatchPendingOperationContext = @fieldParentPtr("iosb", iosb);	3011	const context: *WindowsBatchPendingOperationContext = @fieldParentPtr("iosb", iosb);
2898	const erased_context = context.toErased();	3012	const erased_context = context.toErased();
...	@@ -2918,11 +3032,12 @@ fn batchApc(apc_context: ?anyopaque, iosb: windows.IO_STATUS_BLOCK, _: windows	...	@@ -2918,11 +3032,12 @@ fn batchApc(apc_context: ?anyopaque, iosb: windows.IO_STATUS_BLOCK, _: windows
2918	b.unused.tail = .fromIndex(index);	3032	b.unused.tail = .fromIndex(index);
2919	},	3033	},
2920	else => {	3034	else => {
2921	switch (b.completions.tail) {	3035	switch (b.completed.tail) {
2922	.none => b.completions.head = .fromIndex(index),	3036	.none => b.completed.head = .fromIndex(index),
2923	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next = .fromIndex(index),	3037	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next =
		3038	.fromIndex(index),
2924	}	3039	}
2925	b.completions.tail = .fromIndex(index);	3040	b.completed.tail = .fromIndex(index);
2926	const result: Io.Operation.Result = switch (pending.tag) {	3041	const result: Io.Operation.Result = switch (pending.tag) {
2927	.file_read_streaming => .{ .file_read_streaming = ntReadFileResult(iosb) },	3042	.file_read_streaming => .{ .file_read_streaming = ntReadFileResult(iosb) },
2928	.file_write_streaming => .{ .file_write_streaming = ntWriteFileResult(iosb) },	3043	.file_write_streaming => .{ .file_write_streaming = ntWriteFileResult(iosb) },
...	@@ -2934,9 +3049,9 @@ fn batchApc(apc_context: ?anyopaque, iosb: windows.IO_STATUS_BLOCK, _: windows	...	@@ -2934,9 +3049,9 @@ fn batchApc(apc_context: ?anyopaque, iosb: windows.IO_STATUS_BLOCK, _: windows
2934	}	3049	}
2935		3050
2936	/// If `concurrency` is false, `error.ConcurrencyUnavailable` is unreachable.	3051	/// If `concurrency` is false, `error.ConcurrencyUnavailable` is unreachable.
2937	fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, ConcurrencyUnavailable }!void {	3052	fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentError \|\| Io.Cancelable)!void {
2938	var index = b.submissions.head;	3053	var index = b.submitted.head;
2939	errdefer b.submissions.head = index;	3054	errdefer b.submitted.head = index;
2940	while (index != .none) {	3055	while (index != .none) {
2941	const storage = &b.storage[index.toIndex()];	3056	const storage = &b.storage[index.toIndex()];
2942	const submission = storage.submission;	3057	const submission = storage.submission;
...	@@ -2952,7 +3067,7 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren	...	@@ -2952,7 +3067,7 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren
2952	b.pending.tail = index;	3067	b.pending.tail = index;
2953	const context: *WindowsBatchPendingOperationContext = .fromErased(&storage.pending.context);	3068	const context: *WindowsBatchPendingOperationContext = .fromErased(&storage.pending.context);
2954	errdefer {	3069	errdefer {
2955	context.iosb.u.Status = .CANCELLED;	3070	context.iosb = .{ .u = .{ .Status = .CANCELLED }, .Information = undefined };
2956	batchApc(b, &context.iosb, 0);	3071	batchApc(b, &context.iosb, 0);
2957	}	3072	}
2958	switch (submission.operation) {	3073	switch (submission.operation) {
...	@@ -2960,10 +3075,7 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren	...	@@ -2960,10 +3075,7 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren
2960	var data_index: usize = 0;	3075	var data_index: usize = 0;
2961	while (o.data.len - data_index != 0 and o.data[data_index].len == 0) data_index += 1;	3076	while (o.data.len - data_index != 0 and o.data[data_index].len == 0) data_index += 1;
2962	if (o.data.len - data_index == 0) {	3077	if (o.data.len - data_index == 0) {
2963	context.iosb = .{	3078	context.iosb = .{ .u = .{ .Status = .SUCCESS }, .Information = 0 };
2964	.u = .{ .Status = .SUCCESS },
2965	.Information = 0,
2966	};
2967	batchApc(b, &context.iosb, 0);	3079	batchApc(b, &context.iosb, 0);
2968	break :o;	3080	break :o;
2969	}	3081	}
...	@@ -3023,10 +3135,7 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren	...	@@ -3023,10 +3135,7 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren
3023	.file_write_streaming => \|o\| o: {	3135	.file_write_streaming => \|o\| o: {
3024	const buffer = windowsWriteBuffer(o.header, o.data, o.splat);	3136	const buffer = windowsWriteBuffer(o.header, o.data, o.splat);
3025	if (buffer.len == 0) {	3137	if (buffer.len == 0) {
3026	context.iosb = .{	3138	context.iosb = .{ .u = .{ .Status = .SUCCESS }, .Information = 0 };
3027	.u = .{ .Status = .SUCCESS },
3028	.Information = 0,
3029	};
3030	batchApc(b, &context.iosb, 0);	3139	batchApc(b, &context.iosb, 0);
3031	break :o;	3140	break :o;
3032	}	3141	}
...	@@ -3140,7 +3249,7 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren	...	@@ -3140,7 +3249,7 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren
3140	}	3249	}
3141	index = submission.node.next;	3250	index = submission.node.next;
3142	}	3251	}
3143	b.submissions = .{ .head = .none, .tail = .none };	3252	b.submitted = .{ .head = .none, .tail = .none };
3144	}	3253	}
3145		3254
3146	/// Since Windows only supports writing one contiguous buffer, returns the	3255	/// Since Windows only supports writing one contiguous buffer, returns the
...	@@ -3155,7 +3264,7 @@ fn windowsWriteBuffer(header: []const u8, data: []const []const u8, splat: usize	...	@@ -3155,7 +3264,7 @@ fn windowsWriteBuffer(header: []const u8, data: []const []const u8, splat: usize
3155	if (splat == 0) return &.{};	3264	if (splat == 0) return &.{};
3156	break :b data[data.len - 1];	3265	break :b data[data.len - 1];
3157	};	3266	};
3158	return buffer[0..@min(buffer.len, std.math.maxInt(u32))];	3267	return buffer[0..std.math.lossyCast(u32, buffer.len)];
3159	}	3268	}
3160		3269
3161	fn submitComplete(ring: []u32, complete_tail: *Io.Batch.RingIndex, op: u32) void {	3270	fn submitComplete(ring: []u32, complete_tail: *Io.Batch.RingIndex, op: u32) void {
...	@@ -4677,8 +4786,8 @@ fn atomicFileInit(	...	@@ -4677,8 +4786,8 @@ fn atomicFileInit(
4677	dir: Dir,	4786	dir: Dir,
4678	close_dir_on_deinit: bool,	4787	close_dir_on_deinit: bool,
4679	) Dir.CreateFileAtomicError!File.Atomic {	4788	) Dir.CreateFileAtomicError!File.Atomic {
4680	var random_integer: u64 = undefined;
4681	while (true) {	4789	while (true) {
		4790	var random_integer: u64 = undefined;
4682	t_io.random(@ptrCast(&random_integer));	4791	t_io.random(@ptrCast(&random_integer));
4683	const tmp_sub_path = std.fmt.hex(random_integer);	4792	const tmp_sub_path = std.fmt.hex(random_integer);
4684	const file = dir.createFile(t_io, &tmp_sub_path, .{	4793	const file = dir.createFile(t_io, &tmp_sub_path, .{
...	@@ -14317,11 +14426,11 @@ pub fn posixProtocol(protocol: ?net.Protocol) u32 {	...	@@ -14317,11 +14426,11 @@ pub fn posixProtocol(protocol: ?net.Protocol) u32 {
14317	return @intFromEnum(protocol orelse return 0);	14426	return @intFromEnum(protocol orelse return 0);
14318	}	14427	}
14319		14428
14320	fn recoverableOsBugDetected() void {	14429	pub fn recoverableOsBugDetected() void {
14321	if (is_debug) unreachable;	14430	if (is_debug) unreachable;
14322	}	14431	}
14323		14432
14324	fn clockToPosix(clock: Io.Clock) posix.clockid_t {	14433	pub fn clockToPosix(clock: Io.Clock) posix.clockid_t {
14325	return switch (clock) {	14434	return switch (clock) {
14326	.real => posix.CLOCK.REALTIME,	14435	.real => posix.CLOCK.REALTIME,
14327	.awake => switch (native_os) {	14436	.awake => switch (native_os) {
...	@@ -14355,7 +14464,7 @@ fn clockToWasi(clock: Io.Clock) std.os.wasi.clockid_t {	...	@@ -14355,7 +14464,7 @@ fn clockToWasi(clock: Io.Clock) std.os.wasi.clockid_t {
14355	};	14464	};
14356	}	14465	}
14357		14466
14358	const linux_statx_request: std.os.linux.STATX = .{	14467	pub const linux_statx_request: std.os.linux.STATX = .{
14359	.TYPE = true,	14468	.TYPE = true,
14360	.MODE = true,	14469	.MODE = true,
14361	.ATIME = true,	14470	.ATIME = true,
...	@@ -14367,7 +14476,7 @@ const linux_statx_request: std.os.linux.STATX = .{	...	@@ -14367,7 +14476,7 @@ const linux_statx_request: std.os.linux.STATX = .{
14367	.BLOCKS = true,	14476	.BLOCKS = true,
14368	};	14477	};
14369		14478
14370	const linux_statx_check: std.os.linux.STATX = .{	14479	pub const linux_statx_check: std.os.linux.STATX = .{
14371	.TYPE = true,	14480	.TYPE = true,
14372	.MODE = true,	14481	.MODE = true,
14373	.ATIME = false,	14482	.ATIME = false,
...	@@ -14379,7 +14488,7 @@ const linux_statx_check: std.os.linux.STATX = .{	...	@@ -14379,7 +14488,7 @@ const linux_statx_check: std.os.linux.STATX = .{
14379	.BLOCKS = false,	14488	.BLOCKS = false,
14380	};	14489	};
14381		14490
14382	fn statFromLinux(stx: *const std.os.linux.Statx) Io.UnexpectedError!File.Stat {	14491	pub fn statFromLinux(stx: *const std.os.linux.Statx) Io.UnexpectedError!File.Stat {
14383	const actual_mask_int: u32 = @bitCast(stx.mask);	14492	const actual_mask_int: u32 = @bitCast(stx.mask);
14384	const wanted_mask_int: u32 = @bitCast(linux_statx_check);	14493	const wanted_mask_int: u32 = @bitCast(linux_statx_check);
14385	if ((actual_mask_int \| wanted_mask_int) != actual_mask_int) return error.Unexpected;	14494	if ((actual_mask_int \| wanted_mask_int) != actual_mask_int) return error.Unexpected;
...	@@ -14470,11 +14579,11 @@ fn statFromWasi(st: *const std.os.wasi.filestat_t) File.Stat {	...	@@ -14470,11 +14579,11 @@ fn statFromWasi(st: *const std.os.wasi.filestat_t) File.Stat {
14470	};	14579	};
14471	}	14580	}
14472		14581
14473	fn timestampFromPosix(timespec: *const posix.timespec) Io.Timestamp {	14582	pub fn timestampFromPosix(timespec: *const posix.timespec) Io.Timestamp {
14474	return .{ .nanoseconds = nanosecondsFromPosix(timespec) };	14583	return .{ .nanoseconds = nanosecondsFromPosix(timespec) };
14475	}	14584	}
14476		14585
14477	fn nanosecondsFromPosix(timespec: *const posix.timespec) i96 {	14586	pub fn nanosecondsFromPosix(timespec: *const posix.timespec) i96 {
14478	return @intCast(@as(i128, timespec.sec) * std.time.ns_per_s + timespec.nsec);	14587	return @intCast(@as(i128, timespec.sec) * std.time.ns_per_s + timespec.nsec);
14479	}	14588	}
14480		14589
...	@@ -14492,7 +14601,7 @@ fn timestampToPosix(nanoseconds: i96) posix.timespec {	...	@@ -14492,7 +14601,7 @@ fn timestampToPosix(nanoseconds: i96) posix.timespec {
14492	};	14601	};
14493	}	14602	}
14494		14603
14495	fn setTimestampToPosix(set_ts: File.SetTimestamp) posix.timespec {	14604	pub fn setTimestampToPosix(set_ts: File.SetTimestamp) posix.timespec {
14496	return switch (set_ts) {	14605	return switch (set_ts) {
14497	.unchanged => .OMIT,	14606	.unchanged => .OMIT,
14498	.now => .NOW,	14607	.now => .NOW,
...	@@ -14500,7 +14609,7 @@ fn setTimestampToPosix(set_ts: File.SetTimestamp) posix.timespec {	...	@@ -14500,7 +14609,7 @@ fn setTimestampToPosix(set_ts: File.SetTimestamp) posix.timespec {
14500	};	14609	};
14501	}	14610	}
14502		14611
14503	fn pathToPosix(file_path: []const u8, buffer: *[posix.PATH_MAX]u8) Dir.PathNameError![:0]u8 {	14612	pub fn pathToPosix(file_path: []const u8, buffer: *[posix.PATH_MAX]u8) Dir.PathNameError![:0]u8 {
14504	if (std.mem.containsAtLeastScalar2(u8, file_path, 0, 1)) return error.BadPathName;	14613	if (std.mem.containsAtLeastScalar2(u8, file_path, 0, 1)) return error.BadPathName;
14505	// >= rather than > to make room for the null byte	14614	// >= rather than > to make room for the null byte
14506	if (file_path.len >= buffer.len) return error.NameTooLong;	14615	if (file_path.len >= buffer.len) return error.NameTooLong;
...	@@ -14996,126 +15105,7 @@ const WindowsEnvironStrings = struct {	...	@@ -14996,126 +15105,7 @@ const WindowsEnvironStrings = struct {
14996	fn scanEnviron(t: *Threaded) void {	15105	fn scanEnviron(t: *Threaded) void {
14997	mutexLock(&t.mutex);	15106	mutexLock(&t.mutex);
14998	defer mutexUnlock(&t.mutex);	15107	defer mutexUnlock(&t.mutex);
14999		15108	t.environ.scan(t.allocator);
15000	if (t.environ.initialized) return;
15001	t.environ.initialized = true;
15002
15003	if (is_windows) {
15004	// This value expires with any call that modifies the environment,
15005	// which is outside of this Io implementation's control, so references
15006	// must be short-lived.
15007	const peb = windows.peb();
15008	assert(windows.ntdll.RtlEnterCriticalSection(peb.FastPebLock) == .SUCCESS);
15009	defer assert(windows.ntdll.RtlLeaveCriticalSection(peb.FastPebLock) == .SUCCESS);
15010	const ptr = peb.ProcessParameters.Environment;
15011
15012	var i: usize = 0;
15013	while (ptr[i] != 0) {
15014
15015	// There are some special environment variables that start with =,
15016	// so we need a special case to not treat = as a key/value separator
15017	// if it's the first character.
15018	// https://devblogs.microsoft.com/oldnewthing/20100506-00/?p=14133
15019	const key_start = i;
15020	if (ptr[i] == '=') i += 1;
15021	while (ptr[i] != 0 and ptr[i] != '=') : (i += 1) {}
15022	const key_w = ptr[key_start..i];
15023
15024	const value_start = i + 1;
15025	while (ptr[i] != 0) : (i += 1) {} // skip over '=' and value
15026	const value_w = ptr[value_start..i];
15027	i += 1; // skip over null byte
15028
15029	if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'N', 'O', '_', 'C', 'O', 'L', 'O', 'R' })) {
15030	t.environ.exist.NO_COLOR = true;
15031	} else if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'C', 'L', 'I', 'C', 'O', 'L', 'O', 'R', '_', 'F', 'O', 'R', 'C', 'E' })) {
15032	t.environ.exist.CLICOLOR_FORCE = true;
15033	} else if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'Z', 'I', 'G', '_', 'P', 'R', 'O', 'G', 'R', 'E', 'S', 'S' })) {
15034	t.environ.zig_progress_file = file: {
15035	var value_buf: [std.fmt.count("{d}", .{std.math.maxInt(usize)})]u8 = undefined;
15036	const len = std.unicode.calcWtf8Len(value_w);
15037	if (len > value_buf.len) break :file error.UnrecognizedFormat;
15038	assert(std.unicode.wtf16LeToWtf8(&value_buf, value_w) == len);
15039	break :file .{
15040	.handle = @ptrFromInt(std.fmt.parseInt(usize, value_buf[0..len], 10) catch
15041	break :file error.UnrecognizedFormat),
15042	.flags = .{ .nonblocking = true },
15043	};
15044	};
15045	}
15046	comptime assert(@sizeOf(Environ.String) == 0);
15047	}
15048	} else if (native_os == .wasi and !builtin.link_libc) {
15049	var environ_count: usize = undefined;
15050	var environ_buf_size: usize = undefined;
15051
15052	switch (std.os.wasi.environ_sizes_get(&environ_count, &environ_buf_size)) {
15053	.SUCCESS => {},
15054	else => \|err\| {
15055	t.environ.err = posix.unexpectedErrno(err);
15056	return;
15057	},
15058	}
15059	if (environ_count == 0) return;
15060
15061	const environ = t.allocator.alloc([*:0]u8, environ_count) catch \|err\| {
15062	t.environ.err = err;
15063	return;
15064	};
15065	defer t.allocator.free(environ);
15066	const environ_buf = t.allocator.alloc(u8, environ_buf_size) catch \|err\| {
15067	t.environ.err = err;
15068	return;
15069	};
15070	defer t.allocator.free(environ_buf);
15071
15072	switch (std.os.wasi.environ_get(environ.ptr, environ_buf.ptr)) {
15073	.SUCCESS => {},
15074	else => \|err\| {
15075	t.environ.err = posix.unexpectedErrno(err);
15076	return;
15077	},
15078	}
15079
15080	for (environ) \|env\| {
15081	const pair = std.mem.sliceTo(env, 0);
15082	var parts = std.mem.splitScalar(u8, pair, '=');
15083	const key = parts.first();
15084	if (std.mem.eql(u8, key, "NO_COLOR")) {
15085	t.environ.exist.NO_COLOR = true;
15086	} else if (std.mem.eql(u8, key, "CLICOLOR_FORCE")) {
15087	t.environ.exist.CLICOLOR_FORCE = true;
15088	}
15089	comptime assert(@sizeOf(Environ.String) == 0);
15090	}
15091	} else {
15092	for (t.environ.process_environ.block.slice) \|opt_entry\| {
15093	const entry = opt_entry.?;
15094	var entry_i: usize = 0;
15095	while (entry[entry_i] != 0 and entry[entry_i] != '=') : (entry_i += 1) {}
15096	const key = entry[0..entry_i];
15097
15098	var end_i: usize = entry_i;
15099	while (entry[end_i] != 0) : (end_i += 1) {}
15100	const value = entry[entry_i + 1 .. end_i :0];
15101
15102	if (std.mem.eql(u8, key, "NO_COLOR")) {
15103	t.environ.exist.NO_COLOR = true;
15104	} else if (std.mem.eql(u8, key, "CLICOLOR_FORCE")) {
15105	t.environ.exist.CLICOLOR_FORCE = true;
15106	} else if (std.mem.eql(u8, key, "ZIG_PROGRESS")) {
15107	t.environ.zig_progress_file = file: {
15108	break :file .{
15109	.handle = std.fmt.parseInt(u31, value, 10) catch
15110	break :file error.UnrecognizedFormat,
15111	.flags = .{ .nonblocking = true },
15112	};
15113	};
15114	} else inline for (@typeInfo(Environ.String).@"struct".fields) \|field\| {
15115	if (std.mem.eql(u8, key, field.name)) @field(t.environ.string, field.name) = value;
15116	}
15117	}
15118	}
15119	}	15109	}
15120		15110
15121	fn processReplace(userdata: ?*anyopaque, options: process.ReplaceOptions) process.ReplaceError {	15111	fn processReplace(userdata: ?*anyopaque, options: process.ReplaceOptions) process.ReplaceError {
...	@@ -15213,17 +15203,17 @@ fn spawnPosix(t: *Threaded, options: process.SpawnOptions) process.SpawnError!Sp	...	@@ -15213,17 +15203,17 @@ fn spawnPosix(t: *Threaded, options: process.SpawnOptions) process.SpawnError!Sp
15213	const any_ignore = (options.stdin == .ignore or options.stdout == .ignore or options.stderr == .ignore);	15203	const any_ignore = (options.stdin == .ignore or options.stdout == .ignore or options.stderr == .ignore);
15214	const dev_null_fd = if (any_ignore) try getDevNullFd(t) else undefined;	15204	const dev_null_fd = if (any_ignore) try getDevNullFd(t) else undefined;
15215		15205
15216	const prog_pipe: [2]posix.fd_t = if (options.progress_node.index != .none)	15206	const prog_pipe: [2]posix.fd_t = if (options.progress_node.index != .none) pipe: {
15217	// We use CLOEXEC for the same reason as in `pipe_flags`.	15207	// We use CLOEXEC for the same reason as in `pipe_flags`.
15218	try pipe2(.{ .NONBLOCK = true, .CLOEXEC = true })	15208	const pipe = try pipe2(.{ .NONBLOCK = true, .CLOEXEC = true });
15219	else	15209	switch (native_os) {
15220	.{ -1, -1 };	15210	.linux => _ = posix.system.fcntl(pipe[0], posix.F.SETPIPE_SZ, @as(u32, std.Progress.max_packet_len * 2)),
		15211	else => {},
		15212	}
		15213	break :pipe pipe;
		15214	} else .{ -1, -1 };
15221	errdefer destroyPipe(prog_pipe);	15215	errdefer destroyPipe(prog_pipe);
15222		15216
15223	if (native_os == .linux and prog_pipe[0] != -1) {
15224	_ = posix.system.fcntl(prog_pipe[0], posix.F.SETPIPE_SZ, @as(u32, std.Progress.max_packet_len * 2));
15225	}
15226
15227	var arena_allocator = std.heap.ArenaAllocator.init(t.allocator);	15217	var arena_allocator = std.heap.ArenaAllocator.init(t.allocator);
15228	defer arena_allocator.deinit();	15218	defer arena_allocator.deinit();
15229	const arena = arena_allocator.allocator();	15219	const arena = arena_allocator.allocator();
...	@@ -17241,16 +17231,7 @@ fn randomMainThread(t: *Threaded, buffer: []u8) void {	...	@@ -17241,16 +17231,7 @@ fn randomMainThread(t: *Threaded, buffer: []u8) void {
17241		17231
17242	randomSecure(t, &seed) catch \|err\| switch (err) {	17232	randomSecure(t, &seed) catch \|err\| switch (err) {
17243	error.Canceled => unreachable,	17233	error.Canceled => unreachable,
17244	error.EntropyUnavailable => {	17234	error.EntropyUnavailable => fallbackSeed(t, &seed),
17245	@memset(&seed, 0);
17246	const aslr_addr = @intFromPtr(t);
17247	std.mem.writeInt(usize, seed[seed.len - @sizeOf(usize) ..][0..@sizeOf(usize)], aslr_addr, .native);
17248	switch (native_os) {
17249	.windows => fallbackSeedWindows(&seed),
17250	.wasi => if (builtin.link_libc) fallbackSeedPosix(&seed) else fallbackSeedWasi(&seed),
17251	else => fallbackSeedPosix(&seed),
17252	}
17253	},
17254	};	17235	};
17255	}	17236	}
17256	t.csprng.rng = .init(seed);	17237	t.csprng.rng = .init(seed);
...	@@ -17259,6 +17240,17 @@ fn randomMainThread(t: *Threaded, buffer: []u8) void {	...	@@ -17259,6 +17240,17 @@ fn randomMainThread(t: *Threaded, buffer: []u8) void {
17259	t.csprng.rng.fill(buffer);	17240	t.csprng.rng.fill(buffer);
17260	}	17241	}
17261		17242
		17243	pub fn fallbackSeed(aslr_addr: ?anyopaque, seed: [Csprng.seed_len]u8) void {
		17244	@memset(seed, 0);
		17245	std.mem.writeInt(usize, seed[seed.len - @sizeOf(usize) ..][0..@sizeOf(usize)], @intFromPtr(aslr_addr), .native);
		17246	const fallbackSeedImpl = switch (native_os) {
		17247	.windows => fallbackSeedWindows,
		17248	.wasi => if (builtin.link_libc) fallbackSeedPosix else fallbackSeedWasi,
		17249	else => fallbackSeedPosix,
		17250	};
		17251	fallbackSeedImpl(seed);
		17252	}
		17253
17262	fn fallbackSeedPosix(seed: *[Csprng.seed_len]u8) void {	17254	fn fallbackSeedPosix(seed: *[Csprng.seed_len]u8) void {
17263	std.mem.writeInt(posix.pid_t, seed[0..@sizeOf(posix.pid_t)], posix.system.getpid(), .native);	17255	std.mem.writeInt(posix.pid_t, seed[0..@sizeOf(posix.pid_t)], posix.system.getpid(), .native);
17264	const i_1 = @sizeOf(posix.pid_t);	17256	const i_1 = @sizeOf(posix.pid_t);

lib/std/os/linux.zig+6-3

lib/std/process.zig+3-3

lib/std/tar.zig+4-4

...	@@ -1128,10 +1128,10 @@ fn filePermissions(mode: u32, options: PipeOptions) Io.File.Permissions {	...	@@ -1128,10 +1128,10 @@ fn filePermissions(mode: u32, options: PipeOptions) Io.File.Permissions {
1128		1128
1129	test filePermissions {	1129	test filePermissions {
1130	if (!Io.File.Permissions.has_executable_bit) return error.SkipZigTest;	1130	if (!Io.File.Permissions.has_executable_bit) return error.SkipZigTest;
1131	try testing.expectEqual(.default_file, filePermissions(0o744, .{ .mode_mode = .ignore }));	1131	try testing.expectEqual(Io.File.Permissions.default_file, filePermissions(0o744, .{ .mode_mode = .ignore }));
1132	try testing.expectEqual(.executable_file, filePermissions(0o744, .{}));	1132	try testing.expectEqual(Io.File.Permissions.executable_file, filePermissions(0o744, .{}));
1133	try testing.expectEqual(.default_file, filePermissions(0o644, .{}));	1133	try testing.expectEqual(Io.File.Permissions.default_file, filePermissions(0o644, .{}));
1134	try testing.expectEqual(.default_file, filePermissions(0o655, .{}));	1134	try testing.expectEqual(Io.File.Permissions.default_file, filePermissions(0o655, .{}));
1135	}	1135	}
1136		1136
1137	test "executable bit" {	1137	test "executable bit" {

src/Compilation.zig+13-33

 @@ -4891,11 +4891,7 @@ fn performAllTheWork(
     work: while (true) {
         for (&comp.work_queues) |*work_queue| if (work_queue.popFront()) |job| {
-            try processOneJob(
+            try processOneJob(.main, comp, job);
-                @intFromEnum(Zcu.PerThread.Id.main),
-                comp,
-                job,
-            );
             continue :work;
         };
         if (comp.zcu) |zcu| {
 @@ -5160,11 +5156,7 @@ pub fn queueJobs(comp: *Compilation, jobs: []const Job) !void {
     for (jobs) |job| try comp.queueJob(job);
 }
-fn processOneJob(
+fn processOneJob(tid: Zcu.PerThread.Id, comp: *Compilation, job: Job) JobError!void {
-    tid: usize,
-    comp: *Compilation,
-    job: Job,
-) JobError!void {
     switch (job) {
         .codegen_func => |func| {
             const zcu = comp.zcu.?;
 @@ -5232,7 +5224,7 @@ fn processOneJob(
             const named_frame = tracy.namedFrame("analyze_func");
             defer named_frame.end();
-            const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
+            const pt: Zcu.PerThread = .activate(comp.zcu.?, tid);
             defer pt.deactivate();
             pt.ensureFuncBodyUpToDate(func) catch |err| switch (err) {
 @@ -5245,7 +5237,7 @@ fn processOneJob(
             const named_frame = tracy.namedFrame("analyze_comptime_unit");
             defer named_frame.end();
-            const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
+            const pt: Zcu.PerThread = .activate(comp.zcu.?, tid);
             defer pt.deactivate();
             const maybe_err: Zcu.SemaError!void = switch (unit.unwrap()) {
 @@ -5285,7 +5277,7 @@ fn processOneJob(
             const named_frame = tracy.namedFrame("resolve_type_fully");
             defer named_frame.end();
-            const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
+            const pt: Zcu.PerThread = .activate(comp.zcu.?, tid);
             defer pt.deactivate();
             Type.fromInterned(ty).resolveFully(pt) catch |err| switch (err) {
                 error.OutOfMemory, error.Canceled => |e| return e,
 @@ -5296,7 +5288,7 @@ fn processOneJob(
             const named_frame = tracy.namedFrame("analyze_mod");
             defer named_frame.end();
-            const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
+            const pt: Zcu.PerThread = .activate(comp.zcu.?, tid);
             defer pt.deactivate();
             pt.semaMod(mod) catch |err| switch (err) {
                 error.OutOfMemory, error.Canceled => |e| return e,
 @@ -5642,13 +5634,14 @@ fn workerUpdateFile(
     prog_node: std.Progress.Node,
     group: *Io.Group,
 ) void {
-    const tid = Compilation.getTid();
     const io = comp.io;
+    const tid: Zcu.PerThread.Id = .acquire(io);
+    defer tid.release(io);
     const child_prog_node = prog_node.start(fs.path.basename(file.path.sub_path), 0);
     defer child_prog_node.end();
-    const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
+    const pt: Zcu.PerThread = .activate(comp.zcu.?, tid);
     defer pt.deactivate();
     pt.updateFile(file_index, file) catch |err| {
         pt.reportRetryableFileError(file_index, "unable to load '{s}': {s}", .{ fs.path.basename(file.path.sub_path), @errorName(err) }) catch |oom| switch (oom) {
 @@ -5708,9 +5701,10 @@ fn workerUpdateBuiltinFile(comp: *Compilation, file: *Zcu.File) void {
 }
 fn workerUpdateEmbedFile(comp: *Compilation, ef_index: Zcu.EmbedFile.Index, ef: *Zcu.EmbedFile) void {
-    const tid = Compilation.getTid();
     const io = comp.io;
-    comp.detectEmbedFileUpdate(@enumFromInt(tid), ef_index, ef) catch |err| switch (err) {
+    const tid: Zcu.PerThread.Id = .acquire(io);
+    defer tid.release(io);
+    comp.detectEmbedFileUpdate(tid, ef_index, ef) catch |err| switch (err) {
         error.OutOfMemory => {
             comp.mutex.lockUncancelable(io);
             defer comp.mutex.unlock(io);
 @@ -5868,7 +5862,7 @@ pub fn translateC(
     }
     var stdout: []u8 = undefined;
-    try @import("main.zig").translateC(gpa, arena, io, argv.items, environ_map, prog_node, &stdout);
+    try @import("main.zig").translateC(gpa, arena, io, argv.items, environ_map, prog_node, comp.thread_limit, &stdout);
     if (out_dep_path) |dep_file_path| add_deps: {
         if (comp.verbose_cimport) log.info("processing dep file at {s}", .{dep_file_path});
 @@ -8394,17 +8388,3 @@ pub fn compilerRtOptMode(comp: Compilation) std.builtin.OptimizeMode {
 pub fn compilerRtStrip(comp: Compilation) bool {
     return comp.root_mod.strip;
 }
-/// This is a temporary workaround put in place to migrate from `std.Thread.Pool`
-/// to `std.Io.Threaded` for asynchronous/concurrent work. The eventual solution
-/// will likely involve significant changes to the `InternPool` implementation.
-pub fn getTid() usize {
-    if (my_tid == null) my_tid = next_tid.fetchAdd(1, .monotonic);
-    return my_tid.?;
-}
-pub fn setMainThread() void {
-    my_tid = 0;
-}
-/// TID 0 is reserved for the main thread.
-var next_tid: std.atomic.Value(usize) = .init(1);
-threadlocal var my_tid: ?usize = null;

src/InternPool.zig+6-7

 @@ -3,6 +3,7 @@
 const InternPool = @This();
 const builtin = @import("builtin");
+const build_options = @import("build_options");
 const std = @import("std");
 const Io = std.Io;
 @@ -86,13 +87,11 @@ dep_entries: std.ArrayList(DepEntry),
 /// garbage collection pass.
 free_dep_entries: std.ArrayList(DepEntry.Index),
-/// Whether a multi-threaded intern pool is useful.
-/// Currently `false` until the intern pool is actually accessed
-/// from multiple threads to reduce the cost of this data structure.
-const want_multi_threaded = true;
 /// Whether a single-threaded intern pool impl is in use.
-pub const single_threaded = builtin.single_threaded or !want_multi_threaded;
+pub const single_threaded = switch (build_options.io_mode) {
+    .threaded => builtin.single_threaded,
+    .evented => false, // even without threads, evented can be access from multiple tasks at a time
+};
 pub const empty: InternPool = .{
     .locals = &.{},
 @@ -6915,7 +6914,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, io: Io, available_threads: usize) !
     assert(ip.locals.len == 0 and ip.shards.len == 0);
     assert(available_threads > 0 and available_threads <= std.math.maxInt(u8));
-    const used_threads = if (single_threaded) 1 else available_threads;
+    const used_threads = if (single_threaded) 1 else @max(available_threads, 2);
     ip.locals = try gpa.alloc(Local, used_threads);
     @memset(ip.locals, .{
         .shared = .{

src/Zcu.zig+8-4

 @@ -4954,8 +4954,10 @@ pub const CodegenTaskPool = struct {
         // We own `air` now, so we are responsbile for freeing it.
         var air = orig_air;
         defer air.deinit(zcu.comp.gpa);
-        const tid = Compilation.getTid();
+        const io = zcu.comp.io;
-        const pt: Zcu.PerThread = .activate(zcu, @enumFromInt(tid));
+        const tid: Zcu.PerThread.Id = .acquire(io);
+        defer tid.release(io);
+        const pt: Zcu.PerThread = .activate(zcu, tid);
         defer pt.deactivate();
         return pt.runCodegen(func_index, &air);
     }
 @@ -4964,8 +4966,10 @@ pub const CodegenTaskPool = struct {
         func_index: InternPool.Index,
         air: *Air,
     ) CodegenResult {
-        const tid = Compilation.getTid();
+        const io = zcu.comp.io;
-        const pt: Zcu.PerThread = .activate(zcu, @enumFromInt(tid));
+        const tid: Zcu.PerThread.Id = .acquire(io);
+        defer tid.release(io);
+        const pt: Zcu.PerThread = .activate(zcu, tid);
         defer pt.deactivate();
         return pt.runCodegen(func_index, air);
     }

src/Zcu/PerThread.zig+75-2

...	@@ -41,13 +41,86 @@ zcu: *Zcu,	...	@@ -41,13 +41,86 @@ zcu: *Zcu,
41	tid: Id,	41	tid: Id,
42		42
43	pub const IdBacking = u7;	43	pub const IdBacking = u7;
44	pub const Id = if (InternPool.single_threaded) enum { main } else enum(IdBacking) { main, _ };	44	pub const Id = if (InternPool.single_threaded) enum {
		45	main,
		46
		47	pub fn allocate(arena: Allocator, n: usize) Allocator.Error!void {
		48	_ = arena;
		49	_ = n;
		50	}
		51	pub fn acquire(io: std.Io) Id {
		52	_ = io;
		53	return .main;
		54	}
		55	pub fn release(tid: Id, io: std.Io) void {
		56	_ = io;
		57	_ = tid;
		58	}
		59	} else enum(IdBacking) {
		60	main,
		61	_,
		62
		63	var tid_mutex: std.Io.Mutex = .init;
		64	var tid_cond: std.Io.Condition = .init;
		65	/// This is a temporary workaround put in place to migrate from `std.Thread.Pool`
		66	/// to `std.Io.Threaded` for asynchronous/concurrent work. The eventual solution
		67	/// will likely involve significant changes to the `InternPool` implementation.
		68	var available_tids: std.ArrayList(Id) = .empty;
		69	threadlocal var recursive_depth: usize = 0;
		70	threadlocal var recursive_tid: Id = .main;
		71
		72	pub fn allocate(arena: Allocator, n: usize) Allocator.Error!void {
		73	assert(available_tids.items.len == 0);
		74	try available_tids.ensureTotalCapacityPrecise(arena, n - 1);
		75	for (1..n) \|tid\| available_tids.appendAssumeCapacity(@enumFromInt(tid));
		76	}
		77	pub fn acquire(io: std.Io) Id {
		78	switch (build_options.io_mode) {
		79	.threaded => {
		80	recursive_depth += 1;
		81	if (recursive_depth > 1) {
		82	assert(recursive_tid != .main);
		83	return recursive_tid;
		84	}
		85	},
		86	.evented => {},
		87	}
		88	tid_mutex.lockUncancelable(io);
		89	defer tid_mutex.unlock(io);
		90	while (true) {
		91	if (available_tids.pop()) \|tid\| {
		92	switch (build_options.io_mode) {
		93	.threaded => recursive_tid = tid,
		94	.evented => {},
		95	}
		96	return tid;
		97	}
		98	tid_cond.waitUncancelable(io, &tid_mutex);
		99	}
		100	}
		101	pub fn release(tid: Id, io: std.Io) void {
		102	switch (build_options.io_mode) {
		103	.threaded => {
		104	assert(recursive_tid == tid);
		105	recursive_depth -= 1;
		106	if (recursive_depth > 0) return;
		107	recursive_tid = .main;
		108	},
		109	.evented => {},
		110	}
		111	{
		112	tid_mutex.lockUncancelable(io);
		113	defer tid_mutex.unlock(io);
		114	available_tids.appendAssumeCapacity(tid);
		115	}
		116	tid_cond.signal(io);
		117	}
		118	};
45		119
46	pub fn activate(zcu: *Zcu, tid: Id) Zcu.PerThread {	120	pub fn activate(zcu: *Zcu, tid: Id) Zcu.PerThread {
47	zcu.intern_pool.activate();	121	zcu.intern_pool.activate();
48	return .{ .zcu = zcu, .tid = tid };	122	return .{ .zcu = zcu, .tid = tid };
49	}	123	}
50
51	pub fn deactivate(pt: Zcu.PerThread) void {	124	pub fn deactivate(pt: Zcu.PerThread) void {
52	pt.zcu.intern_pool.deactivate();	125	pt.zcu.intern_pool.deactivate();
53	}	126	}

src/crash_report.zig+9-38

 @@ -1,34 +1,14 @@
-/// We override the panic implementation to our own one, so we can print our own information before
-/// calling the default panic handler. This declaration must be re-exposed from `@import("root")`.
-pub const panic = if (dev.env == .bootstrap)
-    std.debug.simple_panic
-else
-    std.debug.FullPanic(panicImpl);
-/// We let std install its segfault handler, but we override the target-agnostic handler it calls,
-/// so we can print our own information before calling the default segfault logic. This declaration
-/// must be re-exposed from `@import("root")`.
-pub const debug = struct {
-    pub const handleSegfault = handleSegfaultImpl;
-};
 /// Printed in panic messages when suggesting a command to run, allowing copy-pasting the command.
 /// Set by `main` as soon as arguments are known. The value here is a default in case we somehow
 /// crash earlier than that.
 pub var zig_argv0: []const u8 = "zig";
-fn handleSegfaultImpl(addr: ?usize, name: []const u8, opt_ctx: ?std.debug.CpuContextPtr) noreturn {
+const enabled = switch (build_options.io_mode) {
-    @branchHint(.cold);
+    .threaded => build_options.enable_debug_extensions,
-    dumpCrashContext() catch {};
+    .evented => false, // would use threadlocals in a way incompatible with evented
-    std.debug.defaultHandleSegfault(addr, name, opt_ctx);
+};
-}
-fn panicImpl(msg: []const u8, first_trace_addr: ?usize) noreturn {
-    @branchHint(.cold);
-    dumpCrashContext() catch {};
-    std.debug.defaultPanic(msg, first_trace_addr orelse @returnAddress());
-}
-pub const AnalyzeBody = if (build_options.enable_debug_extensions) struct {
+pub const AnalyzeBody = if (enabled) struct {
     parent: ?*AnalyzeBody,
     sema: *Sema,
     block: *Sema.Block,
 @@ -63,7 +43,7 @@ pub const AnalyzeBody = if (build_options.enable_debug_extensions) struct {
     pub inline fn setBodyIndex(_: @This(), _: usize) void {}
 };
-pub const CodegenFunc = if (build_options.enable_debug_extensions) struct {
+pub const CodegenFunc = if (enabled) struct {
     zcu: *const Zcu,
     func_index: InternPool.Index,
     threadlocal var current: ?CodegenFunc = null;
 @@ -82,23 +62,14 @@ pub const CodegenFunc = if (build_options.enable_debug_extensions) struct {
     pub fn stop(_: InternPool.Index) void {}
 };
-fn dumpCrashContext() Io.Writer.Error!void {
+pub fn dumpCrashContext(terminal: Io.Terminal) Io.Writer.Error!void {
     const S = struct {
-        /// In the case of recursive panics or segfaults, don't print the context for a second time.
-        threadlocal var already_dumped = false;
         /// TODO: make this unnecessary. It exists because `print_zir` currently needs an allocator,
         /// but that shouldn't be necessary---it's already only used in one place.
-        threadlocal var crash_heap: [64 * 1024]u8 = undefined;
+        var crash_heap: [64 * 1024]u8 = undefined;
     };
-    if (S.already_dumped) return;
-    S.already_dumped = true;
-    // TODO: this does mean that a different thread could grab the stderr mutex between the context
-    // and the actual panic printing, which would be quite confusing.
-    const stderr = std.debug.lockStderr(&.{});
-    defer std.debug.unlockStderr();
-    const w = &stderr.file_writer.interface;
+    const w = terminal.writer;
     try w.writeAll("Compiler crash context:\n");
     if (CodegenFunc.current) |*cg| {

src/introspect.zig+1-5

 @@ -54,11 +54,7 @@ pub fn findZigLibDir(gpa: Allocator, io: Io) !Cache.Directory {
 /// Like `std.process.currentPathAlloc`, but also resolves the path with `Dir.path.resolve`. This
 /// means the path has no repeated separators, no "." or ".." components, and no trailing separator.
 /// On WASI, "" is returned instead of ".".
-pub fn getResolvedCwd(io: Io, gpa: Allocator) error{
+pub fn getResolvedCwd(io: Io, gpa: Allocator) std.process.CurrentPathAllocError![]u8 {
-    OutOfMemory,
-    CurrentDirUnlinked,
-    Unexpected,
-}![]u8 {
     if (builtin.target.os.tag == .wasi) {
         if (std.debug.runtime_safety) {
             const cwd = try std.process.currentPathAlloc(io, gpa);

src/link.zig+4-4

...	@@ -1500,12 +1500,12 @@ pub fn doPrelinkTask(comp: *Compilation, task: PrelinkTask) void {	...	@@ -1500,12 +1500,12 @@ pub fn doPrelinkTask(comp: *Compilation, task: PrelinkTask) void {
1500	},	1500	},
1501	}	1501	}
1502	}	1502	}
1503	pub fn doZcuTask(comp: *Compilation, tid: usize, task: ZcuTask) void {	1503	pub fn doZcuTask(comp: *Compilation, tid: Zcu.PerThread.Id, task: ZcuTask) void {
1504	const io = comp.io;	1504	const io = comp.io;
1505	const diags = &comp.link_diags;	1505	const diags = &comp.link_diags;
1506	const zcu = comp.zcu.?;	1506	const zcu = comp.zcu.?;
1507	const ip = &zcu.intern_pool;	1507	const ip = &zcu.intern_pool;
1508	const pt: Zcu.PerThread = .activate(zcu, @enumFromInt(tid));	1508	const pt: Zcu.PerThread = .activate(zcu, tid);
1509	defer pt.deactivate();	1509	defer pt.deactivate();
1510		1510
1511	var timer = comp.startTimer();	1511	var timer = comp.startTimer();
...	@@ -1610,8 +1610,8 @@ pub fn doZcuTask(comp: *Compilation, tid: usize, task: ZcuTask) void {	...	@@ -1610,8 +1610,8 @@ pub fn doZcuTask(comp: *Compilation, tid: usize, task: ZcuTask) void {
1610	}	1610	}
1611	}	1611	}
1612	}	1612	}
1613	pub fn doIdleTask(comp: *Compilation, tid: usize) error{ OutOfMemory, LinkFailure }!bool {	1613	pub fn doIdleTask(comp: *Compilation, tid: Zcu.PerThread.Id) error{ OutOfMemory, LinkFailure }!bool {
1614	return if (comp.bin_file) \|lf\| lf.idle(@enumFromInt(tid)) else false;	1614	return if (comp.bin_file) \|lf\| lf.idle(tid) else false;
1615	}	1615	}
1616	/// After the main pipeline is done, but before flush, the compilation may need to link one final	1616	/// After the main pipeline is done, but before flush, the compilation may need to link one final
1617	/// `Nav` into the binary: the `builtin.test_functions` value. Since the link thread isn't running	1617	/// `Nav` into the binary: the `builtin.test_functions` value. Since the link thread isn't running

src/link/Queue.zig+7-5

 @@ -96,12 +96,12 @@ pub fn enqueuePrelink(q: *Queue, comp: *Compilation, tasks: []const PrelinkTask)
 pub fn enqueueZcu(
     q: *Queue,
     comp: *Compilation,
-    tid: usize,
+    tid: Zcu.PerThread.Id,
     task: ZcuTask,
 ) Io.Cancelable!void {
     const io = comp.io;
-    assert(tid == 0);
+    assert(tid == .main);
     if (q.future != null) {
         if (q.zcu_queue.putOne(io, task)) |_| {
 @@ -148,8 +148,9 @@ pub fn finishZcuQueue(q: *Queue, comp: *Compilation) void {
 }
 fn runLinkTasks(q: *Queue, comp: *Compilation) void {
-    const tid = Compilation.getTid();
     const io = comp.io;
+    const tid: Zcu.PerThread.Id = .acquire(io);
+    defer tid.release(io);
     var have_idle_tasks = true;
 @@ -198,7 +199,7 @@ fn runLinkTasks(q: *Queue, comp: *Compilation) void {
         }
     }
 }
-fn runIdleTask(comp: *Compilation, tid: usize) bool {
+fn runIdleTask(comp: *Compilation, tid: Zcu.PerThread.Id) bool {
     return link.doIdleTask(comp, tid) catch |err| switch (err) {
         error.OutOfMemory => have_more: {
             comp.link_diags.setAllocFailure();
 @@ -217,5 +218,6 @@ const Compilation = @import("../Compilation.zig");
 const InternPool = @import("../InternPool.zig");
 const link = @import("../link.zig");
 const PrelinkTask = link.PrelinkTask;
-const ZcuTask = link.ZcuTask;
 const Queue = @This();
+const Zcu = @import("../Zcu.zig");
+const ZcuTask = link.ZcuTask;

src/main.zig+93-49

...	@@ -52,8 +52,11 @@ pub const std_options: std.Options = .{	...	@@ -52,8 +52,11 @@ pub const std_options: std.Options = .{
52	};	52	};
53	pub const std_options_cwd = if (native_os == .wasi) wasi_cwd else null;	53	pub const std_options_cwd = if (native_os == .wasi) wasi_cwd else null;
54		54
55	pub const panic = crash_report.panic;	55	pub const debug = struct {
56	pub const debug = crash_report.debug;	56	pub fn printCrashContext(terminal: Io.Terminal) void {
		57	crash_report.dumpCrashContext(terminal) catch {};
		58	}
		59	};
57		60
58	var preopens: std.process.Preopens = .empty;	61	var preopens: std.process.Preopens = .empty;
59	pub fn wasi_cwd() Io.Dir {	62	pub fn wasi_cwd() Io.Dir {
...	@@ -158,25 +161,55 @@ pub fn log(	...	@@ -158,25 +161,55 @@ pub fn log(
158	std.log.defaultLog(level, scope, format, args);	161	std.log.defaultLog(level, scope, format, args);
159	}	162	}
160		163
161	var debug_allocator: std.heap.DebugAllocator(.{
162	.stack_trace_frames = build_options.mem_leak_frames,
163	}) = .init;
164
165	const use_debug_allocator = build_options.debug_gpa or	164	const use_debug_allocator = build_options.debug_gpa or
166	(native_os != .wasi and !builtin.link_libc and switch (builtin.mode) {	165	(native_os != .wasi and !builtin.link_libc and switch (builtin.mode) {
167	.Debug, .ReleaseSafe => true,	166	.Debug, .ReleaseSafe => true,
168	.ReleaseFast, .ReleaseSmall => false,	167	.ReleaseFast, .ReleaseSmall => false,
169	});	168	});
170		169
		170	const RootAllocator = if (use_debug_allocator) std.heap.DebugAllocator(.{
		171	.stack_trace_frames = build_options.mem_leak_frames,
		172	.thread_safe = switch (build_options.io_mode) {
		173	.threaded => true,
		174	.evented => false,
		175	},
		176	}) else struct {
		177	pub const init: RootAllocator = .{};
		178	pub fn allocator(_: RootAllocator) Allocator {
		179	if (native_os == .wasi) return std.heap.wasm_allocator;
		180	if (builtin.link_libc) return std.heap.c_allocator;
		181	return std.heap.smp_allocator;
		182	}
		183	pub fn deinit(_: RootAllocator) std.heap.Check {
		184	return .ok;
		185	}
		186	};
		187
171	pub fn main(init: std.process.Init.Minimal) anyerror!void {	188	pub fn main(init: std.process.Init.Minimal) anyerror!void {
172	const gpa = gpa: {	189	var root_allocator: RootAllocator = .init;
173	if (use_debug_allocator) break :gpa debug_allocator.allocator();	190	defer _ = root_allocator.deinit();
174	if (native_os == .wasi) break :gpa std.heap.wasm_allocator;	191	const root_gpa = root_allocator.allocator();
175	if (builtin.link_libc) break :gpa std.heap.c_allocator;	192	var io_impl: IoImpl = undefined;
176	break :gpa std.heap.smp_allocator;	193	switch (build_options.io_mode) {
177	};	194	.threaded => io_impl = .init(root_gpa, .{
178	defer if (use_debug_allocator) {	195	.stack_size = thread_stack_size,
179	_ = debug_allocator.deinit();	196
		197	.argv0 = .init(init.args),
		198	.environ = init.environ,
		199	}),
		200	.evented => try io_impl.init(root_gpa, .{
		201	.argv0 = .init(init.args),
		202	.environ = init.environ,
		203
		204	.backing_allocator_needs_mutex = use_debug_allocator,
		205	}),
		206	}
		207	defer io_impl.deinit();
		208	io_impl_ptr = &io_impl;
		209	const io = io_impl.io();
		210	const gpa = switch (build_options.io_mode) {
		211	.threaded => root_gpa,
		212	.evented => io_impl.allocator(),
180	};	213	};
181	var arena_instance = std.heap.ArenaAllocator.init(gpa);	214	var arena_instance = std.heap.ArenaAllocator.init(gpa);
182	defer arena_instance.deinit();	215	defer arena_instance.deinit();
...	@@ -193,17 +226,6 @@ pub fn main(init: std.process.Init.Minimal) anyerror!void {	...	@@ -193,17 +226,6 @@ pub fn main(init: std.process.Init.Minimal) anyerror!void {
193		226
194	var environ_map = init.environ.createMap(arena) catch \|err\| fatal("failed to parse environment: {t}", .{err});	227	var environ_map = init.environ.createMap(arena) catch \|err\| fatal("failed to parse environment: {t}", .{err});
195		228
196	Compilation.setMainThread();
197
198	var threaded: Io.Threaded = .init(gpa, .{
199	.argv0 = .init(init.args),
200	.environ = init.environ,
201	});
202	defer threaded.deinit();
203	threaded_impl_ptr = &threaded;
204	threaded.stack_size = thread_stack_size;
205	const io = threaded.io();
206
207	if (tracy.enable_allocation) {	229	if (tracy.enable_allocation) {
208	var gpa_tracy = tracy.tracyAllocator(gpa);	230	var gpa_tracy = tracy.tracyAllocator(gpa);
209	return mainArgs(gpa_tracy.allocator(), arena, io, args, &environ_map);	231	return mainArgs(gpa_tracy.allocator(), arena, io, args, &environ_map);
...	@@ -3400,7 +3422,7 @@ fn buildOutputType(	...	@@ -3400,7 +3422,7 @@ fn buildOutputType(
3400	@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1),	3422	@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1),
3401	std.math.maxInt(Zcu.PerThread.IdBacking),	3423	std.math.maxInt(Zcu.PerThread.IdBacking),
3402	);	3424	);
3403	setThreadLimit(thread_limit);	3425	try setThreadLimit(arena, thread_limit);
3404		3426
3405	for (create_module.c_source_files.items) \|*src\| {	3427	for (create_module.c_source_files.items) \|*src\| {
3406	dev.check(.c_compiler);	3428	dev.check(.c_compiler);
...	@@ -4731,13 +4753,13 @@ pub fn translateC(	...	@@ -4731,13 +4753,13 @@ pub fn translateC(
4731	argv: []const []const u8,	4753	argv: []const []const u8,
4732	environ_map: *const process.Environ.Map,	4754	environ_map: *const process.Environ.Map,
4733	prog_node: std.Progress.Node,	4755	prog_node: std.Progress.Node,
		4756	thread_limit: usize,
4734	capture: ?*[]u8,	4757	capture: ?*[]u8,
4735	) !void {	4758	) !void {
4736	try jitCmd(gpa, arena, io, argv, environ_map, .{	4759	try jitCmdInner(gpa, arena, io, argv, environ_map, prog_node, thread_limit, .{
4737	.cmd_name = "translate-c",	4760	.cmd_name = "translate-c",
4738	.root_src_path = "translate-c/main.zig",	4761	.root_src_path = "translate-c/main.zig",
4739	.depend_on_aro = true,	4762	.depend_on_aro = true,
4740	.progress_node = prog_node,
4741	.capture = capture,	4763	.capture = capture,
4742	});	4764	});
4743	}	4765	}
...	@@ -5187,7 +5209,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,	...	@@ -5187,7 +5209,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
5187	@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1),	5209	@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1),
5188	std.math.maxInt(Zcu.PerThread.IdBacking),	5210	std.math.maxInt(Zcu.PerThread.IdBacking),
5189	);	5211	);
5190	setThreadLimit(thread_limit);	5212	try setThreadLimit(arena, thread_limit);
5191		5213
5192	// Dummy http client that is not actually used when fetch_command is unsupported.	5214	// Dummy http client that is not actually used when fetch_command is unsupported.
5193	// Prevents bootstrap from depending on a bunch of unnecessary stuff.	5215	// Prevents bootstrap from depending on a bunch of unnecessary stuff.
...	@@ -5651,7 +5673,7 @@ const JitCmdOptions = struct {	...	@@ -5651,7 +5673,7 @@ const JitCmdOptions = struct {
5651	capture: ?*[]u8 = null,	5673	capture: ?*[]u8 = null,
5652	/// Send error bundles via std.zig.Server over stdout	5674	/// Send error bundles via std.zig.Server over stdout
5653	server: bool = false,	5675	server: bool = false,
5654	progress_node: ?std.Progress.Node = null,	5676	color: Color = .auto,
5655	};	5677	};
5656		5678
5657	fn jitCmd(	5679	fn jitCmd(
...	@@ -5664,12 +5686,30 @@ fn jitCmd(	...	@@ -5664,12 +5686,30 @@ fn jitCmd(
5664	) !void {	5686	) !void {
5665	dev.check(.jit_command);	5687	dev.check(.jit_command);
5666		5688
5667	const color: Color = .auto;	5689	const root_prog_node = std.Progress.start(io, .{
5668	const root_prog_node = if (options.progress_node) \|node\| node else std.Progress.start(io, .{	5690	.disable_printing = (options.color == .off),
5669	.disable_printing = (color == .off),
5670	});	5691	});
5671	defer root_prog_node.end();	5692	defer root_prog_node.end();
5672		5693
		5694	const thread_limit = @min(
		5695	@max(std.Thread.getCpuCount() catch 1, 1),
		5696	std.math.maxInt(Zcu.PerThread.IdBacking),
		5697	);
		5698	try setThreadLimit(arena, thread_limit);
		5699
		5700	return jitCmdInner(gpa, arena, io, args, environ_map, root_prog_node, thread_limit, options);
		5701	}
		5702
		5703	fn jitCmdInner(
		5704	gpa: Allocator,
		5705	arena: Allocator,
		5706	io: Io,
		5707	args: []const []const u8,
		5708	environ_map: *const process.Environ.Map,
		5709	root_prog_node: std.Progress.Node,
		5710	thread_limit: usize,
		5711	options: JitCmdOptions,
		5712	) !void {
5673	const target_query: std.Target.Query = .{};	5713	const target_query: std.Target.Query = .{};
5674	const resolved_target: Package.Module.ResolvedTarget = .{	5714	const resolved_target: Package.Module.ResolvedTarget = .{
5675	.result = std.zig.resolveTargetQueryOrFatal(io, target_query),	5715	.result = std.zig.resolveTargetQueryOrFatal(io, target_query),
...	@@ -5702,12 +5742,6 @@ fn jitCmd(	...	@@ -5702,12 +5742,6 @@ fn jitCmd(
5702	);	5742	);
5703	defer dirs.deinit(io);	5743	defer dirs.deinit(io);
5704		5744
5705	const thread_limit = @min(
5706	@max(std.Thread.getCpuCount() catch 1, 1),
5707	std.math.maxInt(Zcu.PerThread.IdBacking),
5708	);
5709	setThreadLimit(thread_limit);
5710
5711	var child_argv: std.ArrayList([]const u8) = .empty;	5745	var child_argv: std.ArrayList([]const u8) = .empty;
5712	try child_argv.ensureUnusedCapacity(arena, args.len + 4);	5746	try child_argv.ensureUnusedCapacity(arena, args.len + 4);
5713		5747
...	@@ -5795,7 +5829,7 @@ fn jitCmd(	...	@@ -5795,7 +5829,7 @@ fn jitCmd(
5795	process.exit(2);	5829	process.exit(2);
5796	}	5830	}
5797	} else {	5831	} else {
5798	updateModule(comp, color, root_prog_node) catch \|err\| switch (err) {	5832	updateModule(comp, options.color, root_prog_node) catch \|err\| switch (err) {
5799	error.CompileErrorsReported => process.exit(2),	5833	error.CompileErrorsReported => process.exit(2),
5800	else => \|e\| return e,	5834	else => \|e\| return e,
5801	};	5835	};
...	@@ -7777,15 +7811,25 @@ fn addLibDirectoryWarn2(	...	@@ -7777,15 +7811,25 @@ fn addLibDirectoryWarn2(
7777	});	7811	});
7778	}	7812	}
7779		7813
7780	var threaded_impl_ptr: *Io.Threaded = undefined;	7814	const IoImpl = switch (build_options.io_mode) {
7781	fn setThreadLimit(n: usize) void {	7815	.threaded => Io.Threaded,
7782	// We want a maximum of n total threads to keep the InternPool happy, but	7816	.evented => Io.Evented,
7783	// the main thread doesn't count towards the limits, so use n-1. Also, the	7817	};
7784	// linker can run concurrently, so we need to set both the async and the	7818	var io_impl_ptr: *IoImpl = undefined;
7785	// concurrency limit.	7819	fn setThreadLimit(arena: std.mem.Allocator, n: usize) Allocator.Error!void {
7786	const limit: Io.Limit = .limited(n - 1);	7820	switch (build_options.io_mode) {
7787	threaded_impl_ptr.setAsyncLimit(limit);	7821	.threaded => {
7788	threaded_impl_ptr.concurrent_limit = limit;	7822	// We want a maximum of n total threads to keep the InternPool happy, but
		7823	// the main thread doesn't count towards the limits, so use n-1. Also, the
		7824	// linker can run concurrently, so we need to set both the async and the
		7825	// concurrency limit.
		7826	const limit: Io.Limit = .limited(n - 1);
		7827	io_impl_ptr.setAsyncLimit(limit);
		7828	io_impl_ptr.concurrent_limit = limit;
		7829	},
		7830	.evented => {},
		7831	}
		7832	try Zcu.PerThread.Id.allocate(arena, @max(n, 2));
7789	}	7833	}
7790		7834
7791	fn randInt(io: Io, comptime T: type) T {	7835	fn randInt(io: Io, comptime T: type) T {

stage1/config.zig.in+1

...	@@ -13,4 +13,5 @@ pub const value_tracing = false;	...	@@ -13,4 +13,5 @@ pub const value_tracing = false;
13	pub const skip_non_native = false;	13	pub const skip_non_native = false;
14	pub const debug_gpa = false;	14	pub const debug_gpa = false;
15	pub const dev = .core;	15	pub const dev = .core;
		16	pub const io_mode: enum { threaded, evented } = .threaded;
16	pub const value_interpret_mode = .direct;	17	pub const value_interpret_mode = .direct;

...	@@ -13,6 +13,7 @@ const DevEnv = @import("src/dev.zig").Env;	...	@@ -13,6 +13,7 @@ const DevEnv = @import("src/dev.zig").Env;
13	const zig_version: std.SemanticVersion = .{ .major = 0, .minor = 16, .patch = 0 };	13	const zig_version: std.SemanticVersion = .{ .major = 0, .minor = 16, .patch = 0 };
14	const stack_size = 46 * 1024 * 1024;	14	const stack_size = 46 * 1024 * 1024;
15		15
		16	const IoMode = enum { threaded, evented };
16	const ValueInterpretMode = enum { direct, by_name };	17	const ValueInterpretMode = enum { direct, by_name };
17		18
18	pub fn build(b: *std.Build) !void {	19	pub fn build(b: *std.Build) !void {
...	@@ -188,6 +189,7 @@ pub fn build(b: *std.Build) !void {	...	@@ -188,6 +189,7 @@ pub fn build(b: *std.Build) !void {
188	const strip = b.option(bool, "strip", "Omit debug information");	189	const strip = b.option(bool, "strip", "Omit debug information");
189	const valgrind = b.option(bool, "valgrind", "Enable valgrind integration");	190	const valgrind = b.option(bool, "valgrind", "Enable valgrind integration");
190	const pie = b.option(bool, "pie", "Produce a Position Independent Executable");	191	const pie = b.option(bool, "pie", "Produce a Position Independent Executable");
		192	const io_mode = b.option(IoMode, "io-mode", "How the compiler performs IO") orelse .threaded;
191	const value_interpret_mode = b.option(ValueInterpretMode, "value-interpret-mode", "How the compiler translates between 'std.builtin' types and its internal datastructures") orelse .direct;	193	const value_interpret_mode = b.option(ValueInterpretMode, "value-interpret-mode", "How the compiler translates between 'std.builtin' types and its internal datastructures") orelse .direct;
192	const value_tracing = b.option(bool, "value-tracing", "Enable extra state tracking to help troubleshoot bugs in the compiler (using the std.debug.Trace API)") orelse false;	194	const value_tracing = b.option(bool, "value-tracing", "Enable extra state tracking to help troubleshoot bugs in the compiler (using the std.debug.Trace API)") orelse false;
193		195
...	@@ -236,6 +238,7 @@ pub fn build(b: *std.Build) !void {	...	@@ -236,6 +238,7 @@ pub fn build(b: *std.Build) !void {
236	exe_options.addOption(bool, "llvm_has_xtensa", llvm_has_xtensa);	238	exe_options.addOption(bool, "llvm_has_xtensa", llvm_has_xtensa);
237	exe_options.addOption(bool, "debug_gpa", debug_gpa);	239	exe_options.addOption(bool, "debug_gpa", debug_gpa);
238	exe_options.addOption(DevEnv, "dev", b.option(DevEnv, "dev", "Build a compiler with a reduced feature set for development of specific features") orelse if (only_c) .bootstrap else .full);	240	exe_options.addOption(DevEnv, "dev", b.option(DevEnv, "dev", "Build a compiler with a reduced feature set for development of specific features") orelse if (only_c) .bootstrap else .full);
		241	exe_options.addOption(IoMode, "io_mode", io_mode);
239	exe_options.addOption(ValueInterpretMode, "value_interpret_mode", value_interpret_mode);	242	exe_options.addOption(ValueInterpretMode, "value_interpret_mode", value_interpret_mode);
240		243
241	if (link_libc) {	244	if (link_libc) {
...	@@ -710,6 +713,7 @@ fn addWasiUpdateStep(b: *std.Build, version: [:0]const u8) !void {	...	@@ -710,6 +713,7 @@ fn addWasiUpdateStep(b: *std.Build, version: [:0]const u8) !void {
710	exe_options.addOption(u32, "tracy_callstack_depth", 0);	713	exe_options.addOption(u32, "tracy_callstack_depth", 0);
711	exe_options.addOption(bool, "value_tracing", false);	714	exe_options.addOption(bool, "value_tracing", false);
712	exe_options.addOption(DevEnv, "dev", .bootstrap);	715	exe_options.addOption(DevEnv, "dev", .bootstrap);
		716	exe_options.addOption(IoMode, "io_mode", .threaded);
713		717
714	// zig1 chooses to interpret values by name. The tradeoff is as follows:	718	// zig1 chooses to interpret values by name. The tradeoff is as follows:
715	//	719	//

...	@@ -6717,9 +6717,10 @@ pub const IORING_ACCEPT_MULTISHOT = 1 << 0;	...	@@ -6717,9 +6717,10 @@ pub const IORING_ACCEPT_MULTISHOT = 1 << 0;
6717	/// IORING_OP_MSG_RING command types, stored in sqe->addr	6717	/// IORING_OP_MSG_RING command types, stored in sqe->addr
6718	pub const IORING_MSG_RING_COMMAND = enum(u8) {	6718	pub const IORING_MSG_RING_COMMAND = enum(u8) {
6719	/// pass sqe->len as 'res' and off as user_data	6719	/// pass sqe->len as 'res' and off as user_data
6720	DATA,	6720	DATA = 0,
6721	/// send a registered fd to another ring	6721	/// send a registered fd to another ring
6722	SEND_FD,	6722	SEND_FD = 1,
		6723	_,
6723	};	6724	};
6724		6725
6725	// io_uring_sqe.msg_ring_flags (rw_flags in the Zig struct)	6726	// io_uring_sqe.msg_ring_flags (rw_flags in the Zig struct)
...	@@ -6772,6 +6773,8 @@ pub const IORING_CQE_F_SOCK_NONEMPTY = 1 << 2;	...	@@ -6772,6 +6773,8 @@ pub const IORING_CQE_F_SOCK_NONEMPTY = 1 << 2;
6772	pub const IORING_CQE_F_NOTIF = 1 << 3;	6773	pub const IORING_CQE_F_NOTIF = 1 << 3;
6773	/// If set, the buffer ID set in the completion will get more completions.	6774	/// If set, the buffer ID set in the completion will get more completions.
6774	pub const IORING_CQE_F_BUF_MORE = 1 << 4;	6775	pub const IORING_CQE_F_BUF_MORE = 1 << 4;
		6776	pub const IORING_CQE_F_SKIP = 1 << 5;
		6777	pub const IORING_CQE_F_32 = 1 << 15;
6775		6778
6776	pub const IORING_CQE_BUFFER_SHIFT = 16;	6779	pub const IORING_CQE_BUFFER_SHIFT = 16;
6777		6780
...	@@ -7068,7 +7071,7 @@ pub const IORING_RESTRICTION = enum(u16) {	...	@@ -7068,7 +7071,7 @@ pub const IORING_RESTRICTION = enum(u16) {
7068	_,	7071	_,
7069	};	7072	};
7070		7073
7071	pub const IO_URING_SOCKET_OP = enum(u16) {	7074	pub const IO_URING_SOCKET_OP = enum(u32) {
7072	SIOCIN = 0,	7075	SIOCIN = 0,
7073	SIOCOUTQ = 1,	7076	SIOCOUTQ = 1,
7074	GETSOCKOPT = 2,	7077	GETSOCKOPT = 2,

...	@@ -60,7 +60,7 @@ pub const CurrentPathError = error{	...	@@ -60,7 +60,7 @@ pub const CurrentPathError = error{
60	NameTooLong,	60	NameTooLong,
61	/// Not possible on Windows. Always returned on WASI.	61	/// Not possible on Windows. Always returned on WASI.
62	CurrentDirUnlinked,	62	CurrentDirUnlinked,
63	} \|\| Io.UnexpectedError;	63	} \|\| Io.Cancelable \|\| Io.UnexpectedError;
64		64
65	/// On Windows, the result is encoded as [WTF-8](https://wtf-8.codeberg.page/).	65	/// On Windows, the result is encoded as [WTF-8](https://wtf-8.codeberg.page/).
66	/// On other platforms, the result is an opaque sequence of bytes with no	66	/// On other platforms, the result is an opaque sequence of bytes with no
...	@@ -72,7 +72,7 @@ pub fn currentPath(io: Io, buffer: []u8) CurrentPathError!usize {	...	@@ -72,7 +72,7 @@ pub fn currentPath(io: Io, buffer: []u8) CurrentPathError!usize {
72	pub const CurrentPathAllocError = Allocator.Error \|\| error{	72	pub const CurrentPathAllocError = Allocator.Error \|\| error{
73	/// Not possible on Windows. Always returned on WASI.	73	/// Not possible on Windows. Always returned on WASI.
74	CurrentDirUnlinked,	74	CurrentDirUnlinked,
75	} \|\| Io.UnexpectedError;	75	} \|\| Io.Cancelable \|\| Io.UnexpectedError;
76		76
77	/// On Windows, the result is encoded as [WTF-8](https://wtf-8.codeberg.page/).	77	/// On Windows, the result is encoded as [WTF-8](https://wtf-8.codeberg.page/).
78	/// On other platforms, the result is an opaque sequence of bytes with no	78	/// On other platforms, the result is an opaque sequence of bytes with no
...	@@ -355,7 +355,7 @@ pub const SpawnError = error{	...	@@ -355,7 +355,7 @@ pub const SpawnError = error{
355	/// On Windows, the volume does not contain a recognized file system. File	355	/// On Windows, the volume does not contain a recognized file system. File
356	/// system drivers might not be loaded, or the volume may be corrupt.	356	/// system drivers might not be loaded, or the volume may be corrupt.
357	UnrecognizedVolume,	357	UnrecognizedVolume,
358	} \|\| Io.Dir.PathNameError \|\| Io.Cancelable \|\| Io.UnexpectedError;	358	} \|\| Io.File.OpenError \|\| Io.Dir.PathNameError \|\| Io.Cancelable \|\| Io.UnexpectedError;
359		359
360	pub const SpawnOptions = struct {	360	pub const SpawnOptions = struct {
361	argv: []const []const u8,	361	argv: []const []const u8,