Commit d2c862e6ff355000f3a04ce8e2531dd4fe01d820 | zig

author	Andrew Kelley <andrew@ziglang.org> 2026-02-13 23:19:32+01:00
committer	Andrew Kelley <andrew@ziglang.org> 2026-02-13 23:19:32+01:00
log	d2c862e6ff355000f3a04ce8e2531dd4fe01d820
tree	13d1c479d3a285ec9a586e429794fd23185ff013
parent	e262a32ad1d039ee575009c8ea770743b72e8160
parent	0eb1e0c30a4f143dfcb15b4c6a01ec9c566dc39f

Merge pull request 'Io.Dispatch: introduce grand central dispatch io impl' (#31198) from dispatch into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/31198
Reviewed-by: Andrew Kelley <andrew@ziglang.org>

16 files changed, 12005 insertions(+), 6773 deletions(-)

lib
- std
  - Build
    Watch
    FsEvents.zig
  - Io.zig
  - Io
    Dispatch.zig
    IoUring.zig
    Kqueue.zig
    Reader.zig
    Threaded.zig
    Uring.zig
    fiber.zig
  - c.zig
  - c
    darwin.zig
    darwin
    dispatch.zig
  - debug.zig
  - process
    Environ.zig
src
- crash_report.zig
- main.zig

lib/std/Build/Watch/FsEvents.zig+13-31

...	...	@@ -34,11 +34,11 @@ watch_paths: std.StringArrayHashMapUnmanaged([]const *std.Build.Step),
34	34
35	35	/// The semaphore we use to block the thread calling `wait` until the callback determines a relevant
36	36	/// event has occurred. This is retained across `wait` calls for simplicity and efficiency.
37		waiting_semaphore: dispatch_semaphore_t,
	37	waiting_semaphore: dispatch.semaphore_t,
38	38	/// This dispatch queue is created by us and executes serially. It exists exclusively to trigger the
39	39	/// callbacks of the FSEventStream we create. This is not in use outside of `wait`, but is retained
40	40	/// across `wait` calls for simplicity and efficiency.
41		dispatch_queue: dispatch_queue_t,
	41	dispatch_queue: dispatch.queue_t,
42	42	/// In theory, this field avoids race conditions. In practice, it is essentially unused at the time
43	43	/// of writing. See the comment at the start of `wait` for details.
44	44	since_event: FSEventStreamEventId,
...	...	@@ -57,7 +57,7 @@ const ResolvedSymbols = struct {
57	57	latency: CFTimeInterval,
58	58	flags: FSEventStreamCreateFlags,
59	59	) callconv(.c) FSEventStreamRef,
60		FSEventStreamSetDispatchQueue: *const fn (stream: FSEventStreamRef, queue: dispatch_queue_t) callconv(.c) void,
	60	FSEventStreamSetDispatchQueue: *const fn (stream: FSEventStreamRef, queue: dispatch.queue_t) callconv(.c) void,
61	61	FSEventStreamStart: *const fn (stream: FSEventStreamRef) callconv(.c) bool,
62	62	FSEventStreamStop: *const fn (stream: FSEventStreamRef) callconv(.c) void,
63	63	FSEventStreamInvalidate: *const fn (stream: FSEventStreamRef) callconv(.c) void,
...	...	@@ -80,7 +80,7 @@ const ResolvedSymbols = struct {
80	80	kCFAllocatorUseContext: *const CFAllocatorRef,
81	81	};
82	82
83		pub fn init(cwd_path: []const u8) error{ OpenFrameworkFailed, MissingCoreServicesSymbol }!FsEvents {
	83	pub fn init(cwd_path: []const u8) error{ OpenFrameworkFailed, MissingCoreServicesSymbol, SystemResources }!FsEvents {
84	84	var core_services = std.DynLib.open("/System/Library/Frameworks/CoreServices.framework/CoreServices") catch
85	85	return error.OpenFrameworkFailed;
86	86	errdefer core_services.close();
...	...	@@ -96,8 +96,8 @@ pub fn init(cwd_path: []const u8) error{ OpenFrameworkFailed, MissingCoreService
96	96	.paths_arena = .{},
97	97	.watch_roots = &.{},
98	98	.watch_paths = .empty,
99		.waiting_semaphore = dispatch_semaphore_create(0),
100		.dispatch_queue = dispatch_queue_create("zig-watch", .SERIAL),
	99	.waiting_semaphore = dispatch.semaphore_create(0) orelse return error.SystemResources,
	100	.dispatch_queue = dispatch.queue_create("zig-watch", .SERIAL()) orelse return error.SystemResources,
101	101	// Not `.since_now`, because this means we can init `FsEvents` before we do work in order
102	102	// to notice any changes which happened during said work.
103	103	.since_event = resolved_symbols.FSEventsGetCurrentEventId(),
...	...	@@ -106,8 +106,8 @@ pub fn init(cwd_path: []const u8) error{ OpenFrameworkFailed, MissingCoreService
106	106	}
107	107
108	108	pub fn deinit(fse: *FsEvents, gpa: Allocator, io: Io) void {
109		dispatch_release(fse.waiting_semaphore);
110		dispatch_release(fse.dispatch_queue);
	109	fse.waiting_semaphore.as_object().release();
	110	fse.dispatch_queue.as_object().release();
111	111	fse.core_services.close(io);
112	112
113	113	gpa.free(fse.watch_roots);
...	...	@@ -275,9 +275,9 @@ pub fn wait(fse: *FsEvents, gpa: Allocator, timeout_ns: ?u64) error{ OutOfMemory
275	275	defer rs.FSEventStreamInvalidate(event_stream);
276	276	if (!rs.FSEventStreamStart(event_stream)) return error.StartFailed;
277	277	defer rs.FSEventStreamStop(event_stream);
278		const result = dispatch_semaphore_wait(fse.waiting_semaphore, timeout: {
279		const ns = timeout_ns orelse break :timeout .forever;
280		break :timeout dispatch_time(.now, @intCast(ns));
	278	const result = fse.waiting_semaphore.wait(timeout: {
	279	const ns = timeout_ns orelse break :timeout .FOREVER;
	280	break :timeout .time(.NOW, @intCast(ns));
281	281	});
282	282	return switch (result) {
283	283	0 => .dirty,
...	...	@@ -382,7 +382,7 @@ fn eventCallback(
382	382	}
383	383	if (any_dirty) {
384	384	fse.since_event = rs.FSEventStreamGetLatestEventId(stream);
385		_ = dispatch_semaphore_signal(fse.waiting_semaphore);
	385	_ = fse.waiting_semaphore.signal();
386	386	}
387	387	}
388	388	fn dirStartsWith(path: []const u8, prefix: []const u8) bool {
...	...	@@ -392,25 +392,6 @@ fn dirStartsWith(path: []const u8, prefix: []const u8) bool {
392	392	return true; // `path` is `/foo/bar/...`, `prefix` is `/foo/bar`
393	393	}
394	394
395		const dispatch_time_t = enum(u64) {
396		now = 0,
397		forever = std.math.maxInt(u64),
398		_,
399		};
400		extern fn dispatch_time(base: dispatch_time_t, delta_ns: i64) dispatch_time_t;
401
402		const dispatch_semaphore_t = *opaque {};
403		extern fn dispatch_semaphore_create(value: isize) dispatch_semaphore_t;
404		extern fn dispatch_semaphore_wait(dsema: dispatch_semaphore_t, timeout: dispatch_time_t) isize;
405		extern fn dispatch_semaphore_signal(dsema: dispatch_semaphore_t) isize;
406
407		const dispatch_queue_t = *opaque {};
408		const dispatch_queue_attr_t = ?*opaque {
409		const SERIAL: dispatch_queue_attr_t = null;
410		};
411		extern fn dispatch_queue_create(label: [*:0]const u8, attr: dispatch_queue_attr_t) dispatch_queue_t;
412		extern fn dispatch_release(object: *anyopaque) void;
413
414	395	const CFAllocatorRef = ?*const opaque {};
415	396	const CFArrayRef = *const opaque {};
416	397	const CFStringRef = *const opaque {};
...	...	@@ -489,6 +470,7 @@ const FSEventStreamEventFlags = packed struct(u32) {
489	470	_: u24 = 0,
490	471	};
491	472
	473	const dispatch = std.c.dispatch;
492	474	const std = @import("std");
493	475	const Io = std.Io;
494	476	const assert = std.debug.assert;

lib/std/Io.zig+56-56

...	...	@@ -26,19 +26,17 @@ userdata: ?*anyopaque,
26	26	vtable: *const VTable,
27	27
28	28	pub const Threaded = @import("Io/Threaded.zig");
29		pub const Evented = switch (builtin.os.tag) {
30		.linux => switch (builtin.cpu.arch) {
31		.x86_64, .aarch64 => IoUring,
32		else => void, // context-switching code not implemented yet
33		},
34		.dragonfly, .freebsd, .netbsd, .openbsd, .driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => switch (builtin.cpu.arch) {
35		.x86_64, .aarch64 => Kqueue,
36		else => void, // context-switching code not implemented yet
37		},
	29
	30	pub const fiber = @import("Io/fiber.zig");
	31	pub const Evented = if (fiber.supported) switch (builtin.os.tag) {
	32	.linux => Uring,
	33	.dragonfly, .freebsd, .netbsd, .openbsd => Kqueue,
	34	.driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => Dispatch,
38	35	else => void,
39		};
	36	} else void; // context-switching code not implemented yet
	37	pub const Dispatch = @import("Io/Dispatch.zig");
40	38	pub const Kqueue = @import("Io/Kqueue.zig");
41		pub const IoUring = @import("Io/IoUring.zig");
	39	pub const Uring = @import("Io/Uring.zig");
42	40
43	41	pub const Reader = @import("Io/Reader.zig");
44	42	pub const Writer = @import("Io/Writer.zig");
...	...	@@ -51,6 +49,8 @@ pub const RwLock = @import("Io/RwLock.zig");
51	49	pub const Semaphore = @import("Io/Semaphore.zig");
52	50
53	51	pub const VTable = struct {
	52	crashHandler: const fn (?anyopaque) void,
	53
54	54	/// If it returns `null` it means `result` has been already populated and
55	55	/// `await` will be a no-op.
56	56	///
...	...	@@ -378,9 +378,9 @@ pub const Operation = union(enum) {
378	378	pub const Pending = struct {
379	379	node: List.DoubleNode,
380	380	tag: Tag,
381		context: Context align(@max(@alignOf(usize), 4)),
	381	userdata: Userdata align(@max(@alignOf(usize), 4)),
382	382
383		pub const Context = [3]usize;
	383	pub const Userdata = [7]usize;
384	384	};
385	385
386	386	pub const Completion = struct {
...	...	@@ -431,7 +431,7 @@ pub const Batch = struct {
431	431	submitted: Operation.List,
432	432	pending: Operation.List,
433	433	completed: Operation.List,
434		context: ?anyopaque align(@max(@alignOf(?anyopaque), 4)),
	434	userdata: ?anyopaque align(@max(@alignOf(?anyopaque), 4)),
435	435
436	436	/// After calling this, it is safe to unconditionally defer a call to
437	437	/// `cancel`. `storage` is a pre-allocated buffer of undefined memory that
...	...	@@ -453,40 +453,40 @@ pub const Batch = struct {
453	453	.submitted = .empty,
454	454	.pending = .empty,
455	455	.completed = .empty,
456		.context = null,
	456	.userdata = null,
457	457	};
458	458	}
459	459
460	460	/// Adds an operation to be performed at the next await call.
461	461	/// Returns the index that will be returned by `next` after the operation completes.
462	462	/// Asserts that no more than `storage.len` operations are active at a time.
463		pub fn add(b: *Batch, operation: Operation) u32 {
464		const index = b.unused.next;
465		b.addAt(index.toIndex(), operation);
	463	pub fn add(batch: *Batch, operation: Operation) u32 {
	464	const index = batch.unused.next;
	465	batch.addAt(index.toIndex(), operation);
466	466	return index;
467	467	}
468	468
469	469	/// Adds an operation to be performed at the next await call.
470	470	/// After the operation completes, `next` will return `index`.
471	471	/// Asserts that the operation at `index` is not active.
472		pub fn addAt(b: *Batch, index: u32, operation: Operation) void {
473		const storage = &b.storage[index];
	472	pub fn addAt(batch: *Batch, index: u32, operation: Operation) void {
	473	const storage = &batch.storage[index];
474	474	const unused = storage.unused;
475	475	switch (unused.prev) {
476		.none => b.unused.head = unused.next,
477		else => \|prev_index\| b.storage[prev_index.toIndex()].unused.next = unused.next,
	476	.none => batch.unused.head = unused.next,
	477	else => \|prev_index\| batch.storage[prev_index.toIndex()].unused.next = unused.next,
478	478	}
479	479	switch (unused.next) {
480		.none => b.unused.tail = unused.prev,
481		else => \|next_index\| b.storage[next_index.toIndex()].unused.prev = unused.prev,
	480	.none => batch.unused.tail = unused.prev,
	481	else => \|next_index\| batch.storage[next_index.toIndex()].unused.prev = unused.prev,
482	482	}
483	483
484		switch (b.submitted.tail) {
485		.none => b.submitted.head = .fromIndex(index),
486		else => \|tail_index\| b.storage[tail_index.toIndex()].submission.node.next = .fromIndex(index),
	484	switch (batch.submitted.tail) {
	485	.none => batch.submitted.head = .fromIndex(index),
	486	else => \|tail_index\| batch.storage[tail_index.toIndex()].submission.node.next = .fromIndex(index),
487	487	}
488	488	storage.* = .{ .submission = .{ .node = .{ .next = .none }, .operation = operation } };
489		b.submitted.tail = .fromIndex(index);
	489	batch.submitted.tail = .fromIndex(index);
490	490	}
491	491
492	492	pub const Completion = struct {
...	...	@@ -502,22 +502,22 @@ pub const Batch = struct {
502	502	///
503	503	/// Each completion returned from this function dequeues from the `Batch`.
504	504	/// It is not required to dequeue all completions before awaiting again.
505		pub fn next(b: *Batch) ?Completion {
506		const index = b.completed.head;
	505	pub fn next(batch: *Batch) ?Completion {
	506	const index = batch.completed.head;
507	507	if (index == .none) return null;
508		const storage = &b.storage[index.toIndex()];
	508	const storage = &batch.storage[index.toIndex()];
509	509	const completion = storage.completion;
510	510	const next_index = completion.node.next;
511		b.completed.head = next_index;
512		if (next_index == .none) b.completed.tail = .none;
	511	batch.completed.head = next_index;
	512	if (next_index == .none) batch.completed.tail = .none;
513	513
514		const tail_index = b.unused.tail;
	514	const tail_index = batch.unused.tail;
515	515	switch (tail_index) {
516		.none => b.unused.head = index,
517		else => b.storage[tail_index.toIndex()].unused.next = index,
	516	.none => batch.unused.head = index,
	517	else => batch.storage[tail_index.toIndex()].unused.next = index,
518	518	}
519	519	storage.* = .{ .unused = .{ .prev = tail_index, .next = .none } };
520		b.unused.tail = index;
	520	batch.unused.tail = index;
521	521	return .{ .index = index.toIndex(), .result = completion.result };
522	522	}
523	523
...	...	@@ -529,8 +529,8 @@ pub const Batch = struct {
529	529	/// concurrency into the batched operations, but unlike `awaitConcurrent`,
530	530	/// does not require it, and therefore cannot fail with
531	531	/// `error.ConcurrencyUnavailable`.
532		pub fn awaitAsync(b: *Batch, io: Io) Cancelable!void {
533		return io.vtable.batchAwaitAsync(io.userdata, b);
	532	pub fn awaitAsync(batch: *Batch, io: Io) Cancelable!void {
	533	return io.vtable.batchAwaitAsync(io.userdata, batch);
534	534	}
535	535
536	536	pub const AwaitConcurrentError = ConcurrentError \|\| Cancelable \|\| Timeout.Error;
...	...	@@ -542,8 +542,8 @@ pub const Batch = struct {
542	542	/// Unlike `awaitAsync`, this function requires the implementation to
543	543	/// perform the operations concurrently and therefore can fail with
544	544	/// `error.ConcurrencyUnavailable`.
545		pub fn awaitConcurrent(b: *Batch, io: Io, timeout: Timeout) AwaitConcurrentError!void {
546		return io.vtable.batchAwaitConcurrent(io.userdata, b, timeout);
	545	pub fn awaitConcurrent(batch: *Batch, io: Io, timeout: Timeout) AwaitConcurrentError!void {
	546	return io.vtable.batchAwaitConcurrent(io.userdata, batch, timeout);
547	547	}
548	548
549	549	/// Requests all pending operations to be interrupted, then waits for all
...	...	@@ -552,28 +552,28 @@ pub const Batch = struct {
552	552	/// canceled operations will be absent from the iteration. Some operations
553	553	/// may have successfully completed regardless of the cancel request and
554	554	/// will appear in the iteration.
555		pub fn cancel(b: *Batch, io: Io) void {
	555	pub fn cancel(batch: *Batch, io: Io) void {
556	556	{ // abort pending submissions
557		var tail_index = b.unused.tail;
558		defer b.unused.tail = tail_index;
559		var index = b.submitted.head;
560		errdefer b.submissions.head = index;
	557	var tail_index = batch.unused.tail;
	558	defer batch.unused.tail = tail_index;
	559	var index = batch.submitted.head;
	560	errdefer batch.submissions.head = index;
561	561	while (index != .none) {
562		const next_index = b.storage[index.toIndex()].submission.node.next;
	562	const next_index = batch.storage[index.toIndex()].submission.node.next;
563	563	switch (tail_index) {
564		.none => b.unused.head = index,
565		else => b.storage[tail_index.toIndex()].unused.next = index,
	564	.none => batch.unused.head = index,
	565	else => batch.storage[tail_index.toIndex()].unused.next = index,
566	566	}
567		b.storage[index.toIndex()] = .{ .unused = .{ .prev = tail_index, .next = .none } };
	567	batch.storage[index.toIndex()] = .{ .unused = .{ .prev = tail_index, .next = .none } };
568	568	tail_index = index;
569	569	index = next_index;
570	570	}
571		b.submitted = .{ .head = .none, .tail = .none };
	571	batch.submitted = .{ .head = .none, .tail = .none };
572	572	}
573		io.vtable.batchCancel(io.userdata, b);
574		assert(b.submitted.head == .none and b.submitted.tail == .none);
575		assert(b.pending.head == .none and b.pending.tail == .none);
576		assert(b.context == null); // that was the last chance to deallocate resources
	573	io.vtable.batchCancel(io.userdata, batch);
	574	assert(batch.submitted.head == .none and batch.submitted.tail == .none);
	575	assert(batch.pending.head == .none and batch.pending.tail == .none);
	576	assert(batch.userdata == null); // that was the last chance to deallocate resources
577	577	}
578	578	};
579	579
...	...	@@ -643,7 +643,7 @@ pub const Limit = enum(usize) {
643	643	}
644	644
645	645	pub fn nonzero(l: Limit) bool {
646		return @intFromEnum(l) > 0;
	646	return l != .nothing;
647	647	}
648	648
649	649	/// Return a new limit reduced by `amount` or return `null` indicating

lib/std/Io/Dispatch.zig created+5038

...	...	@@ -0,0 +1,5038 @@
	1	const Alignment = std.mem.Alignment;
	2	const Allocator = std.mem.Allocator;
	3	const Argv0 = Io.Threaded.Argv0;
	4	const assert = std.debug.assert;
	5	const builtin = @import("builtin");
	6	const c = std.c;
	7	const ChdirError = Io.Threaded.ChdirError;
	8	const clockToPosix = Io.Threaded.clockToPosix;
	9	const closeFd = Io.Threaded.closeFd;
	10	const Csprng = Io.Threaded.Csprng;
	11	const default_PATH = Io.Threaded.default_PATH;
	12	const Dir = Io.Dir;
	13	const Environ = Io.Threaded.Environ;
	14	const errnoBug = Io.Threaded.errnoBug;
	15	const Evented = @This();
	16	const fallbackSeed = Io.Threaded.fallbackSeed;
	17	const File = Io.File;
	18	const Io = std.Io;
	19	const iovec = std.posix.iovec;
	20	const iovec_const = std.posix.iovec_const;
	21	const log = std.log.scoped(.dispatch);
	22	const max_iovecs_len = Io.Threaded.max_iovecs_len;
	23	const nanosecondsFromPosix = Io.Threaded.nanosecondsFromPosix;
	24	const net = Io.net;
	25	const pathToPosix = Io.Threaded.pathToPosix;
	26	const process = std.process;
	27	const recoverableOsBugDetected = Io.Threaded.recoverableOsBugDetected;
	28	const setTimestampToPosix = Io.Threaded.setTimestampToPosix;
	29	const splat_buffer_size = Io.Threaded.splat_buffer_size;
	30	const statFromPosix = Io.Threaded.statFromPosix;
	31	const statusToTerm = Io.Threaded.statusToTerm;
	32	const std = @import("std");
	33	const timestampFromPosix = Io.Threaded.timestampFromPosix;
	34	const unexpectedErrno = std.posix.unexpectedErrno;
	35	const UseSendfile = Io.Threaded.UseSendfile;
	36	const UseFcopyfile = Io.Threaded.UseFcopyfile;
	37
	38	/// Empirically saw >4KB being used by the llvm aarch64 backend.
	39	const main_loop_stack_size = 8 * 1024;
	40
	41	queue: c.dispatch.queue_t,
	42	backing_allocator_needs_mutex: bool,
	43	backing_allocator_mutex: Mutex,
	44	/// Does not need to be thread-safe if not used elsewhere.
	45	backing_allocator: Allocator,
	46	main_fiber: Fiber,
	47	main_loop_stack: [*]align(builtin.target.stackAlignment()) u8,
	48	exit_semaphore: c.dispatch.semaphore_t,
	49
	50	use_sendfile: UseSendfile,
	51	use_fcopyfile: UseFcopyfile,
	52	leeway: u64,
	53
	54	futexes: [1 << 8]Futex,
	55
	56	init_stderr_writer: c.dispatch.once_t,
	57	stderr_mutex: Mutex,
	58	stderr_writer: File.Writer,
	59	stderr_mode: Io.Terminal.Mode,
	60
	61	scan_environ: c.dispatch.once_t,
	62	environ: Environ,
	63
	64	open_dev_null: c.dispatch.once_t,
	65	dev_null_file: File.OpenError!File,
	66
	67	csprng_mutex: Mutex,
	68	csprng: Csprng,
	69
	70	const Thread = struct {
	71	main_context: Io.fiber.Context,
	72	current_context: ?*Io.fiber.Context,
	73	seed_csprng: c.dispatch.once_t,
	74	csprng: Csprng,
	75
	76	threadlocal var self: Thread = .{
	77	.main_context = undefined,
	78	.current_context = null,
	79	.seed_csprng = .init,
	80	.csprng = undefined,
	81	};
	82
	83	noinline fn current() *Thread {
	84	return &self;
	85	}
	86
	87	fn currentFiber(thread: Thread) Fiber {
	88	assert(thread.current_context != &thread.main_context);
	89	return @fieldParentPtr("context", thread.current_context.?);
	90	}
	91
	92	const List = struct {
	93	allocated: []Thread,
	94	reserved: u32,
	95	active: u32,
	96	};
	97	};
	98
	99	const Fiber = struct {
	100	required_align: void align(4),
	101	evented: *Evented,
	102	context: Io.fiber.Context,
	103	await_count: i32,
	104	link: union {
	105	awaiter: ?*Fiber,
	106	group: struct { prev: ?Fiber, next: ?Fiber },
	107	},
	108	status: union(enum) {
	109	queue_next: ?*Fiber,
	110	awaiting_group: Group,
	111	},
	112	cancel_status: CancelStatus,
	113	cancel_protection: CancelProtection,
	114
	115	var next_name: u64 = 0;
	116
	117	const CancelStatus = packed struct(usize) {
	118	requested: bool,
	119	awaiting: Awaiting,
	120
	121	const unrequested: CancelStatus = .{ .requested = false, .awaiting = .nothing };
	122
	123	const Awaiting = enum(@Int(.unsigned, @bitSizeOf(usize) - shift)) {
	124	nothing = 0,
	125	group = 1,
	126	select = 2,
	127	_,
	128
	129	const shift = 1;
	130
	131	fn subWrap(lhs: Awaiting, rhs: Awaiting) Awaiting {
	132	return @enumFromInt(@intFromEnum(lhs) -% @intFromEnum(rhs));
	133	}
	134
	135	fn fromCancelable(cancelable: *Cancelable) Awaiting {
	136	return @enumFromInt(@shrExact(@intFromPtr(cancelable), shift));
	137	}
	138
	139	fn toCancelable(awaiting: Awaiting) *Cancelable {
	140	return @ptrFromInt(@shlExact(@as(usize, @intFromEnum(awaiting)), shift));
	141	}
	142	};
	143
	144	fn changeAwaiting(
	145	cancel_status: *CancelStatus,
	146	old_awaiting: Awaiting,
	147	new_awaiting: Awaiting,
	148	) bool {
	149	const old_cancel_status = @atomicRmw(CancelStatus, cancel_status, .Add, .{
	150	.requested = false,
	151	.awaiting = new_awaiting.subWrap(old_awaiting),
	152	}, .release);
	153	assert(old_cancel_status.awaiting == old_awaiting);
	154	return old_cancel_status.requested;
	155	}
	156	};
	157
	158	const CancelProtection = packed struct {
	159	user: Io.CancelProtection,
	160	acknowledged: bool,
	161
	162	const unblocked: CancelProtection = .{ .user = .unblocked, .acknowledged = false };
	163
	164	fn check(cancel_protection: CancelProtection) Io.CancelProtection {
	165	return @enumFromInt(@intFromBool(cancel_protection != unblocked));
	166	}
	167
	168	fn acknowledge(cancel_protection: *CancelProtection) void {
	169	assert(!cancel_protection.acknowledged);
	170	cancel_protection.acknowledged = true;
	171	}
	172
	173	fn recancel(cancel_protection: *CancelProtection) void {
	174	assert(cancel_protection.acknowledged);
	175	cancel_protection.acknowledged = false;
	176	}
	177
	178	test check {
	179	try std.testing.expectEqual(Io.CancelProtection.unblocked, check(.unblocked));
	180	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
	181	.user = .unblocked,
	182	.acknowledged = true,
	183	}));
	184	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
	185	.user = .blocked,
	186	.acknowledged = false,
	187	}));
	188	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
	189	.user = .blocked,
	190	.acknowledged = true,
	191	}));
	192	}
	193	};
	194
	195	const finished: ?*Fiber = @ptrFromInt(@alignOf(Fiber));
	196
	197	const max_result_align: Alignment = .@"16";
	198	const max_result_size = max_result_align.forward(512);
	199	/// This includes any stack realignments that need to happen, and also the
	200	/// initial frame return address slot and argument frame, depending on target.
	201	const min_stack_size = 60 * 1024 * 1024;
	202	const max_context_align: Alignment = .@"16";
	203	const max_context_size = max_context_align.forward(1024);
	204	const max_closure_size: usize = @sizeOf(AsyncClosure);
	205	const max_closure_align: Alignment = .of(AsyncClosure);
	206	const allocation_size = std.mem.alignForward(
	207	usize,
	208	max_closure_align.max(max_context_align).forward(
	209	max_result_align.forward(@sizeOf(Fiber)) + max_result_size + min_stack_size,
	210	) + max_closure_size + max_context_size,
	211	std.heap.page_size_max,
	212	);
	213
	214	fn create(ev: Evented) error{OutOfMemory}!Fiber {
	215	return @ptrCast(try ev.allocator().alignedAlloc(u8, .of(Fiber), allocation_size));
	216	}
	217
	218	fn destroy(fiber: Fiber, ev: Evented) void {
	219	assert(fiber.status.queue_next == null);
	220	ev.allocator().free(fiber.allocatedSlice());
	221	}
	222
	223	fn allocatedSlice(f: *Fiber) []align(@alignOf(Fiber)) u8 {
	224	return @as([*]align(@alignOf(Fiber)) u8, @ptrCast(f))[0..allocation_size];
	225	}
	226
	227	fn allocatedEnd(f: Fiber) []u8 {
	228	const allocated_slice = f.allocatedSlice();
	229	return allocated_slice[allocated_slice.len..].ptr;
	230	}
	231
	232	fn resultPointer(f: Fiber, comptime Result: type) Result {
	233	return @ptrCast(@alignCast(f.resultBytes(.of(Result))));
	234	}
	235
	236	fn resultBytes(f: Fiber, alignment: Alignment) []u8 {
	237	return @ptrFromInt(alignment.forward(@intFromPtr(f) + @sizeOf(Fiber)));
	238	}
	239
	240	const Queue = struct { head: Fiber, tail: Fiber };
	241
	242	/// Like a `*Fiber`, but 2 bits smaller than a pointer (because the LSBs are always 0 due to
	243	/// alignment) so that those two bits can be used in a `packed struct`.
	244	const PackedPtr = enum(@Int(.unsigned, @bitSizeOf(usize) - 2)) {
	245	null = 0,
	246	all_ones = std.math.maxInt(@Int(.unsigned, @bitSizeOf(usize) - 2)),
	247	_,
	248
	249	const Split = packed struct(usize) { low: u2, high: PackedPtr };
	250	fn pack(ptr: ?*Fiber) PackedPtr {
	251	const split: Split = @bitCast(@intFromPtr(ptr));
	252	assert(split.low == 0);
	253	return split.high;
	254	}
	255	fn unpack(ptr: PackedPtr) ?*Fiber {
	256	const split: Split = .{ .low = 0, .high = ptr };
	257	return @ptrFromInt(@as(usize, @bitCast(split)));
	258	}
	259	};
	260
	261	fn requestCancel(fiber: Fiber, ev: Evented) void {
	262	const cancel_status = @atomicRmw(
	263	Fiber.CancelStatus,
	264	&fiber.cancel_status,
	265	.Or,
	266	.{ .requested = true, .awaiting = .nothing },
	267	.acquire,
	268	);
	269	assert(!cancel_status.requested);
	270	switch (cancel_status.awaiting) {
	271	.nothing => {},
	272	.group => {
	273	// The awaiter received a cancelation request while awaiting a group,
	274	// so propagate the cancelation to the group.
	275	if (fiber.status.awaiting_group.cancel(ev, null)) {
	276	fiber.status = .{ .queue_next = null };
	277	ev.queue.async(fiber, &Fiber.@"resume");
	278	}
	279	},
	280	.select => if (@atomicRmw(i32, &fiber.await_count, .Add, 1, .monotonic) == -1) {
	281	ev.queue.async(fiber, &Fiber.@"resume");
	282	},
	283	_ => \|awaiting\| awaiting.toCancelable().canceled(),
	284	}
	285	}
	286
	287	fn @"resume"(context: ?*anyopaque) callconv(.c) void {
	288	const fiber: *Fiber = @ptrCast(@alignCast(context));
	289	const thread: *Thread = .current();
	290	const message: SwitchMessage = .{
	291	.contexts = .{
	292	.old = &thread.main_context,
	293	.new = &fiber.context,
	294	},
	295	.pending_task = .nothing,
	296	};
	297	contextSwitch(&message).handle(fiber.evented);
	298	}
	299	};
	300
	301	pub fn allocator(ev: *Evented) std.mem.Allocator {
	302	return if (ev.backing_allocator_needs_mutex) .{
	303	.ptr = ev,
	304	.vtable = &.{
	305	.alloc = alloc,
	306	.resize = resize,
	307	.remap = remap,
	308	.free = free,
	309	},
	310	} else ev.backing_allocator;
	311	}
	312
	313	fn alloc(userdata: anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[]u8 {
	314	const ev: *Evented = @ptrCast(@alignCast(userdata));
	315	ev.backing_allocator_mutex.lockUncancelable(ev);
	316	defer ev.backing_allocator_mutex.unlock();
	317	return ev.backing_allocator.rawAlloc(len, alignment, ret_addr);
	318	}
	319
	320	fn resize(
	321	userdata: *anyopaque,
	322	memory: []u8,
	323	alignment: std.mem.Alignment,
	324	new_len: usize,
	325	ret_addr: usize,
	326	) bool {
	327	const ev: *Evented = @ptrCast(@alignCast(userdata));
	328	ev.backing_allocator_mutex.lockUncancelable(ev);
	329	defer ev.backing_allocator_mutex.unlock();
	330	return ev.backing_allocator.rawResize(memory, alignment, new_len, ret_addr);
	331	}
	332
	333	fn remap(
	334	userdata: *anyopaque,
	335	memory: []u8,
	336	alignment: Alignment,
	337	new_len: usize,
	338	ret_addr: usize,
	339	) ?[*]u8 {
	340	const ev: *Evented = @ptrCast(@alignCast(userdata));
	341	ev.backing_allocator_mutex.lockUncancelable(ev);
	342	defer ev.backing_allocator_mutex.unlock();
	343	return ev.backing_allocator.rawRemap(memory, alignment, new_len, ret_addr);
	344	}
	345
	346	fn free(userdata: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void {
	347	const ev: *Evented = @ptrCast(@alignCast(userdata));
	348	ev.backing_allocator_mutex.lockUncancelable(ev);
	349	defer ev.backing_allocator_mutex.unlock();
	350	return ev.backing_allocator.rawFree(memory, alignment, ret_addr);
	351	}
	352
	353	pub fn io(ev: *Evented) Io {
	354	return .{
	355	.userdata = ev,
	356	.vtable = &.{
	357	.crashHandler = crashHandler,
	358
	359	.async = async,
	360	.concurrent = concurrent,
	361	.await = await,
	362	.cancel = cancel,
	363
	364	.groupAsync = groupAsync,
	365	.groupConcurrent = groupConcurrent,
	366	.groupAwait = groupAwait,
	367	.groupCancel = groupCancel,
	368
	369	.recancel = recancel,
	370	.swapCancelProtection = swapCancelProtection,
	371	.checkCancel = checkCancel,
	372
	373	.select = select,
	374
	375	.futexWait = futexWait,
	376	.futexWaitUncancelable = futexWaitUncancelable,
	377	.futexWake = futexWake,
	378
	379	.operate = operate,
	380	.batchAwaitAsync = batchAwaitAsync,
	381	.batchAwaitConcurrent = batchAwaitConcurrent,
	382	.batchCancel = batchCancel,
	383
	384	.dirCreateDir = dirCreateDir,
	385	.dirCreateDirPath = dirCreateDirPath,
	386	.dirCreateDirPathOpen = dirCreateDirPathOpen,
	387	.dirOpenDir = dirOpenDir,
	388	.dirStat = dirStat,
	389	.dirStatFile = dirStatFile,
	390	.dirAccess = dirAccess,
	391	.dirCreateFile = dirCreateFile,
	392	.dirCreateFileAtomic = dirCreateFileAtomic,
	393	.dirOpenFile = dirOpenFile,
	394	.dirClose = dirClose,
	395	.dirRead = dirRead,
	396	.dirRealPath = dirRealPath,
	397	.dirRealPathFile = dirRealPathFile,
	398	.dirDeleteFile = dirDeleteFile,
	399	.dirDeleteDir = dirDeleteDir,
	400	.dirRename = dirRename,
	401	.dirRenamePreserve = dirRenamePreserve,
	402	.dirSymLink = dirSymLink,
	403	.dirReadLink = dirReadLink,
	404	.dirSetOwner = dirSetOwner,
	405	.dirSetFileOwner = dirSetFileOwner,
	406	.dirSetPermissions = dirSetPermissions,
	407	.dirSetFilePermissions = dirSetFilePermissions,
	408	.dirSetTimestamps = dirSetTimestamps,
	409	.dirHardLink = dirHardLink,
	410
	411	.fileStat = fileStat,
	412	.fileLength = fileLength,
	413	.fileClose = fileClose,
	414	.fileWritePositional = fileWritePositional,
	415	.fileWriteFileStreaming = fileWriteFileStreaming,
	416	.fileWriteFilePositional = fileWriteFilePositional,
	417	.fileReadPositional = fileReadPositional,
	418	.fileSeekBy = fileSeekBy,
	419	.fileSeekTo = fileSeekTo,
	420	.fileSync = fileSync,
	421	.fileIsTty = fileIsTty,
	422	.fileEnableAnsiEscapeCodes = fileEnableAnsiEscapeCodes,
	423	.fileSupportsAnsiEscapeCodes = fileIsTty,
	424	.fileSetLength = fileSetLength,
	425	.fileSetOwner = fileSetOwner,
	426	.fileSetPermissions = fileSetPermissions,
	427	.fileSetTimestamps = fileSetTimestamps,
	428	.fileLock = fileLock,
	429	.fileTryLock = fileTryLock,
	430	.fileUnlock = fileUnlock,
	431	.fileDowngradeLock = fileDowngradeLock,
	432	.fileRealPath = fileRealPath,
	433	.fileHardLink = fileHardLink,
	434
	435	.fileMemoryMapCreate = fileMemoryMapCreate,
	436	.fileMemoryMapDestroy = fileMemoryMapDestroy,
	437	.fileMemoryMapSetLength = fileMemoryMapSetLength,
	438	.fileMemoryMapRead = fileMemoryMapRead,
	439	.fileMemoryMapWrite = fileMemoryMapWrite,
	440
	441	.processExecutableOpen = processExecutableOpen,
	442	.processExecutablePath = processExecutablePath,
	443	.lockStderr = lockStderr,
	444	.tryLockStderr = tryLockStderr,
	445	.unlockStderr = unlockStderr,
	446	.processCurrentPath = processCurrentPath,
	447	.processSetCurrentDir = processSetCurrentDir,
	448	.processReplace = processReplace,
	449	.processReplacePath = processReplacePath,
	450	.processSpawn = processSpawn,
	451	.processSpawnPath = processSpawnPath,
	452	.childWait = childWait,
	453	.childKill = childKill,
	454
	455	.progressParentFile = progressParentFile,
	456
	457	.now = now,
	458	.clockResolution = clockResolution,
	459	.sleep = sleep,
	460
	461	.random = random,
	462	.randomSecure = randomSecure,
	463
	464	.netListenIp = netListenIpUnavailable,
	465	.netAccept = netAcceptUnavailable,
	466	.netBindIp = netBindIpUnavailable,
	467	.netConnectIp = netConnectIpUnavailable,
	468	.netListenUnix = netListenUnixUnavailable,
	469	.netConnectUnix = netConnectUnixUnavailable,
	470	.netSocketCreatePair = netSocketCreatePairUnavailable,
	471	.netSend = netSendUnavailable,
	472	.netReceive = netReceiveUnavailable,
	473	.netRead = netReadUnavailable,
	474	.netWrite = netWriteUnavailable,
	475	.netWriteFile = netWriteFileUnavailable,
	476	.netClose = netClose,
	477	.netShutdown = netShutdownUnavailable,
	478	.netInterfaceNameResolve = netInterfaceNameResolveUnavailable,
	479	.netInterfaceName = netInterfaceNameUnavailable,
	480	.netLookup = netLookupUnavailable,
	481	},
	482	};
	483	}
	484
	485	pub const InitOptions = struct {
	486	backing_allocator_needs_mutex: bool = true,
	487	queue: ?c.dispatch.queue_t = null,
	488	/// Upper limit on the allowable delay in processing timeouts in order to improve power
	489	/// consumption and system performance.
	490	leeway: Io.Duration = .fromMilliseconds(10),
	491
	492	/// Affects the following operations:
	493	/// * `processExecutablePath` on OpenBSD and Haiku.
	494	argv0: Argv0 = .empty,
	495	/// Affects the following operations:
	496	/// * `fileIsTty`
	497	/// * `processSpawn`, `processSpawnPath`, `processReplace`, `processReplacePath`
	498	environ: process.Environ = .empty,
	499	};
	500
	501	pub fn init(ev: *Evented, backing_allocator: Allocator, options: InitOptions) !void {
	502	const queue = if (options.queue) \|queue\| queue: {
	503	queue.as_object().retain();
	504	break :queue queue;
	505	} else c.dispatch.queue_create("org.ziglang.std.Io.Dispatch", .CONCURRENT()) orelse
	506	return error.SystemResources;
	507	errdefer queue.as_object().release();
	508	const main_loop_stack = try backing_allocator.alignedAlloc(
	509	u8,
	510	.fromByteUnits(builtin.target.stackAlignment()),
	511	main_loop_stack_size,
	512	);
	513	errdefer backing_allocator.free(main_loop_stack);
	514	const exit_semaphore = c.dispatch.semaphore_create(0) orelse return error.SystemResources;
	515	errdefer exit_semaphore.as_object().release();
	516	ev.* = .{
	517	.queue = queue,
	518	.backing_allocator_needs_mutex = options.backing_allocator_needs_mutex,
	519	.backing_allocator_mutex = undefined,
	520	.backing_allocator = backing_allocator,
	521	.main_fiber = .{
	522	.required_align = {},
	523	.evented = ev,
	524	.context = undefined,
	525	.await_count = 0,
	526	.link = .{ .awaiter = null },
	527	.status = .{ .queue_next = null },
	528	.cancel_status = .unrequested,
	529	.cancel_protection = .unblocked,
	530	},
	531	.main_loop_stack = main_loop_stack.ptr,
	532	.exit_semaphore = exit_semaphore,
	533
	534	.use_fcopyfile = .default,
	535	.use_sendfile = .default,
	536	.leeway = std.math.lossyCast(u64, options.leeway.toNanoseconds()),
	537
	538	.futexes = undefined,
	539
	540	.init_stderr_writer = .init,
	541	.stderr_mutex = undefined,
	542	.stderr_writer = .{
	543	.io = ev.io(),
	544	.interface = Io.File.Writer.initInterface(&.{}),
	545	.file = .stderr(),
	546	.mode = .streaming,
	547	},
	548	.stderr_mode = .no_color,
	549
	550	.scan_environ = if (options.environ.block.isEmpty()) .done else .init,
	551	.environ = .{ .process_environ = options.environ },
	552
	553	.open_dev_null = .init,
	554	.dev_null_file = error.FileNotFound,
	555
	556	.csprng_mutex = undefined,
	557	.csprng = .uninitialized,
	558	};
	559	try ev.backing_allocator_mutex.init(queue);
	560	errdefer ev.backing_allocator_mutex.deinit();
	561	var initialized_futexes: usize = 0;
	562	errdefer for (ev.futexes[0..initialized_futexes]) \|*futex\| futex.deinit();
	563	for (&ev.futexes) \|*futex\| {
	564	try futex.init(queue);
	565	initialized_futexes += 1;
	566	}
	567	try ev.stderr_mutex.init(queue);
	568	errdefer ev.stderr_mutex.deinit();
	569	try ev.csprng_mutex.init(queue);
	570	errdefer ev.csprng_mutex.deinit();
	571	const thread: *Thread = .current();
	572	thread.main_context = switch (builtin.cpu.arch) {
	573	.aarch64 => .{
	574	.sp = @intFromPtr(main_loop_stack[main_loop_stack_size..].ptr),
	575	.fp = @intFromPtr(ev),
	576	.pc = @intFromPtr(&mainLoopEntry),
	577	},
	578	.x86_64 => .{
	579	.rsp = @intFromPtr(main_loop_stack[main_loop_stack_size..].ptr) - 8,
	580	.rbp = @intFromPtr(ev),
	581	.rip = @intFromPtr(&mainLoopEntry),
	582	},
	583	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
	584	};
	585	thread.current_context = &ev.main_fiber.context;
	586	}
	587
	588	pub fn deinit(ev: *Evented) void {
	589	assert(Thread.current().currentFiber() == &ev.main_fiber);
	590	ev.yield(.exit);
	591	ev.csprng_mutex.deinit();
	592	if (ev.dev_null_file) \|file\| fileClose(ev, &.{file}) else \|_\| {}
	593	ev.stderr_mutex.deinit();
	594	for (&ev.futexes) \|*futex\| futex.deinit();
	595	ev.exit_semaphore.as_object().release();
	596	ev.backing_allocator.free(ev.main_loop_stack[0..main_loop_stack_size]);
	597	ev.queue.as_object().release();
	598	}
	599
	600	fn yield(ev: *Evented, pending_task: SwitchMessage.PendingTask) void {
	601	const thread: *Thread = .current();
	602	const message: SwitchMessage = .{
	603	.contexts = .{
	604	.old = thread.current_context.?,
	605	.new = &thread.main_context,
	606	},
	607	.pending_task = pending_task,
	608	};
	609	contextSwitch(&message).handle(ev);
	610	}
	611
	612	fn mainLoopEntry() callconv(.naked) void {
	613	switch (builtin.cpu.arch) {
	614	.aarch64 => asm volatile (
	615	\\ mov x0, fp
	616	\\ mov fp, #0
	617	\\ b %[mainLoop]
	618	:
	619	: [mainLoop] "X" (&mainLoop),
	620	),
	621	.x86_64 => asm volatile (
	622	\\ movq %%rbp, %%rdi
	623	\\ xor %%ebp, %%ebp
	624	\\ jmp %[mainLoop:P]
	625	:
	626	: [mainLoop] "X" (&mainLoop),
	627	),
	628	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
	629	}
	630	}
	631
	632	fn mainLoop(ev: Evented, message: const SwitchMessage) callconv(.c) noreturn {
	633	message.handle(ev);
	634	assert(ev.exit_semaphore.wait(.FOREVER) == 0);
	635	Fiber.@"resume"(&ev.main_fiber);
	636	unreachable; // switched to dead fiber
	637	}
	638
	639	const SwitchMessage = struct {
	640	contexts: Io.fiber.Switch,
	641	pending_task: PendingTask,
	642
	643	const PendingTask = union(enum) {
	644	nothing,
	645	await: u31,
	646	activate: c.dispatch.object_t,
	647	@"resume": c.dispatch.object_t,
	648	group_await: Group,
	649	group_cancel: Group,
	650	mutex_wait: *Mutex.Waiter,
	651	futex_wait: *Futex.Waiter,
	652	futex_wake: *Futex.Waker,
	653	sleep: c.dispatch.time_t,
	654	destroy,
	655	exit,
	656	};
	657
	658	fn handle(message: const SwitchMessage, ev: Evented) void {
	659	const thread: *Thread = .current();
	660	thread.current_context = message.contexts.new;
	661	switch (message.pending_task) {
	662	.nothing => {},
	663	.await => \|count\| {
	664	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
	665	if (@atomicRmw(i32, &fiber.await_count, .Sub, count, .monotonic) > 0)
	666	ev.queue.async(fiber, &Fiber.@"resume");
	667	},
	668	.activate => \|object\| object.activate(),
	669	.@"resume" => \|object\| object.@"resume"(),
	670	.group_await => \|group\| {
	671	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
	672	if (group.await(ev, fiber)) ev.queue.async(fiber, &Fiber.@"resume");
	673	},
	674	.group_cancel => \|group\| {
	675	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
	676	if (group.cancel(ev, fiber)) ev.queue.async(fiber, &Fiber.@"resume");
	677	},
	678	.mutex_wait => \|waiter\| {
	679	waiter.sleeper =
	680	.init(ev.queue, @alignCast(@fieldParentPtr("context", message.contexts.old)));
	681	switch (waiter.sleeper.fiber.cancel_protection.check()) {
	682	.unblocked => {},
	683	.blocked => waiter.cancelable = .blocked,
	684	}
	685	waiter.mutex.queue.async(waiter, &Mutex.Waiter.add);
	686	},
	687	.futex_wait => \|waiter\| {
	688	waiter.sleeper =
	689	.init(ev.queue, @alignCast(@fieldParentPtr("context", message.contexts.old)));
	690	switch (waiter.sleeper.fiber.cancel_protection.check()) {
	691	.unblocked => {},
	692	.blocked => waiter.cancelable = .blocked,
	693	}
	694	waiter.futex.queue.async(waiter, &Futex.Waiter.add);
	695	},
	696	.futex_wake => \|waker\| {
	697	waker.sleeper =
	698	.init(ev.queue, @alignCast(@fieldParentPtr("context", message.contexts.old)));
	699	waker.futex.queue.async(waker, &Futex.Waker.remove);
	700	},
	701	.sleep => \|when\| {
	702	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
	703	when.after(ev.queue, fiber, &Fiber.@"resume");
	704	},
	705	.destroy => {
	706	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
	707	fiber.destroy(ev);
	708	},
	709	.exit => _ = ev.exit_semaphore.signal(),
	710	}
	711	}
	712	};
	713
	714	inline fn contextSwitch(message: const SwitchMessage) const SwitchMessage {
	715	return @fieldParentPtr("contexts", Io.fiber.contextSwitch(&message.contexts));
	716	}
	717
	718	const Cancelable = struct {
	719	required_align: void align(2) = {},
	720	queue: c.dispatch.queue_t,
	721	cancel: c.dispatch.function_t,
	722
	723	const is_blocked: c.dispatch.function_t =
	724	@ptrFromInt(@typeInfo(c.dispatch.function_t).pointer.alignment * 1);
	725	const is_requested: c.dispatch.function_t =
	726	@ptrFromInt(@typeInfo(c.dispatch.function_t).pointer.alignment * 2);
	727
	728	const blocked: Cancelable = .{ .queue = undefined, .cancel = is_blocked };
	729
	730	const AwaitError = error{CancelRequested};
	731
	732	fn await(cancelable: Cancelable, fiber: Fiber) AwaitError!void {
	733	const function = cancelable.cancel;
	734	assert(function != is_requested);
	735	if (function == is_blocked) {
	736	@branchHint(.unlikely);
	737	return;
	738	}
	739	if (@cmpxchgStrong(
	740	Fiber.CancelStatus,
	741	&fiber.cancel_status,
	742	.{ .requested = false, .awaiting = .nothing },
	743	.{ .requested = false, .awaiting = .fromCancelable(cancelable) },
	744	.release,
	745	.monotonic,
	746	)) \|cancel_status\| {
	747	assert(cancel_status.requested and cancel_status.awaiting == .nothing);
	748	cancelable.cancel = is_requested;
	749	return error.CancelRequested;
	750	}
	751	}
	752
	753	fn canceled(cancelable: *Cancelable) void {
	754	assert(cancelable.cancel != is_blocked);
	755	assert(cancelable.cancel != is_requested);
	756	cancelable.queue.async(cancelable, cancelable.cancel);
	757	}
	758
	759	fn check(cancelable: Cancelable, fiber: Fiber) Io.Cancelable!void {
	760	if (cancelable.cancel == is_requested) {
	761	@branchHint(.unlikely);
	762	fiber.cancel_protection.acknowledge();
	763	return error.Canceled;
	764	}
	765	}
	766	};
	767
	768	const Sleeper = struct {
	769	queue: c.dispatch.queue_t,
	770	fiber: *Fiber,
	771
	772	fn init(queue: c.dispatch.queue_t, fiber: *Fiber) Sleeper {
	773	queue.as_object().retain();
	774	return .{ .queue = queue, .fiber = fiber };
	775	}
	776
	777	fn wake(context: ?*anyopaque) callconv(.c) void {
	778	const sleeper: *Sleeper = @ptrCast(@alignCast(context));
	779	const queue = sleeper.queue;
	780	sleeper.queue = undefined;
	781	queue.async(sleeper.fiber, &Fiber.@"resume");
	782	queue.as_object().release();
	783	}
	784	};
	785
	786	const Mutex = struct {
	787	/// including the locker
	788	num_waiters: usize,
	789	queue: c.dispatch.queue_t,
	790	waiters: std.DoublyLinkedList,
	791
	792	const Waiter = struct {
	793	sleeper: Sleeper = undefined,
	794	cancelable: Cancelable,
	795	mutex: *Mutex,
	796	node: std.DoublyLinkedList.Node = .{},
	797
	798	fn add(context: ?*anyopaque) callconv(.c) void {
	799	const waiter: *Waiter = @ptrCast(@alignCast(context));
	800	waiter.tryAdd() catch \|err\| switch (err) {
	801	error.CancelRequested => {
	802	waiter.wake();
	803	assert(@atomicRmw(usize, &waiter.mutex.num_waiters, .Sub, 1, .monotonic) >= 1);
	804	},
	805	};
	806	}
	807
	808	fn tryAdd(waiter: *Waiter) Cancelable.AwaitError!void {
	809	switch (@atomicLoad(usize, &waiter.mutex.num_waiters, .acquire)) {
	810	0 => unreachable,
	811	1 => return waiter.wake(), // already locked exclusively
	812	else => try waiter.cancelable.await(waiter.sleeper.fiber),
	813	}
	814	waiter.mutex.waiters.append(&waiter.node);
	815	}
	816
	817	fn canceled(context: ?*anyopaque) callconv(.c) void {
	818	const cancelable: *Cancelable = @ptrCast(@alignCast(context));
	819	cancelable.cancel = Cancelable.is_requested;
	820	const waiter: *Waiter = @fieldParentPtr("cancelable", cancelable);
	821	assert(@atomicRmw(
	822	Fiber.CancelStatus,
	823	&waiter.sleeper.fiber.cancel_status,
	824	.Xchg,
	825	.{ .requested = true, .awaiting = .nothing },
	826	.monotonic,
	827	) == Fiber.CancelStatus{ .requested = true, .awaiting = .fromCancelable(cancelable) });
	828	const mutex = waiter.mutex;
	829	mutex.waiters.remove(&waiter.node);
	830	waiter.wake();
	831	assert(@atomicRmw(usize, &mutex.num_waiters, .Sub, 1, .monotonic) >= 1);
	832	}
	833
	834	fn remove(context: ?*anyopaque) callconv(.c) void {
	835	const mutex: *Mutex = @ptrCast(@alignCast(context));
	836	var stop_node: ?*std.DoublyLinkedList.Node = null;
	837	while (mutex.waiters.first != stop_node) {
	838	@branchHint(.likely);
	839	const waiter: *Waiter = @fieldParentPtr("node", mutex.waiters.popFirst().?);
	840	if (waiter.cancelable.cancel != Cancelable.is_blocked) {
	841	@branchHint(.likely);
	842	const cancel_status = @atomicRmw(
	843	Fiber.CancelStatus,
	844	&waiter.sleeper.fiber.cancel_status,
	845	.And,
	846	.{ .requested = true, .awaiting = .nothing },
	847	.monotonic,
	848	);
	849	assert(cancel_status.awaiting.toCancelable() == &waiter.cancelable);
	850	if (cancel_status.requested) {
	851	@branchHint(.unlikely);
	852	// carefully place the hot potato out of the way
	853	mutex.waiters.append(&waiter.node);
	854	if (stop_node == null) stop_node = &waiter.node;
	855	continue;
	856	}
	857	}
	858	waiter.wake();
	859	return;
	860	}
	861	// everyone is about to die, nobody will wake up ;-(
	862	}
	863
	864	fn wake(waiter: *Waiter) void {
	865	Sleeper.wake(&waiter.sleeper);
	866	}
	867	};
	868
	869	fn init(mutex: *Mutex, queue: c.dispatch.queue_t) error{SystemResources}!void {
	870	mutex.* = .{
	871	.num_waiters = 0,
	872	.queue = c.dispatch.queue_create_with_target(
	873	"org.ziglang.std.Io.Dispatch.Mutex",
	874	.SERIAL(),
	875	queue,
	876	) orelse return error.SystemResources,
	877	.waiters = .{},
	878	};
	879	}
	880
	881	fn deinit(mutex: *Mutex) void {
	882	assert(mutex.num_waiters == 0 and mutex.waiters.first == null and mutex.waiters.last == null);
	883	mutex.queue.as_object().release();
	884	mutex.* = undefined;
	885	}
	886
	887	fn tryLock(mutex: *Mutex) bool {
	888	if (@cmpxchgWeak(usize, &mutex.num_waiters, 0, 1, .acquire, .monotonic) == null) {
	889	@branchHint(.likely);
	890	return true;
	891	}
	892	return false;
	893	}
	894
	895	fn lock(mutex: Mutex, ev: Evented) Io.Cancelable!void {
	896	switch (@atomicRmw(usize, &mutex.num_waiters, .Add, 1, .acquire)) {
	897	0 => {},
	898	else => {
	899	@branchHint(.unlikely);
	900	var waiter: Waiter = .{
	901	.cancelable = .{ .queue = mutex.queue, .cancel = &Mutex.Waiter.canceled },
	902	.mutex = mutex,
	903	};
	904	ev.yield(.{ .mutex_wait = &waiter });
	905	try waiter.cancelable.check(waiter.sleeper.fiber);
	906	},
	907	}
	908	}
	909
	910	fn lockUncancelable(mutex: Mutex, ev: Evented) void {
	911	switch (@atomicRmw(usize, &mutex.num_waiters, .Add, 1, .acquire)) {
	912	0 => {},
	913	else => {
	914	@branchHint(.unlikely);
	915	var waiter: Waiter = .{ .cancelable = .blocked, .mutex = mutex };
	916	ev.yield(.{ .mutex_wait = &waiter });
	917	waiter.cancelable.check(waiter.sleeper.fiber) catch \|err\| switch (err) {
	918	error.Canceled => unreachable, // blocked
	919	};
	920	},
	921	}
	922	}
	923
	924	fn unlock(mutex: *Mutex) void {
	925	switch (@atomicRmw(usize, &mutex.num_waiters, .Sub, 1, .release)) {
	926	0 => unreachable,
	927	1 => {},
	928	else => {
	929	@branchHint(.unlikely);
	930	mutex.queue.async(mutex, &Waiter.remove);
	931	},
	932	}
	933	}
	934	};
	935
	936	fn crashHandler(userdata: ?*anyopaque) void {
	937	const ev: *Evented = @ptrCast(@alignCast(userdata));
	938	_ = ev;
	939	const thread = &Thread.self;
	940	if (thread.current_context == null) std.process.abort();
	941	if (thread.current_context == &thread.main_context) std.process.abort();
	942	const fiber = thread.currentFiber();
	943	@atomicStore(
	944	Fiber.CancelStatus,
	945	&fiber.cancel_status,
	946	.{ .requested = true, .awaiting = .nothing },
	947	.monotonic,
	948	);
	949	fiber.cancel_protection = .{ .user = .blocked, .acknowledged = true };
	950	}
	951
	952	const AsyncClosure = struct {
	953	ev: *Evented,
	954	fiber: *Fiber,
	955	start: const fn (context: const anyopaque, result: *anyopaque) void,
	956	result_align: Alignment,
	957
	958	fn fromFiber(fiber: Fiber) AsyncClosure {
	959	return @ptrFromInt(Fiber.max_context_align.max(.of(AsyncClosure)).backward(
	960	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
	961	) - @sizeOf(AsyncClosure));
	962	}
	963
	964	fn contextPointer(closure: AsyncClosure) []align(Fiber.max_context_align.toByteUnits()) u8 {
	965	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(AsyncClosure));
	966	}
	967
	968	fn entry() callconv(.naked) void {
	969	switch (builtin.cpu.arch) {
	970	.aarch64 => asm volatile (
	971	\\ mov x0, sp
	972	\\ b %[call]
	973	:
	974	: [call] "X" (&call),
	975	),
	976	.x86_64 => asm volatile (
	977	\\ leaq 8(%%rsp), %%rdi
	978	\\ jmp %[call:P]
	979	:
	980	: [call] "X" (&call),
	981	),
	982	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
	983	}
	984	}
	985
	986	fn call(
	987	closure: *AsyncClosure,
	988	message: *const SwitchMessage,
	989	) callconv(.withStackAlign(.c, @alignOf(AsyncClosure))) noreturn {
	990	message.handle(closure.ev);
	991	const fiber = closure.fiber;
	992	closure.start(closure.contextPointer(), fiber.resultBytes(closure.result_align));
	993	if (@atomicRmw(?*Fiber, &fiber.link.awaiter, .Xchg, Fiber.finished, .acq_rel)) \|awaiter\|
	994	if (@atomicRmw(i32, &awaiter.await_count, .Add, 1, .monotonic) == -1)
	995	closure.ev.queue.async(awaiter, &Fiber.@"resume");
	996	closure.ev.yield(.nothing);
	997	unreachable; // switched to dead fiber
	998	}
	999	};
	1000
	1001	fn async(
	1002	userdata: ?*anyopaque,
	1003	result: []u8,
	1004	result_alignment: Alignment,
	1005	context: []const u8,
	1006	context_alignment: Alignment,
	1007	start: const fn (context: const anyopaque, result: *anyopaque) void,
	1008	) ?*std.Io.AnyFuture {
	1009	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1010	return concurrent(ev, result.len, result_alignment, context, context_alignment, start) catch {
	1011	start(context.ptr, result.ptr);
	1012	return null;
	1013	};
	1014	}
	1015
	1016	fn concurrent(
	1017	userdata: ?*anyopaque,
	1018	result_len: usize,
	1019	result_alignment: Alignment,
	1020	context: []const u8,
	1021	context_alignment: Alignment,
	1022	start: const fn (context: const anyopaque, result: *anyopaque) void,
	1023	) Io.ConcurrentError!*std.Io.AnyFuture {
	1024	assert(result_alignment.compare(.lte, Fiber.max_result_align)); // TODO
	1025	assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
	1026	assert(result_len <= Fiber.max_result_size); // TODO
	1027	assert(context.len <= Fiber.max_context_size); // TODO
	1028
	1029	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1030	const fiber = Fiber.create(ev) catch \|err\| switch (err) {
	1031	error.OutOfMemory => return error.ConcurrencyUnavailable,
	1032	};
	1033
	1034	const closure: *AsyncClosure = .fromFiber(fiber);
	1035	fiber.* = .{
	1036	.required_align = {},
	1037	.evented = ev,
	1038	.context = switch (builtin.cpu.arch) {
	1039	.aarch64 => .{
	1040	.sp = @intFromPtr(closure),
	1041	.fp = 0,
	1042	.pc = @intFromPtr(&AsyncClosure.entry),
	1043	},
	1044	.x86_64 => .{
	1045	.rsp = @intFromPtr(closure) - 8,
	1046	.rbp = 0,
	1047	.rip = @intFromPtr(&AsyncClosure.entry),
	1048	},
	1049	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
	1050	},
	1051	.await_count = 0,
	1052	.link = .{ .awaiter = null },
	1053	.status = .{ .queue_next = null },
	1054	.cancel_status = .unrequested,
	1055	.cancel_protection = .unblocked,
	1056	};
	1057	closure.* = .{
	1058	.ev = ev,
	1059	.fiber = fiber,
	1060	.start = start,
	1061	.result_align = result_alignment,
	1062	};
	1063	@memcpy(closure.contextPointer(), context);
	1064
	1065	ev.queue.async(fiber, &Fiber.@"resume");
	1066	return @ptrCast(fiber);
	1067	}
	1068
	1069	fn await(
	1070	userdata: ?*anyopaque,
	1071	future: *std.Io.AnyFuture,
	1072	result: []u8,
	1073	result_alignment: Alignment,
	1074	) void {
	1075	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1076	const fiber = Thread.current().currentFiber();
	1077	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
	1078	if (@atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, fiber, .acq_rel)) \|awaiter\| {
	1079	assert(awaiter == Fiber.finished);
	1080	} else while (true) {
	1081	ev.yield(.{ .await = 1 });
	1082	const awaiter = @atomicLoad(?*Fiber, &future_fiber.link.awaiter, .acquire);
	1083	if (awaiter == Fiber.finished) break;
	1084	assert(awaiter == fiber); // spurious wakeup
	1085	}
	1086	@memcpy(result, future_fiber.resultBytes(result_alignment));
	1087	future_fiber.destroy(ev);
	1088	}
	1089
	1090	fn cancel(
	1091	userdata: ?*anyopaque,
	1092	future: *std.Io.AnyFuture,
	1093	result: []u8,
	1094	result_alignment: Alignment,
	1095	) void {
	1096	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1097	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
	1098	future_fiber.requestCancel(ev);
	1099	await(ev, future, result, result_alignment);
	1100	}
	1101
	1102	const Group = struct {
	1103	ptr: *Io.Group,
	1104
	1105	const List = packed struct(usize) {
	1106	cancel_requested: bool,
	1107	awaiter_delayed: bool,
	1108	fibers: Fiber.PackedPtr,
	1109	};
	1110	fn listPtr(group: Group) *List {
	1111	return @ptrCast(&group.ptr.token);
	1112	}
	1113
	1114	const Mutex = packed struct(u32) {
	1115	locked: bool,
	1116	contended: bool,
	1117	shared2: u30,
	1118	};
	1119	fn mutexPtr(group: Group) *Group.Mutex {
	1120	return switch (comptime builtin.cpu.arch.endian()) {
	1121	.little => @ptrCast(&group.ptr.state),
	1122	.big => @ptrCast(@alignCast(
	1123	@as([*]u8, @ptrCast(&group.ptr.state)) + @sizeOf(usize) - @sizeOf(u32),
	1124	)),
	1125	};
	1126	}
	1127
	1128	const Awaiter = packed struct(usize) {
	1129	locked: bool,
	1130	contended: bool,
	1131	awaiter: Fiber.PackedPtr,
	1132	};
	1133	fn awaiterPtr(group: Group) *Awaiter {
	1134	return @ptrCast(&group.ptr.state);
	1135	}
	1136
	1137	fn lock(group: Group, ev: *Evented) void {
	1138	const mutex = group.mutexPtr();
	1139	{
	1140	const old_state = @atomicRmw(
	1141	Group.Mutex,
	1142	mutex,
	1143	.Or,
	1144	.{ .locked = true, .contended = false, .shared2 = 0 },
	1145	.acquire,
	1146	);
	1147	if (!old_state.locked) {
	1148	@branchHint(.likely);
	1149	return;
	1150	}
	1151	if (old_state.contended) {
	1152	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
	1153	}
	1154	}
	1155	while (true) {
	1156	var old_state = @atomicRmw(
	1157	Group.Mutex,
	1158	mutex,
	1159	.Or,
	1160	.{ .locked = true, .contended = true, .shared2 = 0 },
	1161	.acquire,
	1162	);
	1163	if (!old_state.locked) {
	1164	@branchHint(.likely);
	1165	return;
	1166	}
	1167	old_state.contended = true;
	1168	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
	1169	}
	1170	}
	1171
	1172	fn unlock(group: Group, ev: *Evented) void {
	1173	const mutex = group.mutexPtr();
	1174	const old_state = @atomicRmw(
	1175	Group.Mutex,
	1176	mutex,
	1177	.And,
	1178	.{ .locked = false, .contended = false, .shared2 = std.math.maxInt(u30) },
	1179	.release,
	1180	);
	1181	assert(old_state.locked);
	1182	if (old_state.contended) futexWake(ev, @ptrCast(mutex), 1);
	1183	}
	1184
	1185	fn addFiber(group: Group, ev: Evented, fiber: Fiber) void {
	1186	group.lock(ev);
	1187	defer group.unlock(ev);
	1188	const list_ptr = group.listPtr();
	1189	const list = @atomicLoad(List, list_ptr, .monotonic);
	1190	if (list.cancel_requested) fiber.cancel_status = .{ .requested = true, .awaiting = .nothing };
	1191	const old_head = list.fibers.unpack();
	1192	if (old_head) \|head\| head.link.group.prev = fiber;
	1193	fiber.link.group.next = old_head;
	1194	@atomicStore(List, list_ptr, .{
	1195	.cancel_requested = list.cancel_requested,
	1196	.awaiter_delayed = list.awaiter_delayed,
	1197	.fibers = .pack(fiber),
	1198	}, .monotonic);
	1199	}
	1200
	1201	fn removeFiber(group: Group, ev: Evented, fiber: Fiber) ?*Fiber {
	1202	group.lock(ev);
	1203	defer group.unlock(ev);
	1204	const list_ptr = group.listPtr();
	1205	const list = @atomicLoad(List, list_ptr, .monotonic);
	1206	if (fiber.link.group.next) \|next\| next.link.group.prev = fiber.link.group.prev;
	1207	if (fiber.link.group.prev) \|prev\| {
	1208	prev.link.group.next = fiber.link.group.next;
	1209	} else if (fiber.link.group.next) \|new_head\| {
	1210	@atomicStore(List, list_ptr, .{
	1211	.cancel_requested = list.cancel_requested,
	1212	.awaiter_delayed = list.awaiter_delayed,
	1213	.fibers = .pack(new_head),
	1214	}, .monotonic);
	1215	} else if (@atomicLoad(Awaiter, group.awaiterPtr(), .monotonic).awaiter.unpack()) \|awaiter\| {
	1216	if (!awaiter.cancel_status.changeAwaiting(.group, .nothing) or list.cancel_requested) {
	1217	@atomicStore(List, list_ptr, .{
	1218	.cancel_requested = false,
	1219	.awaiter_delayed = false,
	1220	.fibers = .null,
	1221	}, .release);
	1222	assert(awaiter.status.awaiting_group.ptr == group.ptr);
	1223	awaiter.status = .{ .queue_next = null };
	1224	return awaiter;
	1225	}
	1226	// Race with `Fiber.requestCancel`
	1227	@atomicStore(List, list_ptr, .{
	1228	.cancel_requested = false,
	1229	.awaiter_delayed = true,
	1230	.fibers = .null,
	1231	}, .monotonic);
	1232	} else @atomicStore(List, list_ptr, .{
	1233	.cancel_requested = false,
	1234	.awaiter_delayed = false,
	1235	.fibers = .null,
	1236	}, .release);
	1237	return null;
	1238	}
	1239
	1240	fn await(group: Group, ev: Evented, awaiter: Fiber) bool {
	1241	group.lock(ev);
	1242	defer group.unlock(ev);
	1243	if (@atomicLoad(List, group.listPtr(), .monotonic).fibers.unpack()) \|_\| {
	1244	if (group.registerAwaiter(awaiter) and awaiter.cancel_protection.check() == .unblocked) {
	1245	// The awaiter already had an unacknowledged cancelation request before
	1246	// attempting to await a group, so propagate the cancelation to the group.
	1247	assert(!group.cancelLocked(ev, null));
	1248	}
	1249	return false;
	1250	}
	1251	return true;
	1252	}
	1253
	1254	fn cancel(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
	1255	group.lock(ev);
	1256	defer group.unlock(ev);
	1257	return group.cancelLocked(ev, maybe_awaiter);
	1258	}
	1259
	1260	/// Assumes the mutex is held.
	1261	fn cancelLocked(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
	1262	const list_ptr = group.listPtr();
	1263	const list = @atomicRmw(
	1264	List,
	1265	list_ptr,
	1266	.Add,
	1267	.{ .cancel_requested = true, .awaiter_delayed = false, .fibers = .null },
	1268	.monotonic,
	1269	);
	1270	assert(!list.cancel_requested);
	1271	if (list.fibers.unpack()) \|head\| {
	1272	var maybe_fiber: ?*Fiber = head;
	1273	while (maybe_fiber) \|fiber\| {
	1274	fiber.requestCancel(ev);
	1275	maybe_fiber = fiber.link.group.next;
	1276	}
	1277	if (maybe_awaiter) \|awaiter\| _ = group.registerAwaiter(awaiter);
	1278	return false;
	1279	}
	1280	@atomicStore(
	1281	List,
	1282	list_ptr,
	1283	.{ .cancel_requested = false, .awaiter_delayed = false, .fibers = .null },
	1284	.release,
	1285	);
	1286	return if (maybe_awaiter) \|_\| true else list.awaiter_delayed;
	1287	}
	1288
	1289	/// Assumes the mutex is held.
	1290	fn registerAwaiter(group: Group, awaiter: *Fiber) bool {
	1291	assert(awaiter.status.queue_next == null);
	1292	awaiter.status = .{ .awaiting_group = group };
	1293	assert(@atomicRmw(
	1294	Awaiter,
	1295	group.awaiterPtr(),
	1296	.Add,
	1297	.{ .locked = false, .contended = false, .awaiter = .pack(awaiter) },
	1298	.monotonic,
	1299	).awaiter == .null);
	1300	return awaiter.cancel_status.changeAwaiting(.nothing, .group);
	1301	}
	1302
	1303	const AsyncClosure = struct {
	1304	ev: *Evented,
	1305	group: Group,
	1306	fiber: *Fiber,
	1307	start: const fn (context: const anyopaque) Io.Cancelable!void,
	1308
	1309	fn fromFiber(fiber: Fiber) Group.AsyncClosure {
	1310	return @ptrFromInt(Fiber.max_context_align.max(.of(Group.AsyncClosure)).backward(
	1311	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
	1312	) - @sizeOf(Group.AsyncClosure));
	1313	}
	1314
	1315	fn contextPointer(
	1316	closure: *Group.AsyncClosure,
	1317	) [*]align(Fiber.max_context_align.toByteUnits()) u8 {
	1318	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(Group.AsyncClosure));
	1319	}
	1320
	1321	fn entry() callconv(.naked) void {
	1322	switch (builtin.cpu.arch) {
	1323	.aarch64 => asm volatile (
	1324	\\ mov x0, sp
	1325	\\ b %[call]
	1326	:
	1327	: [call] "X" (&call),
	1328	),
	1329	.x86_64 => asm volatile (
	1330	\\ leaq 8(%%rsp), %%rdi
	1331	\\ jmp %[call:P]
	1332	:
	1333	: [call] "X" (&call),
	1334	),
	1335	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
	1336	}
	1337	}
	1338
	1339	fn call(
	1340	closure: *Group.AsyncClosure,
	1341	message: *const SwitchMessage,
	1342	) callconv(.withStackAlign(.c, @alignOf(Group.AsyncClosure))) noreturn {
	1343	message.handle(closure.ev);
	1344	assert(closure.fiber.status.queue_next == null);
	1345	const result = closure.start(closure.contextPointer());
	1346	const ev = closure.ev;
	1347	const group = closure.group;
	1348	const fiber = closure.fiber;
	1349	const cancel_acknowledged = fiber.cancel_protection.acknowledged;
	1350	if (result) {
	1351	assert(!cancel_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
	1352	} else \|err\| switch (err) {
	1353	error.Canceled => assert(cancel_acknowledged), // group task returned `error.Canceled` but was never canceled
	1354	}
	1355	if (group.removeFiber(ev, fiber)) \|awaiter\| ev.queue.async(awaiter, &Fiber.@"resume");
	1356	ev.yield(.destroy);
	1357	unreachable; // switched to dead fiber
	1358	}
	1359	};
	1360	};
	1361
	1362	fn groupAsync(
	1363	userdata: ?*anyopaque,
	1364	type_erased: *Io.Group,
	1365	context: []const u8,
	1366	context_alignment: Alignment,
	1367	start: const fn (context: const anyopaque) Io.Cancelable!void,
	1368	) void {
	1369	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1370	return groupConcurrent(ev, type_erased, context, context_alignment, start) catch {
	1371	const fiber = Thread.current().currentFiber();
	1372	const pre_acknowledged = fiber.cancel_protection.acknowledged;
	1373	const result = start(context.ptr);
	1374	const post_acknowledged = fiber.cancel_protection.acknowledged;
	1375	if (result) {
	1376	if (pre_acknowledged) {
	1377	assert(post_acknowledged); // group task called `recancel` but was not canceled
	1378	} else {
	1379	assert(!post_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
	1380	}
	1381	} else \|err\| switch (err) {
	1382	// Don't swallow the cancelation: make it visible to the `Group.async` caller.
	1383	error.Canceled => {
	1384	assert(!pre_acknowledged); // group task called `recancel` but was not canceled
	1385	assert(post_acknowledged); // group task returned `error.Canceled` but was never canceled
	1386	fiber.cancel_protection.recancel();
	1387	},
	1388	}
	1389	};
	1390	}
	1391
	1392	fn groupConcurrent(
	1393	userdata: ?*anyopaque,
	1394	type_erased: *Io.Group,
	1395	context: []const u8,
	1396	context_alignment: Alignment,
	1397	start: const fn (context: const anyopaque) Io.Cancelable!void,
	1398	) Io.ConcurrentError!void {
	1399	assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
	1400	assert(context.len <= Fiber.max_context_size); // TODO
	1401
	1402	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1403	const group: Group = .{ .ptr = type_erased };
	1404	const fiber = Fiber.create(ev) catch \|err\| switch (err) {
	1405	error.OutOfMemory => return error.ConcurrencyUnavailable,
	1406	};
	1407
	1408	const closure: *Group.AsyncClosure = .fromFiber(fiber);
	1409	fiber.* = .{
	1410	.required_align = {},
	1411	.evented = ev,
	1412	.context = switch (builtin.cpu.arch) {
	1413	.aarch64 => .{
	1414	.sp = @intFromPtr(closure),
	1415	.fp = 0,
	1416	.pc = @intFromPtr(&Group.AsyncClosure.entry),
	1417	},
	1418	.x86_64 => .{
	1419	.rsp = @intFromPtr(closure) - 8,
	1420	.rbp = 0,
	1421	.rip = @intFromPtr(&Group.AsyncClosure.entry),
	1422	},
	1423	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
	1424	},
	1425	.await_count = 0,
	1426	.link = .{ .group = .{ .prev = null, .next = null } },
	1427	.status = .{ .queue_next = null },
	1428	.cancel_status = .unrequested,
	1429	.cancel_protection = .unblocked,
	1430	};
	1431	closure.* = .{
	1432	.ev = ev,
	1433	.group = group,
	1434	.fiber = fiber,
	1435	.start = start,
	1436	};
	1437	@memcpy(closure.contextPointer(), context);
	1438	group.addFiber(ev, fiber);
	1439	ev.queue.async(fiber, &Fiber.@"resume");
	1440	}
	1441
	1442	fn groupAwait(
	1443	userdata: ?*anyopaque,
	1444	type_erased: *Io.Group,
	1445	initial_token: *anyopaque,
	1446	) Io.Cancelable!void {
	1447	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1448	_ = initial_token;
	1449	ev.yield(.{ .group_await = .{ .ptr = type_erased } });
	1450	}
	1451
	1452	fn groupCancel(userdata: ?anyopaque, type_erased: Io.Group, initial_token: *anyopaque) void {
	1453	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1454	_ = initial_token;
	1455	ev.yield(.{ .group_cancel = .{ .ptr = type_erased } });
	1456	}
	1457
	1458	fn recancel(userdata: ?*anyopaque) void {
	1459	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1460	_ = ev;
	1461	Thread.current().currentFiber().cancel_protection.recancel();
	1462	}
	1463
	1464	fn swapCancelProtection(userdata: ?*anyopaque, new: Io.CancelProtection) Io.CancelProtection {
	1465	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1466	_ = ev;
	1467	const cancel_protection = &Thread.current().currentFiber().cancel_protection;
	1468	defer cancel_protection.user = new;
	1469	return cancel_protection.user;
	1470	}
	1471
	1472	fn checkCancel(userdata: ?*anyopaque) Io.Cancelable!void {
	1473	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1474	_ = ev;
	1475	const fiber = Thread.current().currentFiber();
	1476	switch (fiber.cancel_protection.check()) {
	1477	.unblocked => {
	1478	const cancel_status = @atomicLoad(Fiber.CancelStatus, &fiber.cancel_status, .monotonic);
	1479	assert(cancel_status.awaiting == .nothing);
	1480	if (cancel_status.requested) {
	1481	@branchHint(.unlikely);
	1482	fiber.cancel_protection.acknowledge();
	1483	return error.Canceled;
	1484	}
	1485	},
	1486	.blocked => {},
	1487	}
	1488	}
	1489
	1490	const Futex = struct {
	1491	num_waiters: usize,
	1492	queue: c.dispatch.queue_t,
	1493	waiters: std.DoublyLinkedList,
	1494
	1495	const Waiter = struct {
	1496	sleeper: Sleeper = undefined,
	1497	cancelable: Cancelable,
	1498	futex: *Futex,
	1499	node: std.DoublyLinkedList.Node = .{},
	1500	ptr: *const u32,
	1501	expected: u32,
	1502	timeout: c.dispatch.time_t = .FOREVER,
	1503	leeway: u64,
	1504	timer: ?c.dispatch.source_t = null,
	1505
	1506	const already_signaled: c.dispatch.source_t = @ptrFromInt(1);
	1507
	1508	fn add(context: ?*anyopaque) callconv(.c) void {
	1509	const waiter: *Waiter = @ptrCast(@alignCast(context));
	1510	const futex = waiter.futex;
	1511	_ = @atomicRmw(usize, &futex.num_waiters, .Add, 1, .acquire);
	1512	waiter.tryAdd() catch \|err\| switch (err) {
	1513	error.CancelRequested => {
	1514	wake(waiter);
	1515	assert(@atomicRmw(usize, &futex.num_waiters, .Sub, 1, .monotonic) >= 1);
	1516	},
	1517	};
	1518	}
	1519
	1520	fn tryAdd(waiter: *Waiter) Cancelable.AwaitError!void {
	1521	if (@atomicLoad(u32, waiter.ptr, .monotonic) != waiter.expected)
	1522	return error.CancelRequested;
	1523	try waiter.cancelable.await(waiter.sleeper.fiber);
	1524	const futex = waiter.futex;
	1525	switch (waiter.timeout) {
	1526	.FOREVER => {},
	1527	else => \|timeout\| {
	1528	const timer = c.dispatch.source_create(.TIMER, 0, .none, futex.queue) orelse {
	1529	log.warn("unable to create timer for futex timeout", .{});
	1530	return error.CancelRequested;
	1531	};
	1532	timer.as_object().set_context(waiter);
	1533	timer.set_event_handler(&timedOut);
	1534	timer.set_cancel_handler(&wake);
	1535	timer.set_timer(timeout, c.dispatch.TIME_FOREVER, waiter.leeway);
	1536	timer.as_object().activate();
	1537	waiter.timer = timer;
	1538	},
	1539	}
	1540	futex.waiters.append(&waiter.node);
	1541	}
	1542
	1543	fn canceled(context: ?*anyopaque) callconv(.c) void {
	1544	const cancelable: *Cancelable = @ptrCast(@alignCast(context));
	1545	cancelable.cancel = Cancelable.is_requested;
	1546	const waiter: *Waiter = @fieldParentPtr("cancelable", cancelable);
	1547	assert(@atomicRmw(
	1548	Fiber.CancelStatus,
	1549	&waiter.sleeper.fiber.cancel_status,
	1550	.Xchg,
	1551	.{ .requested = true, .awaiting = .nothing },
	1552	.monotonic,
	1553	) == Fiber.CancelStatus{ .requested = true, .awaiting = .fromCancelable(cancelable) });
	1554	const futex = waiter.futex;
	1555	waiter.removeUncancelable();
	1556	assert(@atomicRmw(usize, &futex.num_waiters, .Sub, 1, .monotonic) >= 1);
	1557	}
	1558
	1559	fn timedOut(context: ?*anyopaque) callconv(.c) void {
	1560	const waiter: *Waiter = @ptrCast(@alignCast(context));
	1561	const futex = waiter.futex;
	1562	waiter.remove() catch \|err\| switch (err) {
	1563	error.CancelRequested => return,
	1564	};
	1565	assert(@atomicRmw(usize, &futex.num_waiters, .Sub, 1, .monotonic) >= 1);
	1566	}
	1567
	1568	fn remove(waiter: *Waiter) Cancelable.AwaitError!void {
	1569	if (waiter.cancelable.cancel != Cancelable.is_blocked) {
	1570	@branchHint(.likely);
	1571	const cancel_status = @atomicRmw(
	1572	Fiber.CancelStatus,
	1573	&waiter.sleeper.fiber.cancel_status,
	1574	.And,
	1575	.{ .requested = true, .awaiting = .nothing },
	1576	.monotonic,
	1577	);
	1578	assert(cancel_status.awaiting.toCancelable() == &waiter.cancelable);
	1579	if (cancel_status.requested) return error.CancelRequested;
	1580	}
	1581	waiter.removeUncancelable();
	1582	}
	1583
	1584	fn removeUncancelable(waiter: *Waiter) void {
	1585	waiter.futex.waiters.remove(&waiter.node);
	1586	if (waiter.timer) \|timer\| timer.cancel() else wake(waiter);
	1587	}
	1588
	1589	fn wake(context: ?*anyopaque) callconv(.c) void {
	1590	const waiter: *Waiter = @ptrCast(@alignCast(context));
	1591	if (waiter.timer) \|timer\| timer.as_object().release();
	1592	Sleeper.wake(&waiter.sleeper);
	1593	}
	1594	};
	1595
	1596	const Waker = struct {
	1597	sleeper: Sleeper = undefined,
	1598	futex: *Futex,
	1599	ptr: *const u32,
	1600	max_waiters: u32,
	1601
	1602	fn remove(context: ?*anyopaque) callconv(.c) void {
	1603	const waker: *Waker = @ptrCast(@alignCast(context));
	1604	const futex = waker.futex;
	1605	const ptr = waker.ptr;
	1606	const max_waiters = waker.max_waiters;
	1607
	1608	var num_removed: usize = 0;
	1609	var next_node = futex.waiters.first;
	1610	while (num_removed < max_waiters) {
	1611	const waiter: *Waiter = @fieldParentPtr("node", next_node orelse break);
	1612	next_node = waiter.node.next;
	1613	if (waiter.ptr != ptr) {
	1614	@branchHint(.unlikely);
	1615	continue;
	1616	}
	1617	waiter.remove() catch \|err\| switch (err) {
	1618	error.CancelRequested => continue,
	1619	};
	1620	num_removed += 1;
	1621	}
	1622	assert(@atomicRmw(usize, &futex.num_waiters, .Sub, num_removed, .monotonic) >= num_removed);
	1623
	1624	var sleeper = waker.sleeper;
	1625	waker.* = undefined;
	1626	Sleeper.wake(&sleeper);
	1627	}
	1628	};
	1629
	1630	fn init(futex: *Futex, queue: c.dispatch.queue_t) error{SystemResources}!void {
	1631	futex.* = .{
	1632	.num_waiters = 0,
	1633	.queue = c.dispatch.queue_create_with_target(
	1634	"org.ziglang.std.Io.Dispatch.Futex",
	1635	.SERIAL(),
	1636	queue,
	1637	) orelse return error.SystemResources,
	1638	.waiters = .{},
	1639	};
	1640	}
	1641
	1642	fn deinit(futex: *Futex) void {
	1643	assert(futex.num_waiters == 0 and futex.waiters.first == null and futex.waiters.last == null);
	1644	futex.queue.as_object().release();
	1645	futex.* = undefined;
	1646	}
	1647	};
	1648
	1649	fn futexForAddress(ev: Evented, address: usize) Futex {
	1650	// Here we use Fibonacci hashing: the golden ratio can be used to evenly redistribute input
	1651	// values across a range, giving a poor, but extremely quick to compute, hash.
	1652
	1653	// This literal is the rounded value of '2^64 / phi' (where 'phi' is the golden ratio). The
	1654	// shift then converts it to '2^b / phi', where 'b' is the pointer bit width.
	1655	const fibonacci_multiplier = 0x9E3779B97F4A7C15 >> (64 - @bitSizeOf(usize));
	1656	const hashed = address *% fibonacci_multiplier;
	1657	comptime assert(std.math.isPowerOfTwo(ev.futexes.len));
	1658	// The high bits of `hashed` have better entropy than the low bits.
	1659	return &ev.futexes[hashed >> @clz(ev.futexes.len - 1)];
	1660	}
	1661
	1662	fn select(userdata: ?anyopaque, futures: []const Io.AnyFuture) Io.Cancelable!usize {
	1663	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1664	const fiber = Thread.current().currentFiber();
	1665	var await_count: u31, var result = for (futures, 0..) \|future, future_index\| {
	1666	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
	1667	if (@atomicRmw(
	1668	?*Fiber,
	1669	&future_fiber.link.awaiter,
	1670	.Xchg,
	1671	fiber,
	1672	.acq_rel,
	1673	)) \|awaiter\| {
	1674	assert(awaiter == Fiber.finished);
	1675	break .{ @intCast(future_index), future_index };
	1676	}
	1677	} else result: {
	1678	const await_count: u31 = @intCast(futures.len);
	1679	ev.yield(.{ .await = 1 });
	1680	break :result .{ await_count - 1, futures.len };
	1681	};
	1682	for (futures[0..result], 0..) \|future, future_index\| {
	1683	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
	1684	const awaiter = @atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, null, .monotonic);
	1685	if (awaiter == Fiber.finished) {
	1686	@atomicStore(?*Fiber, &future_fiber.link.awaiter, Fiber.finished, .monotonic);
	1687	result = @min(future_index, result);
	1688	} else {
	1689	assert(awaiter == fiber);
	1690	await_count -= 1;
	1691	}
	1692	}
	1693	// Equivalent to `ev.yield(null, .{ .await = await_count });`,
	1694	// but avoiding a context switch in the common case.
	1695	switch (std.math.order(
	1696	@atomicRmw(i32, &fiber.await_count, .Sub, await_count, .monotonic),
	1697	await_count,
	1698	)) {
	1699	.lt => ev.yield(.{ .await = 0 }),
	1700	.eq => {},
	1701	.gt => unreachable,
	1702	}
	1703	return result;
	1704	}
	1705
	1706	fn futexWait(
	1707	userdata: ?*anyopaque,
	1708	ptr: *const u32,
	1709	expected: u32,
	1710	timeout: Io.Timeout,
	1711	) Io.Cancelable!void {
	1712	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1713	const futex = ev.futexForAddress(@intFromPtr(ptr));
	1714	var waiter: Futex.Waiter = .{
	1715	.cancelable = .{ .queue = futex.queue, .cancel = &Futex.Waiter.canceled },
	1716	.futex = futex,
	1717	.ptr = ptr,
	1718	.expected = expected,
	1719	.timeout = ev.timeFromTimeout(timeout),
	1720	.leeway = ev.leeway,
	1721	};
	1722	ev.yield(.{ .futex_wait = &waiter });
	1723	try waiter.cancelable.check(waiter.sleeper.fiber);
	1724	}
	1725
	1726	fn futexWaitUncancelable(userdata: ?anyopaque, ptr: const u32, expected: u32) void {
	1727	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1728	const futex = ev.futexForAddress(@intFromPtr(ptr));
	1729	var waiter: Futex.Waiter = .{
	1730	.cancelable = .blocked,
	1731	.futex = futex,
	1732	.ptr = ptr,
	1733	.expected = expected,
	1734	.leeway = ev.leeway,
	1735	};
	1736	ev.yield(.{ .futex_wait = &waiter });
	1737	waiter.cancelable.check(waiter.sleeper.fiber) catch \|err\| switch (err) {
	1738	error.Canceled => unreachable, // blocked
	1739	};
	1740	}
	1741
	1742	fn futexWake(userdata: ?anyopaque, ptr: const u32, max_waiters: u32) void {
	1743	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1744	if (max_waiters == 0) return;
	1745	const futex = ev.futexForAddress(@intFromPtr(ptr));
	1746	switch (@atomicRmw(usize, &futex.num_waiters, .Add, 0, .release)) {
	1747	0 => return,
	1748	else => {
	1749	@branchHint(.unlikely);
	1750	var waker: Futex.Waker = .{ .futex = futex, .ptr = ptr, .max_waiters = max_waiters };
	1751	ev.yield(.{ .futex_wake = &waker });
	1752	},
	1753	}
	1754	}
	1755
	1756	fn operate(userdata: ?*anyopaque, operation: Io.Operation) Io.Cancelable!Io.Operation.Result {
	1757	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1758	switch (operation) {
	1759	.file_read_streaming => \|o\| return .{
	1760	.file_read_streaming = ev.fileReadStreaming(o.file, o.data) catch \|err\| switch (err) {
	1761	error.Canceled => \|e\| return e,
	1762	else => \|e\| e,
	1763	},
	1764	},
	1765	.file_write_streaming => \|o\| return .{
	1766	.file_write_streaming = ev.fileWriteStreaming(
	1767	o.file,
	1768	o.header,
	1769	o.data,
	1770	o.splat,
	1771	) catch \|err\| switch (err) {
	1772	error.Canceled => \|e\| return e,
	1773	else => \|e\| e,
	1774	},
	1775	},
	1776	.device_io_control => \|*o\| return .{ .device_io_control = try deviceIoControl(o) },
	1777	}
	1778	}
	1779
	1780	fn fileReadStreaming(ev: *Evented, file: File, data: []const []u8) File.ReadStreamingError!usize {
	1781	if (file.flags.nonblocking) nonblocking: {
	1782	return fileReadStreamingLimit(file.handle, data, .unlimited) catch \|err\| switch (err) {
	1783	error.WouldBlock => break :nonblocking,
	1784	else => \|e\| return e,
	1785	};
	1786	}
	1787	const source = c.dispatch.source_create(
	1788	.READ,
	1789	@bitCast(@as(isize, file.handle)),
	1790	.none,
	1791	ev.queue,
	1792	) orelse return error.SystemResources;
	1793	source.as_object().set_context(Thread.current().currentFiber());
	1794	source.set_event_handler(&Fiber.@"resume");
	1795	ev.yield(.{ .activate = source.as_object() });
	1796	const limit = source.get_data();
	1797	source.as_object().release();
	1798	while (true) return fileReadStreamingLimit(
	1799	file.handle,
	1800	data,
	1801	.limited(limit),
	1802	) catch \|err\| switch (err) {
	1803	error.WouldBlock => {
	1804	ev.yield(.nothing);
	1805	continue;
	1806	},
	1807	else => \|e\| return e,
	1808	};
	1809	}
	1810	fn fileReadStreamingLimit(
	1811	handle: File.Handle,
	1812	data: []const []u8,
	1813	limit: Io.Limit,
	1814	) File.ReadStreamingError!usize {
	1815	var iovecs: [max_iovecs_len]iovec = undefined;
	1816	var iovlen: iovlen_t = 0;
	1817	// .nothing can mean that the write side has been closed,
	1818	// in which case the buffer still needs to be drained
	1819	var remaining = if (limit == .nothing) .unlimited else limit;
	1820	for (data) \|buf\| addBuf(false, &iovecs, &iovlen, &remaining, buf);
	1821	if (iovlen == 0) return 0;
	1822	while (true) {
	1823	const rc = c.readv(handle, &iovecs, iovlen);
	1824	switch (c.errno(rc)) {
	1825	.SUCCESS => return if (rc == 0) error.EndOfStream else @intCast(rc),
	1826	.INTR => continue,
	1827	.INVAL => \|err\| return errnoBug(err),
	1828	.FAULT => \|err\| return errnoBug(err),
	1829	.AGAIN => return error.WouldBlock,
	1830	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed
	1831	.IO => return error.InputOutput,
	1832	.ISDIR => return error.IsDir,
	1833	.NOBUFS => return error.SystemResources,
	1834	.NOMEM => return error.SystemResources,
	1835	.NOTCONN => return error.SocketUnconnected,
	1836	.CONNRESET => return error.ConnectionResetByPeer,
	1837	else => \|err\| return unexpectedErrno(err),
	1838	}
	1839	}
	1840	}
	1841
	1842	fn fileWriteStreaming(
	1843	ev: *Evented,
	1844	file: File,
	1845	header: []const u8,
	1846	data: []const []const u8,
	1847	splat: usize,
	1848	) File.Writer.Error!usize {
	1849	if (file.flags.nonblocking) nonblocking: {
	1850	return fileWriteStreamingLimit(
	1851	file.handle,
	1852	header,
	1853	data,
	1854	splat,
	1855	.unlimited,
	1856	) catch \|err\| switch (err) {
	1857	error.WouldBlock => break :nonblocking,
	1858	else => \|e\| return e,
	1859	};
	1860	}
	1861	const source = c.dispatch.source_create(
	1862	.WRITE,
	1863	@bitCast(@as(isize, file.handle)),
	1864	.none,
	1865	ev.queue,
	1866	) orelse return error.SystemResources;
	1867	source.as_object().set_context(Thread.current().currentFiber());
	1868	source.set_event_handler(&Fiber.@"resume");
	1869	ev.yield(.{ .activate = source.as_object() });
	1870	const limit = source.get_data();
	1871	source.as_object().release();
	1872	while (true) return fileWriteStreamingLimit(
	1873	file.handle,
	1874	header,
	1875	data,
	1876	splat,
	1877	.limited(limit),
	1878	) catch \|err\| switch (err) {
	1879	error.WouldBlock => {
	1880	ev.yield(.nothing);
	1881	continue;
	1882	},
	1883	else => \|e\| return e,
	1884	};
	1885	}
	1886	fn fileWriteStreamingLimit(
	1887	handle: File.Handle,
	1888	header: []const u8,
	1889	data: []const []const u8,
	1890	splat: usize,
	1891	limit: Io.Limit,
	1892	) File.Writer.Error!usize {
	1893	if (limit == .nothing) return 0;
	1894	var iovecs: [max_iovecs_len]iovec_const = undefined;
	1895	var iovlen: iovlen_t = 0;
	1896	var remaining = limit;
	1897	addBuf(true, &iovecs, &iovlen, &remaining, header);
	1898	for (data[0 .. data.len - 1]) \|bytes\| addBuf(true, &iovecs, &iovlen, &remaining, bytes);
	1899	const pattern = data[data.len - 1];
	1900	var backup_buffer: [splat_buffer_size]u8 = undefined;
	1901	if (iovecs.len - iovlen != 0 and remaining != .nothing) switch (splat) {
	1902	0 => {},
	1903	1 => addBuf(true, &iovecs, &iovlen, &remaining, pattern),
	1904	else => switch (pattern.len) {
	1905	0 => {},
	1906	1 => {
	1907	const splat_buffer = &backup_buffer;
	1908	const memset_len = @min(splat_buffer.len, splat);
	1909	const buf = splat_buffer[0..memset_len];
	1910	@memset(buf, pattern[0]);
	1911	addBuf(true, &iovecs, &iovlen, &remaining, buf);
	1912	var remaining_splat = splat - buf.len;
	1913	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0 and remaining != .nothing) {
	1914	assert(buf.len == splat_buffer.len);
	1915	addBuf(true, &iovecs, &iovlen, &remaining, splat_buffer);
	1916	remaining_splat -= splat_buffer.len;
	1917	}
	1918	addBuf(true, &iovecs, &iovlen, &remaining, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
	1919	},
	1920	else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
	1921	if (remaining == .nothing) break;
	1922	addBuf(true, &iovecs, &iovlen, &remaining, pattern);
	1923	},
	1924	},
	1925	};
	1926	if (iovlen == 0) return 0;
	1927	while (true) {
	1928	const rc = c.writev(handle, &iovecs, iovlen);
	1929	switch (c.errno(rc)) {
	1930	.SUCCESS => return @intCast(rc),
	1931	.INTR => continue,
	1932	.INVAL => \|err\| return errnoBug(err),
	1933	.FAULT => \|err\| return errnoBug(err),
	1934	.AGAIN => return error.WouldBlock,
	1935	.BADF => return error.NotOpenForWriting, // Can be a race condition.
	1936	.DESTADDRREQ => \|err\| return errnoBug(err), // `connect` was never called.
	1937	.DQUOT => return error.DiskQuota,
	1938	.FBIG => return error.FileTooBig,
	1939	.IO => return error.InputOutput,
	1940	.NOSPC => return error.NoSpaceLeft,
	1941	.PERM => return error.PermissionDenied,
	1942	.PIPE => return error.BrokenPipe,
	1943	.CONNRESET => \|err\| return errnoBug(err), // Not a socket handle.
	1944	.BUSY => return error.DeviceBusy,
	1945	else => \|err\| return unexpectedErrno(err),
	1946	}
	1947	}
	1948	}
	1949
	1950	fn deviceIoControl(o: *const Io.Operation.DeviceIoControl) Io.Cancelable!i32 {
	1951	while (true) {
	1952	const rc = c.ioctl(o.file.handle, @bitCast(o.code), @intFromPtr(o.arg));
	1953	switch (c.errno(rc)) {
	1954	.SUCCESS => return rc,
	1955	.INTR => {},
	1956	else => \|err\| return -@as(i32, @intFromEnum(err)),
	1957	}
	1958	}
	1959	}
	1960
	1961	const BatchWaiter = struct {
	1962	sleeper: Sleeper,
	1963	queue: c.dispatch.queue_t,
	1964	timer: ?c.dispatch.source_t = null,
	1965
	1966	const already_signaled: c.dispatch.source_t = @ptrFromInt(1);
	1967
	1968	fn signal(context: ?*anyopaque) callconv(.c) void {
	1969	const waiter: *BatchWaiter = @ptrCast(@alignCast(context));
	1970	if (waiter.timer) \|timer\| {
	1971	if (timer != already_signaled) timer.cancel();
	1972	} else {
	1973	waiter.timer = already_signaled;
	1974	waiter.queue.async(waiter, &@"suspend");
	1975	}
	1976	}
	1977
	1978	fn @"suspend"(context: ?*anyopaque) callconv(.c) void {
	1979	const waiter: *BatchWaiter = @ptrCast(@alignCast(context));
	1980	if (waiter.timer) \|timer\| if (timer != already_signaled) timer.as_object().release();
	1981	waiter.queue.as_object().@"suspend"();
	1982	waiter.wake();
	1983	}
	1984
	1985	fn wake(waiter: *BatchWaiter) void {
	1986	var sleeper = waiter.sleeper;
	1987	waiter.* = undefined;
	1988	Sleeper.wake(&sleeper);
	1989	}
	1990	};
	1991
	1992	fn batchAwaitAsync(userdata: ?anyopaque, batch: Io.Batch) Io.Cancelable!void {
	1993	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1994	const queue = ev.batchDrainSubmitted(batch, false) catch \|err\| switch (err) {
	1995	error.ConcurrencyUnavailable => unreachable, // passed concurrency=false
	1996	error.Canceled => \|e\| return e,
	1997	} orelse return;
	1998	if (batch.pending.head == .none) return;
	1999	var waiter: BatchWaiter = .{
	2000	.sleeper = .init(ev.queue, Thread.current().currentFiber()),
	2001	.queue = queue,
	2002	};
	2003	if (batch.completed.head != .none) BatchWaiter.signal(&waiter);
	2004	queue.as_object().set_context(&waiter);
	2005	ev.yield(.{ .@"resume" = queue.as_object() });
	2006	}
	2007
	2008	fn batchAwaitConcurrent(
	2009	userdata: ?*anyopaque,
	2010	batch: *Io.Batch,
	2011	timeout: Io.Timeout,
	2012	) Io.Batch.AwaitConcurrentError!void {
	2013	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2014	const queue = try ev.batchDrainSubmitted(batch, true) orelse return;
	2015	if (batch.pending.head == .none) return;
	2016	var waiter: BatchWaiter = .{
	2017	.sleeper = .init(ev.queue, Thread.current().currentFiber()),
	2018	.queue = queue,
	2019	};
	2020	if (batch.completed.head == .none) switch (timeout) {
	2021	.none => {},
	2022	else => {
	2023	const timer = c.dispatch.source_create(.TIMER, 0, .none, queue) orelse
	2024	return error.ConcurrencyUnavailable;
	2025	assert(timer != BatchWaiter.already_signaled);
	2026	timer.as_object().set_context(&waiter);
	2027	timer.set_event_handler(&BatchWaiter.signal);
	2028	timer.set_cancel_handler(&BatchWaiter.@"suspend");
	2029	timer.set_timer(ev.timeFromTimeout(timeout), c.dispatch.TIME_FOREVER, ev.leeway);
	2030	timer.as_object().activate();
	2031	waiter.timer = timer;
	2032	},
	2033	} else BatchWaiter.signal(&waiter);
	2034	queue.as_object().set_context(&waiter);
	2035	ev.yield(.{ .@"resume" = queue.as_object() });
	2036	}
	2037
	2038	fn batchCancel(userdata: ?anyopaque, batch: Io.Batch) void {
	2039	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2040	var index = batch.pending.head;
	2041	while (index != .none) {
	2042	const storage = &batch.storage[index.toIndex()];
	2043	const pending = &storage.pending;
	2044	const operation_userdata: *BatchOperationUserdata = .fromErased(&pending.userdata);
	2045	assert(operation_userdata.batch == batch);
	2046	operation_userdata.source.cancel();
	2047	}
	2048	const queue: c.dispatch.queue_t = @ptrCast(batch.userdata orelse return);
	2049	if (batch.pending.head != .none) {
	2050	var waiter: BatchWaiter = .{
	2051	.sleeper = .init(ev.queue, Thread.current().currentFiber()),
	2052	.queue = queue,
	2053	.timer = BatchWaiter.already_signaled,
	2054	};
	2055	if (batch.pending.head == .none) queue.async(&waiter, &BatchWaiter.signal);
	2056	queue.as_object().set_context(&waiter);
	2057	ev.yield(.{ .@"resume" = queue.as_object() });
	2058	}
	2059	batch.userdata = null;
	2060	}
	2061
	2062	const BatchOperationUserdata = extern struct {
	2063	batch: *Io.Batch,
	2064	source: c.dispatch.source_t,
	2065	operation: extern union {
	2066	file_read_streaming: extern struct {
	2067	data_ptr: [*]const []u8,
	2068	data_len: usize,
	2069	},
	2070	file_write_streaming: extern struct {
	2071	header_ptr: [*]const u8,
	2072	header_len: usize,
	2073	data_ptr: [*]const []const u8,
	2074	data_len: usize,
	2075	splat: usize,
	2076
	2077	fn header(operation: *const @This()) []const u8 {
	2078	return operation.header_ptr[0..operation.header_len];
	2079	}
	2080
	2081	fn data(operation: *const @This()) []const []const u8 {
	2082	return operation.data_ptr[0..operation.data_len];
	2083	}
	2084	},
	2085	},
	2086
	2087	const Erased = Io.Operation.Storage.Pending.Userdata;
	2088
	2089	comptime {
	2090	assert(@sizeOf(BatchOperationUserdata) <= @sizeOf(Erased));
	2091	}
	2092
	2093	fn toErased(userdata: BatchOperationUserdata) Erased {
	2094	return @ptrCast(userdata);
	2095	}
	2096
	2097	fn fromErased(erased: Erased) BatchOperationUserdata {
	2098	return @ptrCast(erased);
	2099	}
	2100	};
	2101
	2102	/// If `concurrency` is false, `error.ConcurrencyUnavailable` is unreachable.
	2103	fn batchDrainSubmitted(
	2104	ev: *Evented,
	2105	batch: *Io.Batch,
	2106	concurrency: bool,
	2107	) (Io.ConcurrentError \|\| Io.Cancelable)!?c.dispatch.queue_t {
	2108	var index = batch.submitted.head;
	2109	if (index == .none) return @ptrCast(batch.userdata);
	2110	errdefer batch.submitted.head = index;
	2111	const maybe_queue: ?c.dispatch.queue_t = if (batch.userdata) \|batch_userdata\|
	2112	@ptrCast(batch_userdata)
	2113	else maybe_queue: {
	2114	const queue = c.dispatch.queue_create_with_target(
	2115	"org.ziglang.std.Io.Dispatch.Batch",
	2116	.SERIAL(),
	2117	ev.queue,
	2118	) orelse if (concurrency) return error.ConcurrencyUnavailable else break :maybe_queue null;
	2119	queue.as_object().@"suspend"();
	2120	batch.userdata = queue;
	2121	break :maybe_queue queue;
	2122	};
	2123	while (index != .none) {
	2124	const storage = &batch.storage[index.toIndex()];
	2125	const next_index = storage.submission.node.next;
	2126	if (@as(?Io.Operation.Result, result: {
	2127	if (maybe_queue) \|queue\| switch (storage.submission.operation) {
	2128	.file_read_streaming => \|operation\| {
	2129	const data = for (operation.data, 0..) \|buffer, data_index\| {
	2130	if (buffer.len > 0) break operation.data[data_index..];
	2131	} else break :result .{ .file_read_streaming = 0 };
	2132	const source = c.dispatch.source_create(
	2133	.READ,
	2134	@bitCast(@as(isize, operation.file.handle)),
	2135	.none,
	2136	queue,
	2137	) orelse break :result .{ .file_read_streaming = error.SystemResources };
	2138	storage.* = .{ .pending = .{
	2139	.node = .{ .prev = batch.pending.tail, .next = .none },
	2140	.tag = .file_read_streaming,
	2141	.userdata = undefined,
	2142	} };
	2143	const operation_userdata: *BatchOperationUserdata =
	2144	.fromErased(&storage.pending.userdata);
	2145	operation_userdata.* = .{
	2146	.batch = batch,
	2147	.source = source,
	2148	.operation = .{ .file_read_streaming = .{
	2149	.data_ptr = data.ptr,
	2150	.data_len = data.len,
	2151	} },
	2152	};
	2153	source.as_object().set_context(storage);
	2154	source.set_event_handler(&batchSourceEvent);
	2155	source.set_cancel_handler(&batchSourceCancel);
	2156	source.as_object().activate();
	2157	break :result null;
	2158	},
	2159	.file_write_streaming => \|operation\| {
	2160	const data = for (operation.data, 0..) \|buffer, data_index\| {
	2161	if (buffer.len > 0) break operation.data[data_index..];
	2162	} else if (operation.header.len > 0)
	2163	operation.data[0..1]
	2164	else
	2165	break :result .{ .file_write_streaming = 0 };
	2166	const source = c.dispatch.source_create(
	2167	.WRITE,
	2168	@bitCast(@as(isize, operation.file.handle)),
	2169	.none,
	2170	queue,
	2171	) orelse break :result .{ .file_write_streaming = error.SystemResources };
	2172	storage.* = .{ .pending = .{
	2173	.node = .{ .prev = batch.pending.tail, .next = .none },
	2174	.tag = .file_write_streaming,
	2175	.userdata = undefined,
	2176	} };
	2177	const operation_userdata: *BatchOperationUserdata =
	2178	.fromErased(&storage.pending.userdata);
	2179	operation_userdata.* = .{
	2180	.batch = batch,
	2181	.source = source,
	2182	.operation = .{ .file_write_streaming = .{
	2183	.header_ptr = operation.header.ptr,
	2184	.header_len = operation.header.len,
	2185	.data_ptr = data.ptr,
	2186	.data_len = data.len,
	2187	.splat = operation.splat,
	2188	} },
	2189	};
	2190	source.as_object().set_context(storage);
	2191	source.set_event_handler(&batchSourceEvent);
	2192	source.set_cancel_handler(&batchSourceCancel);
	2193	source.as_object().activate();
	2194	break :result null;
	2195	},
	2196	.device_io_control => {},
	2197	};
	2198	if (concurrency) return error.ConcurrencyUnavailable;
	2199	break :result try operate(ev, storage.submission.operation);
	2200	})) \|result\| {
	2201	switch (batch.completed.tail) {
	2202	.none => batch.completed.head = index,
	2203	else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next = index,
	2204	}
	2205	batch.completed.tail = index;
	2206	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
	2207	} else {
	2208	switch (batch.pending.tail) {
	2209	.none => batch.pending.head = index,
	2210	else => \|tail_index\| batch.storage[tail_index.toIndex()].pending.node.next = index,
	2211	}
	2212	batch.pending.tail = index;
	2213	}
	2214	index = next_index;
	2215	}
	2216	batch.submitted = .{ .head = .none, .tail = .none };
	2217	return maybe_queue;
	2218	}
	2219
	2220	fn batchSourceEvent(context: ?*anyopaque) callconv(.c) void {
	2221	const storage: *Io.Operation.Storage = @ptrCast(@alignCast(context));
	2222	const pending = &storage.pending;
	2223	const operation_userdata: *BatchOperationUserdata = .fromErased(&pending.userdata);
	2224	const batch = operation_userdata.batch;
	2225	const source = operation_userdata.source;
	2226	const index: Io.Operation.OptionalIndex = .fromIndex(storage - batch.storage.ptr);
	2227	const result: Io.Operation.Result = result: switch (pending.tag) {
	2228	.file_read_streaming => {
	2229	const operation = &operation_userdata.operation.file_read_streaming;
	2230	break :result .{ .file_read_streaming = fileReadStreamingLimit(
	2231	@intCast(source.get_handle()),
	2232	operation.data_ptr[0..operation.data_len],
	2233	.limited(source.get_data()),
	2234	) catch \|err\| switch (err) {
	2235	error.Canceled => return Thread.current().currentFiber().cancel_protection.recancel(),
	2236	error.WouldBlock => return,
	2237	else => \|e\| e,
	2238	} };
	2239	},
	2240	.file_write_streaming => {
	2241	const operation = &operation_userdata.operation.file_write_streaming;
	2242	break :result .{ .file_write_streaming = fileWriteStreamingLimit(
	2243	@intCast(source.get_handle()),
	2244	operation.header_ptr[0..operation.header_len],
	2245	operation.data_ptr[0..operation.data_len],
	2246	operation.splat,
	2247	.limited(source.get_data()),
	2248	) catch \|err\| switch (err) {
	2249	error.Canceled => return Thread.current().currentFiber().cancel_protection.recancel(),
	2250	error.WouldBlock => return,
	2251	else => \|e\| e,
	2252	} };
	2253	},
	2254	.device_io_control => unreachable,
	2255	};
	2256
	2257	switch (pending.node.prev) {
	2258	.none => batch.pending.head = pending.node.next,
	2259	else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.next = pending.node.next,
	2260	}
	2261	switch (pending.node.next) {
	2262	.none => batch.pending.tail = pending.node.prev,
	2263	else => \|next_index\| batch.storage[next_index.toIndex()].pending.node.prev = pending.node.prev,
	2264	}
	2265
	2266	switch (batch.completed.tail) {
	2267	.none => batch.completed.head = index,
	2268	else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next = index,
	2269	}
	2270	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
	2271	batch.completed.tail = index;
	2272
	2273	source.as_object().release();
	2274	const queue: c.dispatch.queue_t = @ptrCast(batch.userdata);
	2275	const waiter: *BatchWaiter = @ptrCast(@alignCast(queue.as_object().get_context()));
	2276	BatchWaiter.signal(waiter);
	2277	}
	2278
	2279	fn batchSourceCancel(context: ?*anyopaque) callconv(.c) void {
	2280	const storage: *Io.Operation.Storage = @ptrCast(@alignCast(context));
	2281	const pending = &storage.pending;
	2282	const operation_userdata: *BatchOperationUserdata = .fromErased(&pending.userdata);
	2283	const batch = operation_userdata.batch;
	2284	const source = operation_userdata.source;
	2285	const index: Io.Operation.OptionalIndex = .fromIndex(storage - batch.storage.ptr);
	2286
	2287	switch (pending.node.prev) {
	2288	.none => batch.pending.head = pending.node.next,
	2289	else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.next = pending.node.next,
	2290	}
	2291	switch (pending.node.next) {
	2292	.none => batch.pending.tail = pending.node.prev,
	2293	else => \|next_index\| batch.storage[next_index.toIndex()].pending.node.prev = pending.node.prev,
	2294	}
	2295
	2296	const tail_index = batch.unused.tail;
	2297	switch (tail_index) {
	2298	.none => batch.unused.head = index,
	2299	else => batch.storage[tail_index.toIndex()].unused.next = index,
	2300	}
	2301	storage.* = .{ .unused = .{ .prev = tail_index, .next = .none } };
	2302	batch.unused.tail = index;
	2303
	2304	source.as_object().release();
	2305	if (batch.pending.head != .none) return;
	2306	const queue: c.dispatch.queue_t = @ptrCast(batch.userdata);
	2307	const waiter: *BatchWaiter = @ptrCast(@alignCast(queue.as_object().get_context()));
	2308	queue.as_object().release();
	2309	waiter.wake();
	2310	}
	2311
	2312	fn dirCreateDir(
	2313	userdata: ?*anyopaque,
	2314	dir: Dir,
	2315	sub_path: []const u8,
	2316	permissions: Dir.Permissions,
	2317	) Dir.CreateDirError!void {
	2318	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2319	_ = ev;
	2320
	2321	var path_buffer: [c.PATH_MAX]u8 = undefined;
	2322	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2323
	2324	while (true) {
	2325	switch (c.errno(c.mkdirat(dir.handle, sub_path_posix, permissions.toMode()))) {
	2326	.SUCCESS => return,
	2327	.INTR => {},
	2328	.ACCES => return error.AccessDenied,
	2329	.PERM => return error.PermissionDenied,
	2330	.DQUOT => return error.DiskQuota,
	2331	.EXIST => return error.PathAlreadyExists,
	2332	.LOOP => return error.SymLinkLoop,
	2333	.MLINK => return error.LinkQuotaExceeded,
	2334	.NAMETOOLONG => return error.NameTooLong,
	2335	.NOENT => return error.FileNotFound,
	2336	.NOMEM => return error.SystemResources,
	2337	.NOSPC => return error.NoSpaceLeft,
	2338	.NOTDIR => return error.NotDir,
	2339	.ROFS => return error.ReadOnlyFileSystem,
	2340	.ILSEQ => return error.BadPathName,
	2341	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	2342	.FAULT => \|err\| return errnoBug(err),
	2343	else => \|err\| return unexpectedErrno(err),
	2344	}
	2345	}
	2346	}
	2347
	2348	fn dirCreateDirPath(
	2349	userdata: ?*anyopaque,
	2350	dir: Dir,
	2351	sub_path: []const u8,
	2352	permissions: Dir.Permissions,
	2353	) Dir.CreateDirPathError!Dir.CreatePathStatus {
	2354	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2355
	2356	var it = Dir.path.componentIterator(sub_path);
	2357	var status: Dir.CreatePathStatus = .existed;
	2358	var component = it.last() orelse return error.BadPathName;
	2359	while (true) {
	2360	if (dirCreateDir(ev, dir, component.path, permissions)) \|_\| {
	2361	status = .created;
	2362	} else \|err\| switch (err) {
	2363	error.PathAlreadyExists => {
	2364	// It is important to return an error if it's not a directory
	2365	// because otherwise a dangling symlink could cause an infinite
	2366	// loop.
	2367	const fstat = try dirStatFile(ev, dir, component.path, .{});
	2368	if (fstat.kind != .directory) return error.NotDir;
	2369	},
	2370	error.FileNotFound => \|e\| {
	2371	component = it.previous() orelse return e;
	2372	continue;
	2373	},
	2374	else => \|e\| return e,
	2375	}
	2376	component = it.next() orelse return status;
	2377	}
	2378	}
	2379
	2380	fn dirCreateDirPathOpen(
	2381	userdata: ?*anyopaque,
	2382	dir: Dir,
	2383	sub_path: []const u8,
	2384	permissions: Dir.Permissions,
	2385	options: Dir.OpenOptions,
	2386	) Dir.CreateDirPathOpenError!Dir {
	2387	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2388	return dirOpenDir(ev, dir, sub_path, options) catch \|err\| switch (err) {
	2389	error.FileNotFound => {
	2390	_ = try dirCreateDirPath(ev, dir, sub_path, permissions);
	2391	return dirOpenDir(ev, dir, sub_path, options);
	2392	},
	2393	else => \|e\| return e,
	2394	};
	2395	}
	2396
	2397	fn dirOpenDir(
	2398	userdata: ?*anyopaque,
	2399	dir: Dir,
	2400	sub_path: []const u8,
	2401	options: Dir.OpenOptions,
	2402	) Dir.OpenError!Dir {
	2403	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2404	_ = ev;
	2405
	2406	var path_buffer: [c.PATH_MAX]u8 = undefined;
	2407	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2408
	2409	const flags: c.O = .{
	2410	.ACCMODE = .RDONLY,
	2411	.NOFOLLOW = !options.follow_symlinks,
	2412	.DIRECTORY = true,
	2413	.CLOEXEC = true,
	2414	};
	2415
	2416	while (true) {
	2417	const rc = c.openat(dir.handle, sub_path_posix, flags);
	2418	switch (c.errno(rc)) {
	2419	.SUCCESS => return .{ .handle = @intCast(rc) },
	2420	.INTR => {},
	2421	.INVAL => return error.BadPathName,
	2422	.ACCES => return error.AccessDenied,
	2423	.LOOP => return error.SymLinkLoop,
	2424	.MFILE => return error.ProcessFdQuotaExceeded,
	2425	.NAMETOOLONG => return error.NameTooLong,
	2426	.NFILE => return error.SystemFdQuotaExceeded,
	2427	.NODEV => return error.NoDevice,
	2428	.NOENT => return error.FileNotFound,
	2429	.NOMEM => return error.SystemResources,
	2430	.NOTDIR => return error.NotDir,
	2431	.PERM => return error.PermissionDenied,
	2432	.NXIO => return error.NoDevice,
	2433	.ILSEQ => return error.BadPathName,
	2434	.FAULT => \|err\| return errnoBug(err),
	2435	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	2436	.BUSY => \|err\| return errnoBug(err), // O_EXCL not passed
	2437	else => \|err\| return unexpectedErrno(err),
	2438	}
	2439	}
	2440	}
	2441
	2442	fn dirStat(userdata: ?*anyopaque, dir: Dir) Dir.StatError!Dir.Stat {
	2443	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2444	return fileStat(ev, .{
	2445	.handle = dir.handle,
	2446	.flags = .{ .nonblocking = false },
	2447	});
	2448	}
	2449
	2450	fn dirStatFile(
	2451	userdata: ?*anyopaque,
	2452	dir: Dir,
	2453	sub_path: []const u8,
	2454	options: Dir.StatFileOptions,
	2455	) Dir.StatFileError!File.Stat {
	2456	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2457	_ = ev;
	2458
	2459	var path_buffer: [c.PATH_MAX]u8 = undefined;
	2460	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2461
	2462	const flags: u32 = if (options.follow_symlinks) 0 else c.AT.SYMLINK_NOFOLLOW;
	2463
	2464	while (true) {
	2465	var stat = std.mem.zeroes(c.Stat);
	2466	switch (c.errno(c.fstatat(dir.handle, sub_path_posix, &stat, flags))) {
	2467	.SUCCESS => return statFromPosix(&stat),
	2468	.INTR => {},
	2469	.INVAL => \|err\| return errnoBug(err),
	2470	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	2471	.NOMEM => return error.SystemResources,
	2472	.ACCES => return error.AccessDenied,
	2473	.PERM => return error.PermissionDenied,
	2474	.FAULT => \|err\| return errnoBug(err),
	2475	.NAMETOOLONG => return error.NameTooLong,
	2476	.LOOP => return error.SymLinkLoop,
	2477	.NOENT => return error.FileNotFound,
	2478	.NOTDIR => return error.FileNotFound,
	2479	.ILSEQ => return error.BadPathName,
	2480	else => \|err\| return unexpectedErrno(err),
	2481	}
	2482	}
	2483	}
	2484
	2485	fn dirAccess(
	2486	userdata: ?*anyopaque,
	2487	dir: Dir,
	2488	sub_path: []const u8,
	2489	options: Dir.AccessOptions,
	2490	) Dir.AccessError!void {
	2491	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2492	_ = ev;
	2493
	2494	var path_buffer: [c.PATH_MAX]u8 = undefined;
	2495	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2496
	2497	const flags: u32 = if (options.follow_symlinks) 0 else c.AT.SYMLINK_NOFOLLOW;
	2498
	2499	const mode: u32 =
	2500	@as(u32, if (options.read) c.R_OK else 0) \|
	2501	@as(u32, if (options.write) c.W_OK else 0) \|
	2502	@as(u32, if (options.execute) c.X_OK else 0);
	2503
	2504	while (true) switch (c.errno(c.faccessat(dir.handle, sub_path_posix, mode, flags))) {
	2505	.SUCCESS => return,
	2506	.INTR => {},
	2507	.ACCES => return error.AccessDenied,
	2508	.PERM => return error.PermissionDenied,
	2509	.ROFS => return error.ReadOnlyFileSystem,
	2510	.LOOP => return error.SymLinkLoop,
	2511	.TXTBSY => return error.FileBusy,
	2512	.NOTDIR => return error.FileNotFound,
	2513	.NOENT => return error.FileNotFound,
	2514	.NAMETOOLONG => return error.NameTooLong,
	2515	.INVAL => \|err\| return errnoBug(err),
	2516	.FAULT => \|err\| return errnoBug(err),
	2517	.IO => return error.InputOutput,
	2518	.NOMEM => return error.SystemResources,
	2519	.ILSEQ => return error.BadPathName,
	2520	else => \|err\| return unexpectedErrno(err),
	2521	};
	2522	}
	2523
	2524	fn dirCreateFile(
	2525	userdata: ?*anyopaque,
	2526	dir: Dir,
	2527	sub_path: []const u8,
	2528	flags: File.CreateFlags,
	2529	) File.OpenError!File {
	2530	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2531	_ = ev;
	2532
	2533	var path_buffer: [c.PATH_MAX]u8 = undefined;
	2534	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2535
	2536	const os_flags: c.O = .{
	2537	.ACCMODE = if (flags.read) .RDWR else .WRONLY,
	2538	.NONBLOCK = flags.lock == .none or flags.lock_nonblocking,
	2539	.SHLOCK = flags.lock == .shared,
	2540	.EXLOCK = flags.lock == .exclusive,
	2541	.CREAT = true,
	2542	.TRUNC = flags.truncate,
	2543	.EXCL = flags.exclusive,
	2544	.CLOEXEC = true,
	2545	};
	2546
	2547	const fd: c.fd_t = while (true) {
	2548	const rc = c.openat(dir.handle, sub_path_posix, os_flags, flags.permissions.toMode());
	2549	switch (c.errno(rc)) {
	2550	.SUCCESS => break @intCast(rc),
	2551	.INTR => {},
	2552	.FAULT => \|err\| return errnoBug(err),
	2553	.INVAL => return error.BadPathName,
	2554	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	2555	.ACCES => return error.AccessDenied,
	2556	.FBIG => return error.FileTooBig,
	2557	.OVERFLOW => return error.FileTooBig,
	2558	.ISDIR => return error.IsDir,
	2559	.LOOP => return error.SymLinkLoop,
	2560	.MFILE => return error.ProcessFdQuotaExceeded,
	2561	.NAMETOOLONG => return error.NameTooLong,
	2562	.NFILE => return error.SystemFdQuotaExceeded,
	2563	.NODEV => return error.NoDevice,
	2564	.NOENT => return error.FileNotFound,
	2565	.NOMEM => return error.SystemResources,
	2566	.NOSPC => return error.NoSpaceLeft,
	2567	.NOTDIR => return error.NotDir,
	2568	.PERM => return error.PermissionDenied,
	2569	.EXIST => return error.PathAlreadyExists,
	2570	.BUSY => return error.DeviceBusy,
	2571	.OPNOTSUPP => return error.FileLocksUnsupported,
	2572	.AGAIN => return error.WouldBlock,
	2573	.TXTBSY => return error.FileBusy,
	2574	.NXIO => return error.NoDevice,
	2575	.ILSEQ => return error.BadPathName,
	2576	else => \|err\| return unexpectedErrno(err),
	2577	}
	2578	};
	2579	errdefer closeFd(fd);
	2580
	2581	return .{
	2582	.handle = fd,
	2583	.flags = .{ .nonblocking = os_flags.NONBLOCK },
	2584	};
	2585	}
	2586
	2587	fn dirCreateFileAtomic(
	2588	userdata: ?*anyopaque,
	2589	dir: Dir,
	2590	dest_path: []const u8,
	2591	options: Dir.CreateFileAtomicOptions,
	2592	) Dir.CreateFileAtomicError!File.Atomic {
	2593	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2594	if (Dir.path.dirname(dest_path)) \|dirname\| {
	2595	const new_dir = if (options.make_path)
	2596	dirCreateDirPathOpen(ev, dir, dirname, .default_dir, .{}) catch \|err\| switch (err) {
	2597	// None of these make sense in this context.
	2598	error.IsDir,
	2599	error.Streaming,
	2600	error.DiskQuota,
	2601	error.PathAlreadyExists,
	2602	error.LinkQuotaExceeded,
	2603	error.PipeBusy,
	2604	error.FileTooBig,
	2605	error.FileLocksUnsupported,
	2606	error.DeviceBusy,
	2607	=> return error.Unexpected,
	2608
	2609	else => \|e\| return e,
	2610	}
	2611	else
	2612	try dirOpenDir(ev, dir, dirname, .{});
	2613	return ev.atomicFileInit(Dir.path.basename(dest_path), options.permissions, new_dir, true);
	2614	}
	2615	return ev.atomicFileInit(dest_path, options.permissions, dir, false);
	2616	}
	2617
	2618	fn atomicFileInit(
	2619	ev: *Evented,
	2620	dest_basename: []const u8,
	2621	permissions: File.Permissions,
	2622	dir: Dir,
	2623	close_dir_on_deinit: bool,
	2624	) Dir.CreateFileAtomicError!File.Atomic {
	2625	while (true) {
	2626	var random_integer: u64 = undefined;
	2627	random(ev, @ptrCast(&random_integer));
	2628	const tmp_sub_path = std.fmt.hex(random_integer);
	2629	const file = dirCreateFile(ev, dir, &tmp_sub_path, .{
	2630	.permissions = permissions,
	2631	.exclusive = true,
	2632	}) catch \|err\| switch (err) {
	2633	error.PathAlreadyExists => continue,
	2634	error.DeviceBusy => continue,
	2635	error.FileBusy => continue,
	2636
	2637	error.IsDir => return error.Unexpected, // No path components.
	2638	error.FileTooBig => return error.Unexpected, // Creating, not opening.
	2639	error.FileLocksUnsupported => return error.Unexpected, // Not asking for locks.
	2640	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
	2641
	2642	else => \|e\| return e,
	2643	};
	2644	return .{
	2645	.file = file,
	2646	.file_basename_hex = random_integer,
	2647	.dest_sub_path = dest_basename,
	2648	.file_open = true,
	2649	.file_exists = true,
	2650	.close_dir_on_deinit = close_dir_on_deinit,
	2651	.dir = dir,
	2652	};
	2653	}
	2654	}
	2655
	2656	fn dirOpenFile(
	2657	userdata: ?*anyopaque,
	2658	dir: Dir,
	2659	sub_path: []const u8,
	2660	flags: File.OpenFlags,
	2661	) File.OpenError!File {
	2662	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2663
	2664	var path_buffer: [c.PATH_MAX]u8 = undefined;
	2665	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2666
	2667	const os_flags: c.O = .{
	2668	.ACCMODE = switch (flags.mode) {
	2669	.read_only => .RDONLY,
	2670	.write_only => .WRONLY,
	2671	.read_write => .RDWR,
	2672	},
	2673	.NONBLOCK = flags.lock == .none or flags.lock_nonblocking,
	2674	.SHLOCK = flags.lock == .shared,
	2675	.EXLOCK = flags.lock == .exclusive,
	2676	.NOFOLLOW = !flags.follow_symlinks,
	2677	.NOCTTY = !flags.allow_ctty,
	2678	.CLOEXEC = true,
	2679	};
	2680
	2681	const fd: c.fd_t = while (true) {
	2682	const rc = c.openat(dir.handle, sub_path_posix, os_flags);
	2683	switch (c.errno(rc)) {
	2684	.SUCCESS => break @intCast(rc),
	2685	.INTR => {},
	2686	.FAULT => \|err\| return errnoBug(err),
	2687	.INVAL => return error.BadPathName,
	2688	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	2689	.ACCES => return error.AccessDenied,
	2690	.FBIG => return error.FileTooBig,
	2691	.OVERFLOW => return error.FileTooBig,
	2692	.ISDIR => return error.IsDir,
	2693	.LOOP => return error.SymLinkLoop,
	2694	.MFILE => return error.ProcessFdQuotaExceeded,
	2695	.NAMETOOLONG => return error.NameTooLong,
	2696	.NFILE => return error.SystemFdQuotaExceeded,
	2697	.NODEV => return error.NoDevice,
	2698	.NOENT => return error.FileNotFound,
	2699	.NOMEM => return error.SystemResources,
	2700	.NOSPC => return error.NoSpaceLeft,
	2701	.NOTDIR => return error.NotDir,
	2702	.PERM => return error.PermissionDenied,
	2703	.EXIST => return error.PathAlreadyExists,
	2704	.BUSY => return error.DeviceBusy,
	2705	.OPNOTSUPP => return error.FileLocksUnsupported,
	2706	.AGAIN => return error.WouldBlock,
	2707	.TXTBSY => return error.FileBusy,
	2708	.NXIO => return error.NoDevice,
	2709	.ILSEQ => return error.BadPathName,
	2710	else => \|err\| return unexpectedErrno(err),
	2711	}
	2712	};
	2713	errdefer closeFd(fd);
	2714
	2715	if (!flags.allow_directory) {
	2716	const is_dir = is_dir: {
	2717	const stat = fileStat(ev, .{
	2718	.handle = fd,
	2719	.flags = .{ .nonblocking = false },
	2720	}) catch \|err\| switch (err) {
	2721	// The directory-ness is either unknown or unknowable
	2722	error.Streaming => break :is_dir false,
	2723	else => \|e\| return e,
	2724	};
	2725	break :is_dir stat.kind == .directory;
	2726	};
	2727	if (is_dir) return error.IsDir;
	2728	}
	2729
	2730	return .{
	2731	.handle = fd,
	2732	.flags = .{ .nonblocking = os_flags.NONBLOCK },
	2733	};
	2734	}
	2735
	2736	fn dirClose(userdata: ?*anyopaque, dirs: []const Dir) void {
	2737	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2738	_ = ev;
	2739	for (dirs) \|dir\| closeFd(dir.handle);
	2740	}
	2741
	2742	fn dirRead(userdata: ?anyopaque, dr: Dir.Reader, buffer: []Dir.Entry) Dir.Reader.Error!usize {
	2743	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2744	const Header = extern struct {
	2745	seek: i64,
	2746	};
	2747	const header: *Header = @ptrCast(dr.buffer.ptr);
	2748	const header_end: usize = @sizeOf(Header);
	2749	if (dr.index < header_end) {
	2750	// Initialize header.
	2751	dr.index = header_end;
	2752	dr.end = header_end;
	2753	header.* = .{ .seek = 0 };
	2754	}
	2755	var buffer_index: usize = 0;
	2756	while (buffer.len - buffer_index != 0) {
	2757	if (dr.end - dr.index == 0) {
	2758	// Refill the buffer, unless we've already created references to
	2759	// buffered data.
	2760	if (buffer_index != 0) break;
	2761	if (dr.state == .reset) {
	2762	ev.lseek(dr.dir.handle, 0, c.SEEK.SET) catch \|err\| switch (err) {
	2763	error.Unseekable => return error.Unexpected,
	2764	else => \|e\| return e,
	2765	};
	2766	dr.state = .reading;
	2767	}
	2768	const dents_buffer = dr.buffer[header_end..];
	2769	const n: usize = while (true) {
	2770	const rc = c.getdirentries(dr.dir.handle, dents_buffer.ptr, dents_buffer.len, &header.seek);
	2771	switch (c.errno(rc)) {
	2772	.SUCCESS => break @intCast(rc),
	2773	.INTR => {},
	2774	.BADF => \|err\| return errnoBug(err), // Dir is invalid or was opened without iteration ability.
	2775	.FAULT => \|err\| return errnoBug(err),
	2776	.NOTDIR => \|err\| return errnoBug(err),
	2777	.INVAL => \|err\| return errnoBug(err),
	2778	else => \|err\| return unexpectedErrno(err),
	2779	}
	2780	};
	2781	if (n == 0) {
	2782	dr.state = .finished;
	2783	return 0;
	2784	}
	2785	dr.index = header_end;
	2786	dr.end = header_end + n;
	2787	}
	2788	const darwin_entry = @as(*align(1) c.dirent, @ptrCast(&dr.buffer[dr.index]));
	2789	const next_index = dr.index + darwin_entry.reclen;
	2790	dr.index = next_index;
	2791
	2792	const name = @as([*]u8, @ptrCast(&darwin_entry.name))[0..darwin_entry.namlen];
	2793	if (std.mem.eql(u8, name, ".") or std.mem.eql(u8, name, "..") or (darwin_entry.ino == 0))
	2794	continue;
	2795
	2796	const entry_kind: File.Kind = switch (darwin_entry.type) {
	2797	c.DT.BLK => .block_device,
	2798	c.DT.CHR => .character_device,
	2799	c.DT.DIR => .directory,
	2800	c.DT.FIFO => .named_pipe,
	2801	c.DT.LNK => .sym_link,
	2802	c.DT.REG => .file,
	2803	c.DT.SOCK => .unix_domain_socket,
	2804	c.DT.WHT => .whiteout,
	2805	else => .unknown,
	2806	};
	2807	buffer[buffer_index] = .{
	2808	.name = name,
	2809	.kind = entry_kind,
	2810	.inode = darwin_entry.ino,
	2811	};
	2812	buffer_index += 1;
	2813	}
	2814	return buffer_index;
	2815	}
	2816
	2817	fn dirRealPath(userdata: ?*anyopaque, dir: Dir, out_buffer: []u8) Dir.RealPathError!usize {
	2818	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2819	return ev.realPath(dir.handle, out_buffer);
	2820	}
	2821
	2822	fn realPath(ev: *Evented, fd: c.fd_t, out_buffer: []u8) File.RealPathError!usize {
	2823	_ = ev;
	2824	var buffer: [c.PATH_MAX]u8 = undefined;
	2825	@memset(&buffer, 0);
	2826	while (true) {
	2827	switch (c.errno(c.fcntl(fd, c.F.GETPATH, &buffer))) {
	2828	.SUCCESS => break,
	2829	.INTR => {},
	2830	.ACCES => return error.AccessDenied,
	2831	.BADF => return error.FileNotFound,
	2832	.NOENT => return error.FileNotFound,
	2833	.NOMEM => return error.SystemResources,
	2834	.NOSPC => return error.NameTooLong,
	2835	.RANGE => return error.NameTooLong,
	2836	else => \|err\| return unexpectedErrno(err),
	2837	}
	2838	}
	2839	const n = std.mem.indexOfScalar(u8, &buffer, 0) orelse buffer.len;
	2840	if (n > out_buffer.len) return error.NameTooLong;
	2841	@memcpy(out_buffer[0..n], buffer[0..n]);
	2842	return n;
	2843	}
	2844
	2845	fn dirRealPathFile(
	2846	userdata: ?*anyopaque,
	2847	dir: Dir,
	2848	sub_path: []const u8,
	2849	out_buffer: []u8,
	2850	) Dir.RealPathFileError!usize {
	2851	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2852
	2853	var path_buffer: [c.PATH_MAX]u8 = undefined;
	2854	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2855
	2856	if (dir.handle == c.AT.FDCWD) {
	2857	if (out_buffer.len < c.PATH_MAX) return error.NameTooLong;
	2858	while (true) {
	2859	if (c.realpath(sub_path_posix, out_buffer.ptr)) \|redundant_pointer\| {
	2860	assert(redundant_pointer == out_buffer.ptr);
	2861	return std.mem.indexOfScalar(u8, out_buffer, 0) orelse out_buffer.len;
	2862	}
	2863	const err: c.E = @enumFromInt(c._errno().*);
	2864	switch (err) {
	2865	.INTR => {},
	2866	.INVAL => return errnoBug(err),
	2867	.BADF => return errnoBug(err),
	2868	.FAULT => return errnoBug(err),
	2869	.ACCES => return error.AccessDenied,
	2870	.NOENT => return error.FileNotFound,
	2871	.OPNOTSUPP => return error.OperationUnsupported,
	2872	.NOTDIR => return error.NotDir,
	2873	.NAMETOOLONG => return error.NameTooLong,
	2874	.LOOP => return error.SymLinkLoop,
	2875	.IO => return error.InputOutput,
	2876	else => return unexpectedErrno(err),
	2877	}
	2878	}
	2879	}
	2880
	2881	const os_flags: c.O = .{
	2882	.NONBLOCK = true,
	2883	.CLOEXEC = true,
	2884	};
	2885
	2886	const fd: c.fd_t = while (true) {
	2887	const rc = c.openat(dir.handle, sub_path_posix, os_flags);
	2888	switch (c.errno(rc)) {
	2889	.SUCCESS => break @intCast(rc),
	2890	.INTR => {},
	2891	.FAULT => \|err\| return errnoBug(err),
	2892	.INVAL => return error.BadPathName,
	2893	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	2894	.ACCES => return error.AccessDenied,
	2895	.FBIG => return error.FileTooBig,
	2896	.OVERFLOW => return error.FileTooBig,
	2897	.ISDIR => return error.IsDir,
	2898	.LOOP => return error.SymLinkLoop,
	2899	.MFILE => return error.ProcessFdQuotaExceeded,
	2900	.NAMETOOLONG => return error.NameTooLong,
	2901	.NFILE => return error.SystemFdQuotaExceeded,
	2902	.NODEV => return error.NoDevice,
	2903	.NOENT => return error.FileNotFound,
	2904	.NOMEM => return error.SystemResources,
	2905	.NOSPC => return error.NoSpaceLeft,
	2906	.NOTDIR => return error.NotDir,
	2907	.PERM => return error.PermissionDenied,
	2908	.EXIST => return error.PathAlreadyExists,
	2909	.BUSY => return error.DeviceBusy,
	2910	.NXIO => return error.NoDevice,
	2911	.ILSEQ => return error.BadPathName,
	2912	else => \|err\| return unexpectedErrno(err),
	2913	}
	2914	};
	2915	defer closeFd(fd);
	2916	return ev.realPath(fd, out_buffer);
	2917	}
	2918
	2919	fn dirDeleteFile(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteFileError!void {
	2920	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2921	_ = ev;
	2922
	2923	var path_buffer: [c.PATH_MAX]u8 = undefined;
	2924	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2925
	2926	while (true) switch (c.errno(c.unlinkat(dir.handle, sub_path_posix, 0))) {
	2927	.SUCCESS => return,
	2928	.INTR => {},
	2929	// Some systems return permission errors when trying to delete a
	2930	// directory, so we need to handle that case specifically and
	2931	// translate the error.
	2932	.PERM => {
	2933	// Don't follow symlinks to match unlinkat (which acts on symlinks rather than follows them).
	2934	var st = std.mem.zeroes(c.Stat);
	2935	while (true) switch (c.errno(c.fstatat(
	2936	dir.handle,
	2937	sub_path_posix,
	2938	&st,
	2939	c.AT.SYMLINK_NOFOLLOW,
	2940	))) {
	2941	.SUCCESS => break,
	2942	.INTR => {},
	2943	else => return error.PermissionDenied,
	2944	};
	2945	if (st.mode & c.S.IFMT == c.S.IFDIR) return error.IsDir else return error.PermissionDenied;
	2946	},
	2947	.ACCES => return error.AccessDenied,
	2948	.BUSY => return error.FileBusy,
	2949	.FAULT => \|err\| return errnoBug(err),
	2950	.IO => return error.FileSystem,
	2951	.ISDIR => return error.IsDir,
	2952	.LOOP => return error.SymLinkLoop,
	2953	.NAMETOOLONG => return error.NameTooLong,
	2954	.NOENT => return error.FileNotFound,
	2955	.NOTDIR => return error.NotDir,
	2956	.NOMEM => return error.SystemResources,
	2957	.ROFS => return error.ReadOnlyFileSystem,
	2958	.EXIST => \|err\| return errnoBug(err),
	2959	.NOTEMPTY => \|err\| return errnoBug(err), // Not passing AT.REMOVEDIR
	2960	.ILSEQ => return error.BadPathName,
	2961	.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
	2962	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	2963	else => \|err\| return unexpectedErrno(err),
	2964	};
	2965	}
	2966
	2967	fn dirDeleteDir(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteDirError!void {
	2968	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2969	_ = ev;
	2970
	2971	var path_buffer: [c.PATH_MAX]u8 = undefined;
	2972	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2973
	2974	while (true) switch (c.errno(c.unlinkat(dir.handle, sub_path_posix, c.AT.REMOVEDIR))) {
	2975	.SUCCESS => return,
	2976	.INTR => {},
	2977	.ACCES => return error.AccessDenied,
	2978	.PERM => return error.PermissionDenied,
	2979	.BUSY => return error.FileBusy,
	2980	.FAULT => \|err\| return errnoBug(err),
	2981	.IO => return error.FileSystem,
	2982	.ISDIR => \|err\| return errnoBug(err),
	2983	.LOOP => return error.SymLinkLoop,
	2984	.NAMETOOLONG => return error.NameTooLong,
	2985	.NOENT => return error.FileNotFound,
	2986	.NOTDIR => return error.NotDir,
	2987	.NOMEM => return error.SystemResources,
	2988	.ROFS => return error.ReadOnlyFileSystem,
	2989	.EXIST => \|err\| return errnoBug(err),
	2990	.NOTEMPTY => return error.DirNotEmpty,
	2991	.ILSEQ => return error.BadPathName,
	2992	.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
	2993	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	2994	else => \|err\| return unexpectedErrno(err),
	2995	};
	2996	}
	2997
	2998	fn dirRename(
	2999	userdata: ?*anyopaque,
	3000	old_dir: Dir,
	3001	old_sub_path: []const u8,
	3002	new_dir: Dir,
	3003	new_sub_path: []const u8,
	3004	) Dir.RenameError!void {
	3005	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3006	_ = ev;
	3007
	3008	var old_path_buffer: [c.PATH_MAX]u8 = undefined;
	3009	var new_path_buffer: [c.PATH_MAX]u8 = undefined;
	3010
	3011	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
	3012	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
	3013
	3014	while (true) switch (c.errno(c.renameat(old_dir.handle, old_sub_path_posix, new_dir.handle, new_sub_path_posix))) {
	3015	.SUCCESS => return,
	3016	.INTR => {},
	3017	.ACCES => return error.AccessDenied,
	3018	.PERM => return error.PermissionDenied,
	3019	.BUSY => return error.FileBusy,
	3020	.DQUOT => return error.DiskQuota,
	3021	.ISDIR => return error.IsDir,
	3022	.IO => return error.HardwareFailure,
	3023	.LOOP => return error.SymLinkLoop,
	3024	.MLINK => return error.LinkQuotaExceeded,
	3025	.NAMETOOLONG => return error.NameTooLong,
	3026	.NOENT => return error.FileNotFound,
	3027	.NOTDIR => return error.NotDir,
	3028	.NOMEM => return error.SystemResources,
	3029	.NOSPC => return error.NoSpaceLeft,
	3030	.EXIST => return error.DirNotEmpty,
	3031	.NOTEMPTY => return error.DirNotEmpty,
	3032	.ROFS => return error.ReadOnlyFileSystem,
	3033	.XDEV => return error.CrossDevice,
	3034	.ILSEQ => return error.BadPathName,
	3035	.FAULT => \|err\| return errnoBug(err),
	3036	.INVAL => \|err\| return errnoBug(err),
	3037	else => \|err\| return unexpectedErrno(err),
	3038	};
	3039	}
	3040
	3041	fn dirRenamePreserve(
	3042	userdata: ?*anyopaque,
	3043	old_dir: Dir,
	3044	old_sub_path: []const u8,
	3045	new_dir: Dir,
	3046	new_sub_path: []const u8,
	3047	) Dir.RenamePreserveError!void {
	3048	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3049	// Make a hard link then delete the original.
	3050	try dirHardLink(ev, old_dir, old_sub_path, new_dir, new_sub_path, .{ .follow_symlinks = false });
	3051	const prev = swapCancelProtection(ev, .blocked);
	3052	defer _ = swapCancelProtection(ev, prev);
	3053	dirDeleteFile(ev, old_dir, old_sub_path) catch {};
	3054	}
	3055
	3056	fn dirSymLink(
	3057	userdata: ?*anyopaque,
	3058	dir: Dir,
	3059	target_path: []const u8,
	3060	sym_link_path: []const u8,
	3061	flags: Dir.SymLinkFlags,
	3062	) Dir.SymLinkError!void {
	3063	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3064	_ = ev;
	3065	_ = flags;
	3066
	3067	var target_path_buffer: [c.PATH_MAX]u8 = undefined;
	3068	var sym_link_path_buffer: [c.PATH_MAX]u8 = undefined;
	3069
	3070	const target_path_posix = try pathToPosix(target_path, &target_path_buffer);
	3071	const sym_link_path_posix = try pathToPosix(sym_link_path, &sym_link_path_buffer);
	3072
	3073	while (true) switch (c.errno(c.symlinkat(target_path_posix, dir.handle, sym_link_path_posix))) {
	3074	.SUCCESS => return,
	3075	.INTR => {},
	3076	.FAULT => \|err\| return errnoBug(err),
	3077	.INVAL => \|err\| return errnoBug(err),
	3078	.ACCES => return error.AccessDenied,
	3079	.PERM => return error.PermissionDenied,
	3080	.DQUOT => return error.DiskQuota,
	3081	.EXIST => return error.PathAlreadyExists,
	3082	.IO => return error.FileSystem,
	3083	.LOOP => return error.SymLinkLoop,
	3084	.NAMETOOLONG => return error.NameTooLong,
	3085	.NOENT => return error.FileNotFound,
	3086	.NOTDIR => return error.NotDir,
	3087	.NOMEM => return error.SystemResources,
	3088	.NOSPC => return error.NoSpaceLeft,
	3089	.ROFS => return error.ReadOnlyFileSystem,
	3090	.ILSEQ => return error.BadPathName,
	3091	else => \|err\| return unexpectedErrno(err),
	3092	};
	3093	}
	3094
	3095	fn dirReadLink(
	3096	userdata: ?*anyopaque,
	3097	dir: Dir,
	3098	sub_path: []const u8,
	3099	buffer: []u8,
	3100	) Dir.ReadLinkError!usize {
	3101	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3102	_ = ev;
	3103	var sub_path_buffer: [c.PATH_MAX]u8 = undefined;
	3104	const sub_path_posix = try pathToPosix(sub_path, &sub_path_buffer);
	3105	while (true) {
	3106	const rc = c.readlinkat(dir.handle, sub_path_posix, buffer.ptr, buffer.len);
	3107	switch (c.errno(rc)) {
	3108	.SUCCESS => return @intCast(rc),
	3109	.INTR => {},
	3110	.ACCES => return error.AccessDenied,
	3111	.FAULT => \|err\| return errnoBug(err),
	3112	.INVAL => return error.NotLink,
	3113	.IO => return error.FileSystem,
	3114	.LOOP => return error.SymLinkLoop,
	3115	.NAMETOOLONG => return error.NameTooLong,
	3116	.NOENT => return error.FileNotFound,
	3117	.NOMEM => return error.SystemResources,
	3118	.NOTDIR => return error.NotDir,
	3119	.ILSEQ => return error.BadPathName,
	3120	else => \|err\| return unexpectedErrno(err),
	3121	}
	3122	}
	3123	}
	3124
	3125	fn dirSetOwner(
	3126	userdata: ?*anyopaque,
	3127	dir: Dir,
	3128	owner: ?File.Uid,
	3129	group: ?File.Gid,
	3130	) Dir.SetOwnerError!void {
	3131	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3132	_ = ev;
	3133	return fchown(dir.handle, owner, group);
	3134	}
	3135
	3136	fn fchown(fd: c.fd_t, owner: ?File.Uid, group: ?File.Gid) File.SetOwnerError!void {
	3137	const uid = owner orelse std.math.maxInt(c.uid_t);
	3138	const gid = group orelse std.math.maxInt(c.gid_t);
	3139	while (true) switch (c.errno(c.fchown(fd, uid, gid))) {
	3140	.SUCCESS => return,
	3141	.INTR => {},
	3142	.BADF => \|err\| return errnoBug(err), // likely fd refers to directory opened without `Dir.OpenOptions.iterate`
	3143	.FAULT => \|err\| return errnoBug(err),
	3144	.INVAL => \|err\| return errnoBug(err),
	3145	.ACCES => return error.AccessDenied,
	3146	.IO => return error.InputOutput,
	3147	.LOOP => return error.SymLinkLoop,
	3148	.NOENT => return error.FileNotFound,
	3149	.NOMEM => return error.SystemResources,
	3150	.NOTDIR => return error.FileNotFound,
	3151	.PERM => return error.PermissionDenied,
	3152	.ROFS => return error.ReadOnlyFileSystem,
	3153	else => \|err\| return unexpectedErrno(err),
	3154	};
	3155	}
	3156
	3157	fn dirSetFileOwner(
	3158	userdata: ?*anyopaque,
	3159	dir: Dir,
	3160	sub_path: []const u8,
	3161	owner: ?File.Uid,
	3162	group: ?File.Gid,
	3163	options: Dir.SetFileOwnerOptions,
	3164	) Dir.SetFileOwnerError!void {
	3165	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3166	var path_buffer: [c.PATH_MAX]u8 = undefined;
	3167	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3168	_ = ev;
	3169	while (true) switch (c.errno(c.fchownat(
	3170	dir.handle,
	3171	sub_path_posix,
	3172	owner orelse std.math.maxInt(c.uid_t),
	3173	group orelse std.math.maxInt(c.gid_t),
	3174	if (options.follow_symlinks) 0 else c.AT.SYMLINK_NOFOLLOW,
	3175	))) {
	3176	.SUCCESS => return,
	3177	.INTR => continue,
	3178	.BADF => \|err\| return errnoBug(err), // likely fd refers to directory opened without `Dir.OpenOptions.iterate`
	3179	.FAULT => \|err\| return errnoBug(err),
	3180	.INVAL => \|err\| return errnoBug(err),
	3181	.ACCES => return error.AccessDenied,
	3182	.IO => return error.InputOutput,
	3183	.LOOP => return error.SymLinkLoop,
	3184	.NOENT => return error.FileNotFound,
	3185	.NOMEM => return error.SystemResources,
	3186	.NOTDIR => return error.FileNotFound,
	3187	.PERM => return error.PermissionDenied,
	3188	.ROFS => return error.ReadOnlyFileSystem,
	3189	else => \|err\| return unexpectedErrno(err),
	3190	};
	3191	}
	3192
	3193	fn dirSetPermissions(
	3194	userdata: ?*anyopaque,
	3195	dir: Dir,
	3196	permissions: Dir.Permissions,
	3197	) Dir.SetPermissionsError!void {
	3198	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3199	return ev.fchmod(dir.handle, permissions.toMode());
	3200	}
	3201
	3202	fn dirSetFilePermissions(
	3203	userdata: ?*anyopaque,
	3204	dir: Dir,
	3205	sub_path: []const u8,
	3206	permissions: Dir.Permissions,
	3207	options: Dir.SetFilePermissionsOptions,
	3208	) Dir.SetFilePermissionsError!void {
	3209	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3210	_ = ev;
	3211
	3212	var path_buffer: [c.PATH_MAX]u8 = undefined;
	3213	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3214
	3215	const mode = permissions.toMode();
	3216	const flags: u32 = if (options.follow_symlinks) 0 else c.AT.SYMLINK_NOFOLLOW;
	3217
	3218	while (true) switch (c.errno(c.fchmodat(dir.handle, sub_path_posix, mode, flags))) {
	3219	.SUCCESS => return,
	3220	.INTR => {},
	3221	.BADF => \|err\| return errnoBug(err),
	3222	.FAULT => \|err\| return errnoBug(err),
	3223	.INVAL => \|err\| return errnoBug(err),
	3224	.ACCES => return error.AccessDenied,
	3225	.IO => return error.InputOutput,
	3226	.LOOP => return error.SymLinkLoop,
	3227	.MFILE => return error.ProcessFdQuotaExceeded,
	3228	.NAMETOOLONG => return error.NameTooLong,
	3229	.NFILE => return error.SystemFdQuotaExceeded,
	3230	.NOENT => return error.FileNotFound,
	3231	.NOTDIR => return error.FileNotFound,
	3232	.NOMEM => return error.SystemResources,
	3233	.OPNOTSUPP => return error.OperationUnsupported,
	3234	.PERM => return error.PermissionDenied,
	3235	.ROFS => return error.ReadOnlyFileSystem,
	3236	else => \|err\| return unexpectedErrno(err),
	3237	};
	3238	}
	3239
	3240	fn dirSetTimestamps(
	3241	userdata: ?*anyopaque,
	3242	dir: Dir,
	3243	sub_path: []const u8,
	3244	options: Dir.SetTimestampsOptions,
	3245	) Dir.SetTimestampsError!void {
	3246	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3247	_ = ev;
	3248
	3249	var times_buffer: [2]c.timespec = undefined;
	3250	const times = if (options.modify_timestamp == .now and options.access_timestamp == .now) null else p: {
	3251	times_buffer = .{
	3252	setTimestampToPosix(options.access_timestamp),
	3253	setTimestampToPosix(options.modify_timestamp),
	3254	};
	3255	break :p &times_buffer;
	3256	};
	3257
	3258	const flags: u32 = if (options.follow_symlinks) 0 else c.AT.SYMLINK_NOFOLLOW;
	3259
	3260	var path_buffer: [c.PATH_MAX]u8 = undefined;
	3261	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3262
	3263	while (true) switch (c.errno(c.utimensat(dir.handle, sub_path_posix, times, flags))) {
	3264	.SUCCESS => return,
	3265	.INTR => {},
	3266	.BADF => \|err\| return errnoBug(err), // always a race condition
	3267	.FAULT => \|err\| return errnoBug(err),
	3268	.INVAL => \|err\| return errnoBug(err),
	3269	.ACCES => return error.AccessDenied,
	3270	.PERM => return error.PermissionDenied,
	3271	.ROFS => return error.ReadOnlyFileSystem,
	3272	else => \|err\| return unexpectedErrno(err),
	3273	};
	3274	}
	3275
	3276	fn dirHardLink(
	3277	userdata: ?*anyopaque,
	3278	old_dir: Dir,
	3279	old_sub_path: []const u8,
	3280	new_dir: Dir,
	3281	new_sub_path: []const u8,
	3282	options: Dir.HardLinkOptions,
	3283	) Dir.HardLinkError!void {
	3284	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3285	_ = ev;
	3286
	3287	var old_path_buffer: [c.PATH_MAX]u8 = undefined;
	3288	var new_path_buffer: [c.PATH_MAX]u8 = undefined;
	3289
	3290	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
	3291	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
	3292
	3293	const flags: u32 = if (options.follow_symlinks) c.AT.SYMLINK_FOLLOW else 0;
	3294	return linkat(old_dir.handle, old_sub_path_posix, new_dir.handle, new_sub_path_posix, flags);
	3295	}
	3296
	3297	fn fileStat(userdata: ?*anyopaque, file: File) File.StatError!File.Stat {
	3298	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3299	_ = ev;
	3300	while (true) {
	3301	var stat = std.mem.zeroes(c.Stat);
	3302	switch (c.errno(c.fstat(file.handle, &stat))) {
	3303	.SUCCESS => return statFromPosix(&stat),
	3304	.INTR => {},
	3305	.INVAL => \|err\| return errnoBug(err),
	3306	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	3307	.NOMEM => return error.SystemResources,
	3308	.ACCES => return error.AccessDenied,
	3309	else => \|err\| return unexpectedErrno(err),
	3310	}
	3311	}
	3312	}
	3313
	3314	fn fileLength(userdata: ?*anyopaque, file: File) File.LengthError!u64 {
	3315	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3316	const stat = try fileStat(ev, file);
	3317	return stat.size;
	3318	}
	3319
	3320	fn fileClose(userdata: ?*anyopaque, files: []const File) void {
	3321	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3322	_ = ev;
	3323	for (files) \|file\| closeFd(file.handle);
	3324	}
	3325
	3326	fn fileWritePositional(
	3327	userdata: ?*anyopaque,
	3328	file: File,
	3329	header: []const u8,
	3330	data: []const []const u8,
	3331	splat: usize,
	3332	offset: u64,
	3333	) File.WritePositionalError!usize {
	3334	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3335	_ = ev;
	3336	var iovecs: [max_iovecs_len]iovec_const = undefined;
	3337	var iovlen: iovlen_t = 0;
	3338	var remaining: Io.Limit = .unlimited;
	3339	addBuf(true, &iovecs, &iovlen, &remaining, header);
	3340	for (data[0 .. data.len - 1]) \|bytes\| addBuf(true, &iovecs, &iovlen, &remaining, bytes);
	3341	const pattern = data[data.len - 1];
	3342	var backup_buffer: [splat_buffer_size]u8 = undefined;
	3343	if (iovecs.len - iovlen != 0 and remaining != .nothing) switch (splat) {
	3344	0 => {},
	3345	1 => addBuf(true, &iovecs, &iovlen, &remaining, pattern),
	3346	else => switch (pattern.len) {
	3347	0 => {},
	3348	1 => {
	3349	const splat_buffer = &backup_buffer;
	3350	const memset_len = @min(splat_buffer.len, splat);
	3351	const buf = splat_buffer[0..memset_len];
	3352	@memset(buf, pattern[0]);
	3353	addBuf(true, &iovecs, &iovlen, &remaining, buf);
	3354	var remaining_splat = splat - buf.len;
	3355	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0 and remaining != .nothing) {
	3356	assert(buf.len == splat_buffer.len);
	3357	addBuf(true, &iovecs, &iovlen, &remaining, splat_buffer);
	3358	remaining_splat -= splat_buffer.len;
	3359	}
	3360	addBuf(true, &iovecs, &iovlen, &remaining, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
	3361	},
	3362	else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
	3363	if (remaining == .nothing) break;
	3364	addBuf(true, &iovecs, &iovlen, &remaining, pattern);
	3365	},
	3366	},
	3367	};
	3368	if (iovlen == 0) return 0;
	3369	while (true) {
	3370	const rc = c.pwritev(file.handle, &iovecs, iovlen, @bitCast(offset));
	3371	switch (c.errno(rc)) {
	3372	.SUCCESS => return @intCast(rc),
	3373	.INTR => {},
	3374	.INVAL => \|err\| return errnoBug(err),
	3375	.FAULT => \|err\| return errnoBug(err),
	3376	.DESTADDRREQ => \|err\| return errnoBug(err), // `connect` was never called.
	3377	.CONNRESET => \|err\| return errnoBug(err), // Not a socket handle.
	3378	.BADF => return error.NotOpenForWriting,
	3379	.AGAIN => return error.WouldBlock,
	3380	.DQUOT => return error.DiskQuota,
	3381	.FBIG => return error.FileTooBig,
	3382	.IO => return error.InputOutput,
	3383	.NOSPC => return error.NoSpaceLeft,
	3384	.PERM => return error.PermissionDenied,
	3385	.PIPE => return error.BrokenPipe,
	3386	.BUSY => return error.DeviceBusy,
	3387	.TXTBSY => return error.FileBusy,
	3388	.NXIO => return error.Unseekable,
	3389	.SPIPE => return error.Unseekable,
	3390	.OVERFLOW => return error.Unseekable,
	3391	else => \|err\| return unexpectedErrno(err),
	3392	}
	3393	}
	3394	}
	3395
	3396	fn fileWriteFileStreaming(
	3397	userdata: ?*anyopaque,
	3398	file: File,
	3399	header: []const u8,
	3400	file_reader: *File.Reader,
	3401	limit: Io.Limit,
	3402	) File.Writer.WriteFileError!usize {
	3403	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3404	const reader_buffered = file_reader.interface.buffered();
	3405	if (reader_buffered.len >= @intFromEnum(limit)) {
	3406	const n = try fileWriteStreaming(ev, file, header, &.{limit.slice(reader_buffered)}, 1);
	3407	file_reader.interface.toss(n -\| header.len);
	3408	return n;
	3409	}
	3410	const file_limit = @intFromEnum(limit) - reader_buffered.len;
	3411	const out_fd = file.handle;
	3412	const in_fd = file_reader.file.handle;
	3413
	3414	if (file_reader.size) \|size\| {
	3415	if (size - file_reader.pos == 0) {
	3416	if (reader_buffered.len != 0) {
	3417	const n = try fileWriteStreaming(ev, file, header, &.{limit.slice(reader_buffered)}, 1);
	3418	file_reader.interface.toss(n -\| header.len);
	3419	return n;
	3420	} else {
	3421	return error.EndOfStream;
	3422	}
	3423	}
	3424	}
	3425
	3426	if (@atomicLoad(UseSendfile, &ev.use_sendfile, .monotonic) == .disabled) return error.Unimplemented;
	3427	const offset = std.math.cast(c.off_t, file_reader.pos) orelse return error.Unimplemented;
	3428	var hdtr_data: c.sf_hdtr = undefined;
	3429	var headers: [2]iovec_const = undefined;
	3430	var headers_i: u8 = 0;
	3431	if (header.len != 0) {
	3432	headers[headers_i] = .{ .base = header.ptr, .len = header.len };
	3433	headers_i += 1;
	3434	}
	3435	if (reader_buffered.len != 0) {
	3436	headers[headers_i] = .{ .base = reader_buffered.ptr, .len = reader_buffered.len };
	3437	headers_i += 1;
	3438	}
	3439	const hdtr: ?*c.sf_hdtr = if (headers_i == 0) null else b: {
	3440	hdtr_data = .{
	3441	.headers = &headers,
	3442	.hdr_cnt = headers_i,
	3443	.trailers = null,
	3444	.trl_cnt = 0,
	3445	};
	3446	break :b &hdtr_data;
	3447	};
	3448	const max_count = std.math.maxInt(i32); // Avoid EINVAL.
	3449	var len: c.off_t = @min(file_limit, max_count);
	3450	const flags = 0;
	3451	while (true) switch (c.errno(c.sendfile(in_fd, out_fd, offset, &len, hdtr, flags))) {
	3452	.SUCCESS => break,
	3453	.OPNOTSUPP, .NOTSOCK, .NOSYS => {
	3454	// Give calling code chance to observe before trying
	3455	// something else.
	3456	@atomicStore(UseSendfile, &ev.use_sendfile, .disabled, .monotonic);
	3457	return 0;
	3458	},
	3459	.INTR => if (len > 0) break,
	3460	.AGAIN => {
	3461	if (len == 0) return error.WouldBlock;
	3462	break;
	3463	},
	3464	else => \|e\| {
	3465	assert(error.Unexpected == switch (e) {
	3466	.NOTCONN => return error.BrokenPipe,
	3467	.IO => return error.InputOutput,
	3468	.PIPE => return error.BrokenPipe,
	3469	.BADF => \|err\| errnoBug(err),
	3470	.FAULT => \|err\| errnoBug(err),
	3471	.INVAL => \|err\| errnoBug(err),
	3472	else => \|err\| unexpectedErrno(err),
	3473	});
	3474	// Give calling code chance to observe the error before trying
	3475	// something else.
	3476	@atomicStore(UseSendfile, &ev.use_sendfile, .disabled, .monotonic);
	3477	return 0;
	3478	},
	3479	};
	3480	if (len == 0) {
	3481	file_reader.size = file_reader.pos;
	3482	return error.EndOfStream;
	3483	}
	3484	const u_len: usize = @bitCast(len);
	3485	file_reader.interface.toss(u_len -\| header.len);
	3486	return u_len;
	3487	}
	3488
	3489	fn fileWriteFilePositional(
	3490	userdata: ?*anyopaque,
	3491	file: File,
	3492	header: []const u8,
	3493	file_reader: *File.Reader,
	3494	limit: Io.Limit,
	3495	offset: u64,
	3496	) File.WriteFilePositionalError!usize {
	3497	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3498	const reader_buffered = file_reader.interface.buffered();
	3499	if (reader_buffered.len >= @intFromEnum(limit)) {
	3500	const n = try fileWritePositional(
	3501	ev,
	3502	file,
	3503	header,
	3504	&.{limit.slice(reader_buffered)},
	3505	1,
	3506	offset,
	3507	);
	3508	file_reader.interface.toss(n -\| header.len);
	3509	return n;
	3510	}
	3511	const out_fd = file.handle;
	3512	const in_fd = file_reader.file.handle;
	3513
	3514	if (file_reader.size) \|size\| {
	3515	if (size - file_reader.pos == 0) {
	3516	if (reader_buffered.len != 0) {
	3517	const n = try fileWritePositional(
	3518	ev,
	3519	file,
	3520	header,
	3521	&.{limit.slice(reader_buffered)},
	3522	1,
	3523	offset,
	3524	);
	3525	file_reader.interface.toss(n -\| header.len);
	3526	return n;
	3527	} else {
	3528	return error.EndOfStream;
	3529	}
	3530	}
	3531	}
	3532
	3533	if (@atomicLoad(UseFcopyfile, &ev.use_fcopyfile, .monotonic) == .disabled)
	3534	return error.Unimplemented;
	3535	if (file_reader.pos != 0) return error.Unimplemented;
	3536	if (offset != 0) return error.Unimplemented;
	3537	if (limit != .unlimited) return error.Unimplemented;
	3538	const size = file_reader.getSize() catch return error.Unimplemented;
	3539	if (header.len != 0 or reader_buffered.len != 0) {
	3540	const n = try fileWritePositional(
	3541	ev,
	3542	file,
	3543	header,
	3544	&.{limit.slice(reader_buffered)},
	3545	1,
	3546	offset,
	3547	);
	3548	file_reader.interface.toss(n -\| header.len);
	3549	return n;
	3550	}
	3551	while (true) {
	3552	const rc = c.fcopyfile(in_fd, out_fd, null, .{ .DATA = true });
	3553	switch (c.errno(rc)) {
	3554	.SUCCESS => break,
	3555	.INTR => {},
	3556	.OPNOTSUPP => {
	3557	// Give calling code chance to observe before trying
	3558	// something else.
	3559	@atomicStore(UseFcopyfile, &ev.use_fcopyfile, .disabled, .monotonic);
	3560	return 0;
	3561	},
	3562	else => \|e\| {
	3563	assert(error.Unexpected == switch (e) {
	3564	.NOMEM => return error.SystemResources,
	3565	.INVAL => \|err\| errnoBug(err),
	3566	else => \|err\| unexpectedErrno(err),
	3567	});
	3568	return 0;
	3569	},
	3570	}
	3571	}
	3572	file_reader.pos = size;
	3573	return size;
	3574	}
	3575
	3576	fn fileReadPositional(
	3577	userdata: ?*anyopaque,
	3578	file: File,
	3579	data: []const []u8,
	3580	offset: u64,
	3581	) File.ReadPositionalError!usize {
	3582	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3583	_ = ev;
	3584	var iovecs: [max_iovecs_len]iovec = undefined;
	3585	var iovlen: iovlen_t = 0;
	3586	var remaining: Io.Limit = .unlimited;
	3587	for (data) \|buf\| addBuf(false, &iovecs, &iovlen, &remaining, buf);
	3588	if (iovlen == 0) return 0;
	3589	while (true) {
	3590	const rc = c.preadv(file.handle, &iovecs, iovlen, @bitCast(offset));
	3591	switch (c.errno(rc)) {
	3592	.SUCCESS => return @intCast(rc),
	3593	.INTR => {},
	3594	.NXIO => return error.Unseekable,
	3595	.SPIPE => return error.Unseekable,
	3596	.OVERFLOW => return error.Unseekable,
	3597	.NOBUFS => return error.SystemResources,
	3598	.NOMEM => return error.SystemResources,
	3599	.AGAIN => return error.WouldBlock,
	3600	.IO => return error.InputOutput,
	3601	.ISDIR => return error.IsDir,
	3602	.NOTCONN => \|err\| return errnoBug(err), // not a socket
	3603	.CONNRESET => \|err\| return errnoBug(err), // not a socket
	3604	.INVAL => \|err\| return errnoBug(err),
	3605	.FAULT => \|err\| return errnoBug(err),
	3606	.BADF => return error.NotOpenForReading,
	3607	else => \|err\| return unexpectedErrno(err),
	3608	}
	3609	}
	3610	}
	3611
	3612	fn fileSeekBy(userdata: ?*anyopaque, file: File, offset: i64) File.SeekError!void {
	3613	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3614	return ev.lseek(file.handle, @bitCast(offset), c.SEEK.CUR);
	3615	}
	3616
	3617	fn fileSeekTo(userdata: ?*anyopaque, file: File, offset: u64) File.SeekError!void {
	3618	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3619	return ev.lseek(file.handle, offset, c.SEEK.SET);
	3620	}
	3621
	3622	fn lseek(ev: *Evented, fd: c.fd_t, offset: u64, whence: i32) File.SeekError!void {
	3623	_ = ev;
	3624	while (true) switch (c.errno(c.lseek(fd, @bitCast(offset), whence))) {
	3625	.SUCCESS => return,
	3626	.INTR => {},
	3627	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	3628	.INVAL => return error.Unseekable,
	3629	.OVERFLOW => return error.Unseekable,
	3630	.SPIPE => return error.Unseekable,
	3631	.NXIO => return error.Unseekable,
	3632	else => \|err\| return unexpectedErrno(err),
	3633	};
	3634	}
	3635
	3636	fn fileSync(userdata: ?*anyopaque, file: File) File.SyncError!void {
	3637	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3638	_ = ev;
	3639	while (true) switch (c.errno(c.fsync(file.handle))) {
	3640	.SUCCESS => return,
	3641	.INTR => {},
	3642	.BADF => \|err\| return errnoBug(err),
	3643	.INVAL => \|err\| return errnoBug(err),
	3644	.ROFS => \|err\| return errnoBug(err),
	3645	.IO => return error.InputOutput,
	3646	.NOSPC => return error.NoSpaceLeft,
	3647	.DQUOT => return error.DiskQuota,
	3648	else => \|err\| return unexpectedErrno(err),
	3649	};
	3650	}
	3651
	3652	fn fileIsTty(userdata: ?*anyopaque, file: File) Io.Cancelable!bool {
	3653	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3654	_ = ev;
	3655	while (true) {
	3656	const rc = c.isatty(file.handle);
	3657	switch (c.errno(rc - 1)) {
	3658	.SUCCESS => return true,
	3659	.INTR => {},
	3660	else => return false,
	3661	}
	3662	}
	3663	}
	3664
	3665	fn fileEnableAnsiEscapeCodes(userdata: ?*anyopaque, file: File) File.EnableAnsiEscapeCodesError!void {
	3666	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3667	if (!try fileIsTty(ev, file)) return error.NotTerminalDevice;
	3668	}
	3669
	3670	fn fileSetLength(userdata: ?*anyopaque, file: File, length: u64) File.SetLengthError!void {
	3671	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3672	_ = ev;
	3673
	3674	const signed_len: i64 = @bitCast(length);
	3675	if (signed_len < 0) return error.FileTooBig; // Avoid ambiguous EINVAL errors.
	3676
	3677	while (true) switch (c.errno(c.ftruncate(file.handle, signed_len))) {
	3678	.SUCCESS => return,
	3679	.INTR => {},
	3680	.FBIG => return error.FileTooBig,
	3681	.IO => return error.InputOutput,
	3682	.PERM => return error.PermissionDenied,
	3683	.TXTBSY => return error.FileBusy,
	3684	.BADF => \|err\| return errnoBug(err), // Handle not open for writing.
	3685	.INVAL => return error.NonResizable, // This is returned for /dev/null for example.
	3686	else => \|err\| return unexpectedErrno(err),
	3687	};
	3688	}
	3689
	3690	fn fileSetOwner(
	3691	userdata: ?*anyopaque,
	3692	file: File,
	3693	owner: ?File.Uid,
	3694	group: ?File.Gid,
	3695	) File.SetOwnerError!void {
	3696	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3697	_ = ev;
	3698	return fchown(file.handle, owner, group);
	3699	}
	3700
	3701	fn fileSetPermissions(
	3702	userdata: ?*anyopaque,
	3703	file: File,
	3704	permissions: File.Permissions,
	3705	) File.SetPermissionsError!void {
	3706	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3707	return ev.fchmod(file.handle, permissions.toMode());
	3708	}
	3709
	3710	fn fchmod(ev: *Evented, fd: c.fd_t, mode: c.mode_t) File.SetPermissionsError!void {
	3711	_ = ev;
	3712	while (true) switch (c.errno(c.fchmod(fd, mode))) {
	3713	.SUCCESS => return,
	3714	.INTR => {},
	3715	.BADF => \|err\| return errnoBug(err),
	3716	.FAULT => \|err\| return errnoBug(err),
	3717	.INVAL => \|err\| return errnoBug(err),
	3718	.ACCES => return error.AccessDenied,
	3719	.IO => return error.InputOutput,
	3720	.LOOP => return error.SymLinkLoop,
	3721	.NOENT => return error.FileNotFound,
	3722	.NOMEM => return error.SystemResources,
	3723	.NOTDIR => return error.FileNotFound,
	3724	.PERM => return error.PermissionDenied,
	3725	.ROFS => return error.ReadOnlyFileSystem,
	3726	else => \|err\| return unexpectedErrno(err),
	3727	};
	3728	}
	3729
	3730	fn fileSetTimestamps(
	3731	userdata: ?*anyopaque,
	3732	file: File,
	3733	options: File.SetTimestampsOptions,
	3734	) File.SetTimestampsError!void {
	3735	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3736	_ = ev;
	3737
	3738	var times_buffer: [2]c.timespec = undefined;
	3739	const times = if (options.modify_timestamp == .now and options.access_timestamp == .now) null else p: {
	3740	times_buffer = .{
	3741	setTimestampToPosix(options.access_timestamp),
	3742	setTimestampToPosix(options.modify_timestamp),
	3743	};
	3744	break :p &times_buffer;
	3745	};
	3746
	3747	while (true) switch (c.errno(c.futimens(file.handle, times))) {
	3748	.SUCCESS => return,
	3749	.INTR => {},
	3750	.BADF => \|err\| return errnoBug(err), // always a race condition
	3751	.FAULT => \|err\| return errnoBug(err),
	3752	.INVAL => \|err\| return errnoBug(err),
	3753	.ACCES => return error.AccessDenied,
	3754	.PERM => return error.PermissionDenied,
	3755	.ROFS => return error.ReadOnlyFileSystem,
	3756	else => \|err\| return unexpectedErrno(err),
	3757	};
	3758	}
	3759
	3760	fn fileLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!void {
	3761	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3762	_ = ev;
	3763	const operation: i32 = switch (lock) {
	3764	.none => c.LOCK.UN,
	3765	.shared => c.LOCK.SH,
	3766	.exclusive => c.LOCK.EX,
	3767	};
	3768	while (true) switch (c.errno(c.flock(file.handle, operation))) {
	3769	.SUCCESS => return,
	3770	.INTR => {},
	3771	.BADF => \|err\| return errnoBug(err),
	3772	.INVAL => \|err\| return errnoBug(err), // invalid parameters
	3773	.NOLCK => return error.SystemResources,
	3774	.AGAIN => \|err\| return errnoBug(err),
	3775	.OPNOTSUPP => return error.FileLocksUnsupported,
	3776	else => \|err\| return unexpectedErrno(err),
	3777	};
	3778	}
	3779
	3780	fn fileTryLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!bool {
	3781	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3782	_ = ev;
	3783	const operation: i32 = switch (lock) {
	3784	.none => c.LOCK.UN,
	3785	.shared => c.LOCK.SH \| c.LOCK.NB,
	3786	.exclusive => c.LOCK.EX \| c.LOCK.NB,
	3787	};
	3788	while (true) switch (c.errno(c.flock(file.handle, operation))) {
	3789	.SUCCESS => return true,
	3790	.INTR => {},
	3791	.AGAIN => return false,
	3792	.BADF => \|err\| return errnoBug(err),
	3793	.INVAL => \|err\| return errnoBug(err), // invalid parameters
	3794	.NOLCK => return error.SystemResources,
	3795	.OPNOTSUPP => return error.FileLocksUnsupported,
	3796	else => \|err\| return unexpectedErrno(err),
	3797	};
	3798	}
	3799
	3800	fn fileUnlock(userdata: ?*anyopaque, file: File) void {
	3801	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3802	_ = ev;
	3803	while (true) switch (c.errno(c.flock(file.handle, c.LOCK.UN))) {
	3804	.SUCCESS => return,
	3805	.INTR => {},
	3806	.AGAIN => return recoverableOsBugDetected(), // unlocking can't block
	3807	.BADF => return recoverableOsBugDetected(), // File descriptor used after closed.
	3808	.INVAL => return recoverableOsBugDetected(), // invalid parameters
	3809	.NOLCK => return recoverableOsBugDetected(), // Resource deallocation.
	3810	.OPNOTSUPP => return recoverableOsBugDetected(), // We already got the lock.
	3811	else => return recoverableOsBugDetected(), // Resource deallocation must succeed.
	3812	};
	3813	}
	3814
	3815	fn fileDowngradeLock(userdata: ?*anyopaque, file: File) File.DowngradeLockError!void {
	3816	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3817	_ = ev;
	3818	const operation = c.LOCK.SH \| c.LOCK.NB;
	3819	while (true) switch (c.errno(c.flock(file.handle, operation))) {
	3820	.SUCCESS => return,
	3821	.INTR => {},
	3822	.AGAIN => \|err\| return errnoBug(err), // File was not locked in exclusive mode.
	3823	.BADF => \|err\| return errnoBug(err),
	3824	.INVAL => \|err\| return errnoBug(err), // invalid parameters
	3825	.NOLCK => \|err\| return errnoBug(err), // Lock already obtained.
	3826	.OPNOTSUPP => \|err\| return errnoBug(err), // Lock already obtained.
	3827	else => \|err\| return unexpectedErrno(err),
	3828	};
	3829	}
	3830
	3831	fn fileRealPath(userdata: ?*anyopaque, file: File, out_buffer: []u8) File.RealPathError!usize {
	3832	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3833	_ = ev;
	3834	var buffer: [c.PATH_MAX]u8 = undefined;
	3835	@memset(&buffer, 0);
	3836	while (true) {
	3837	switch (c.errno(c.fcntl(file.handle, c.F.GETPATH, &buffer))) {
	3838	.SUCCESS => break,
	3839	.INTR => {},
	3840	.ACCES => return error.AccessDenied,
	3841	.BADF => return error.FileNotFound,
	3842	.NOENT => return error.FileNotFound,
	3843	.NOMEM => return error.SystemResources,
	3844	.NOSPC => return error.NameTooLong,
	3845	.RANGE => return error.NameTooLong,
	3846	else => \|err\| return unexpectedErrno(err),
	3847	}
	3848	}
	3849	const n = std.mem.indexOfScalar(u8, &buffer, 0) orelse buffer.len;
	3850	if (n > out_buffer.len) return error.NameTooLong;
	3851	@memcpy(out_buffer[0..n], buffer[0..n]);
	3852	return n;
	3853	}
	3854
	3855	fn fileHardLink(
	3856	userdata: ?*anyopaque,
	3857	file: File,
	3858	new_dir: Dir,
	3859	new_sub_path: []const u8,
	3860	options: File.HardLinkOptions,
	3861	) File.HardLinkError!void {
	3862	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3863	_ = ev;
	3864	_ = file;
	3865	_ = new_dir;
	3866	_ = new_sub_path;
	3867	_ = options;
	3868	return error.OperationUnsupported;
	3869	}
	3870
	3871	fn linkat(
	3872	old_dir: c.fd_t,
	3873	old_path: [*:0]const u8,
	3874	new_dir: c.fd_t,
	3875	new_path: [*:0]const u8,
	3876	flags: u32,
	3877	) File.HardLinkError!void {
	3878	while (true) switch (c.errno(c.linkat(old_dir, old_path, new_dir, new_path, flags))) {
	3879	.SUCCESS => return,
	3880	.INTR => {},
	3881	.ACCES => return error.AccessDenied,
	3882	.DQUOT => return error.DiskQuota,
	3883	.EXIST => return error.PathAlreadyExists,
	3884	.IO => return error.HardwareFailure,
	3885	.LOOP => return error.SymLinkLoop,
	3886	.MLINK => return error.LinkQuotaExceeded,
	3887	.NAMETOOLONG => return error.NameTooLong,
	3888	.NOENT => return error.FileNotFound,
	3889	.NOMEM => return error.SystemResources,
	3890	.NOSPC => return error.NoSpaceLeft,
	3891	.NOTDIR => return error.NotDir,
	3892	.PERM => return error.PermissionDenied,
	3893	.ROFS => return error.ReadOnlyFileSystem,
	3894	.XDEV => return error.CrossDevice,
	3895	.ILSEQ => return error.BadPathName,
	3896	.FAULT => \|err\| return errnoBug(err),
	3897	.INVAL => \|err\| return errnoBug(err),
	3898	else => \|err\| return unexpectedErrno(err),
	3899	};
	3900	}
	3901
	3902	fn fileMemoryMapCreate(
	3903	userdata: ?*anyopaque,
	3904	file: File,
	3905	options: File.MemoryMap.CreateOptions,
	3906	) File.MemoryMap.CreateError!File.MemoryMap {
	3907	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3908	_ = ev;
	3909
	3910	const prot: c.PROT = .{
	3911	.READ = options.protection.read,
	3912	.WRITE = options.protection.write,
	3913	.EXEC = options.protection.execute,
	3914	};
	3915	const flags: c.MAP = .{
	3916	.TYPE = .SHARED,
	3917	};
	3918
	3919	const page_align = std.heap.page_size_min;
	3920
	3921	const contents = while (true) {
	3922	const casted_offset = std.math.cast(i64, options.offset) orelse return error.Unseekable;
	3923	const rc = c.mmap(null, options.len, prot, flags, file.handle, casted_offset);
	3924	const err: c.E = if (rc != c.MAP_FAILED) .SUCCESS else @enumFromInt(c._errno().*);
	3925	switch (err) {
	3926	.SUCCESS => break @as([*]align(page_align) u8, @ptrCast(@alignCast(rc)))[0..options.len],
	3927	.INTR => {},
	3928	.ACCES => return error.AccessDenied,
	3929	.AGAIN => return error.LockedMemoryLimitExceeded,
	3930	.MFILE => return error.ProcessFdQuotaExceeded,
	3931	.NFILE => return error.SystemFdQuotaExceeded,
	3932	.NOMEM => return error.OutOfMemory,
	3933	.PERM => return error.PermissionDenied,
	3934	.OVERFLOW => return error.Unseekable,
	3935	.BADF => return errnoBug(err), // Always a race condition.
	3936	.INVAL => return errnoBug(err), // Invalid parameters to mmap()
	3937	else => return unexpectedErrno(err),
	3938	}
	3939	};
	3940	return .{
	3941	.file = file,
	3942	.offset = options.offset,
	3943	.memory = contents,
	3944	.section = {},
	3945	};
	3946	}
	3947
	3948	fn fileMemoryMapDestroy(userdata: ?anyopaque, mm: File.MemoryMap) void {
	3949	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3950	_ = ev;
	3951	const memory = mm.memory;
	3952	if (memory.len == 0) return;
	3953	switch (c.errno(c.munmap(memory.ptr, memory.len))) {
	3954	.SUCCESS => {},
	3955	else => \|err\| if (builtin.mode == .Debug)
	3956	std.log.err("failed to unmap {d} bytes at {*}: {t}", .{ memory.len, memory.ptr, err }),
	3957	}
	3958	mm.* = undefined;
	3959	}
	3960
	3961	fn fileMemoryMapSetLength(
	3962	userdata: ?*anyopaque,
	3963	mm: *File.MemoryMap,
	3964	new_len: usize,
	3965	) File.MemoryMap.SetLengthError!void {
	3966	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3967	_ = ev;
	3968
	3969	const page_size = std.heap.pageSize();
	3970	const alignment: Alignment = .fromByteUnits(page_size);
	3971	const old_memory = mm.memory;
	3972
	3973	if (alignment.forward(new_len) == alignment.forward(old_memory.len)) {
	3974	mm.memory.len = new_len;
	3975	return;
	3976	}
	3977	return error.OperationUnsupported;
	3978	}
	3979
	3980	fn fileMemoryMapRead(userdata: ?anyopaque, mm: File.MemoryMap) File.ReadPositionalError!void {
	3981	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3982	_ = ev;
	3983	_ = mm;
	3984	}
	3985
	3986	fn fileMemoryMapWrite(userdata: ?anyopaque, mm: File.MemoryMap) File.WritePositionalError!void {
	3987	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3988	_ = ev;
	3989	_ = mm;
	3990	}
	3991
	3992	fn processExecutableOpen(
	3993	userdata: ?*anyopaque,
	3994	flags: File.OpenFlags,
	3995	) process.OpenExecutableError!File {
	3996	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3997	// _NSGetExecutablePath() returns a path that might be a symlink to
	3998	// the executable. Here it does not matter since we open it.
	3999	var symlink_path_buf: [c.PATH_MAX + 1]u8 = undefined;
	4000	var n: u32 = symlink_path_buf.len;
	4001	const rc = c._NSGetExecutablePath(&symlink_path_buf, &n);
	4002	if (rc != 0) return error.NameTooLong;
	4003	const symlink_path = std.mem.sliceTo(&symlink_path_buf, 0);
	4004	return dirOpenFile(ev, .cwd(), symlink_path, flags);
	4005	}
	4006
	4007	fn processExecutablePath(userdata: ?*anyopaque, out_buffer: []u8) process.ExecutablePathError!usize {
	4008	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4009	// _NSGetExecutablePath() returns a path that might be a symlink to
	4010	// the executable.
	4011	var symlink_path_buf: [c.PATH_MAX + 1]u8 = undefined;
	4012	var n: u32 = symlink_path_buf.len;
	4013	const rc = c._NSGetExecutablePath(&symlink_path_buf, &n);
	4014	if (rc != 0) return error.NameTooLong;
	4015	const symlink_path = std.mem.sliceTo(&symlink_path_buf, 0);
	4016	assert(Dir.path.isAbsolute(symlink_path));
	4017	return dirRealPathFile(ev, .cwd(), symlink_path, out_buffer) catch \|err\| switch (err) {
	4018	error.NetworkNotFound => unreachable, // Windows-only
	4019	error.FileBusy => unreachable, // Windows-only
	4020	else => \|e\| return e,
	4021	};
	4022	}
	4023
	4024	fn lockStderr(userdata: ?*anyopaque, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
	4025	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4026	try ev.stderr_mutex.lock(ev);
	4027	errdefer ev.stderr_mutex.unlock();
	4028	return ev.initLockedStderr(terminal_mode);
	4029	}
	4030
	4031	fn tryLockStderr(
	4032	userdata: ?*anyopaque,
	4033	terminal_mode: ?Io.Terminal.Mode,
	4034	) Io.Cancelable!?Io.LockedStderr {
	4035	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4036	if (!ev.stderr_mutex.tryLock()) return null;
	4037	errdefer ev.stderr_mutex.unlock();
	4038	return try ev.initLockedStderr(terminal_mode);
	4039	}
	4040
	4041	fn initLockedStderr(ev: *Evented, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
	4042	ev.init_stderr_writer.once(ev, &initStderrWriter);
	4043	return .{
	4044	.file_writer = &ev.stderr_writer,
	4045	.terminal_mode = terminal_mode orelse ev.stderr_mode,
	4046	};
	4047	}
	4048
	4049	fn initStderrWriter(context: ?*anyopaque) callconv(.c) void {
	4050	const ev: *Evented = @ptrCast(@alignCast(context));
	4051	const cancel_protection = swapCancelProtection(ev, .blocked);
	4052	defer assert(swapCancelProtection(ev, cancel_protection) == .blocked);
	4053	ev.scan_environ.once(ev, &scanEnviron);
	4054	const NO_COLOR = ev.environ.exist.NO_COLOR;
	4055	const CLICOLOR_FORCE = ev.environ.exist.CLICOLOR_FORCE;
	4056	ev.stderr_mode = Io.Terminal.Mode.detect(
	4057	ev.io(),
	4058	ev.stderr_writer.file,
	4059	NO_COLOR,
	4060	CLICOLOR_FORCE,
	4061	) catch \|err\| switch (err) {
	4062	error.Canceled => unreachable, // blocked
	4063	};
	4064	}
	4065
	4066	fn unlockStderr(userdata: ?*anyopaque) void {
	4067	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4068	if (ev.stderr_writer.err == null) ev.stderr_writer.interface.flush() catch {};
	4069	if (ev.stderr_writer.err) \|err\| {
	4070	switch (err) {
	4071	error.Canceled => Thread.current().currentFiber().cancel_protection.recancel(),
	4072	else => {},
	4073	}
	4074	ev.stderr_writer.err = null;
	4075	}
	4076	ev.stderr_writer.interface.end = 0;
	4077	ev.stderr_writer.interface.buffer.len = 0;
	4078	ev.stderr_mutex.unlock();
	4079	}
	4080
	4081	fn processCurrentPath(userdata: ?*anyopaque, buffer: []u8) process.CurrentPathError!usize {
	4082	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4083	_ = ev;
	4084	const err: c.E = if (c.getcwd(buffer.ptr, buffer.len)) \|_\| .SUCCESS else @enumFromInt(c._errno().*);
	4085	switch (err) {
	4086	.SUCCESS => return std.mem.findScalar(u8, buffer, 0).?,
	4087	.NOENT => return error.CurrentDirUnlinked,
	4088	.RANGE => return error.NameTooLong,
	4089	.FAULT => \|e\| return errnoBug(e),
	4090	.INVAL => \|e\| return errnoBug(e),
	4091	else => return unexpectedErrno(err),
	4092	}
	4093	}
	4094
	4095	fn processSetCurrentDir(userdata: ?*anyopaque, dir: Dir) process.SetCurrentDirError!void {
	4096	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4097	_ = ev;
	4098	if (dir.handle == c.AT.FDCWD) return;
	4099	while (true) switch (c.errno(c.fchdir(dir.handle))) {
	4100	.SUCCESS => return,
	4101	.INTR => {},
	4102	.ACCES => return error.AccessDenied,
	4103	.NOTDIR => return error.NotDir,
	4104	.IO => return error.FileSystem,
	4105	.BADF => \|err\| return errnoBug(err),
	4106	else => \|err\| return unexpectedErrno(err),
	4107	};
	4108	}
	4109
	4110	fn processSetCurrentPath(userdata: ?*anyopaque, dir_path: []const u8) ChdirError!void {
	4111	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4112	_ = ev;
	4113	var path_buffer: [c.PATH_MAX]u8 = undefined;
	4114	const dir_path_posix = try pathToPosix(dir_path, &path_buffer);
	4115	while (true) switch (c.errno(c.chdir(dir_path_posix))) {
	4116	.SUCCESS => return,
	4117	.INTR => {},
	4118	.ACCES => return error.AccessDenied,
	4119	.IO => return error.FileSystem,
	4120	.LOOP => return error.SymLinkLoop,
	4121	.NAMETOOLONG => return error.NameTooLong,
	4122	.NOENT => return error.FileNotFound,
	4123	.NOMEM => return error.SystemResources,
	4124	.NOTDIR => return error.NotDir,
	4125	.ILSEQ => return error.BadPathName,
	4126	.FAULT => \|err\| return errnoBug(err),
	4127	else => \|err\| return unexpectedErrno(err),
	4128	};
	4129	}
	4130
	4131	fn processReplace(userdata: ?*anyopaque, options: process.ReplaceOptions) process.ReplaceError {
	4132	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4133
	4134	if (!process.can_replace) return error.OperationUnsupported;
	4135
	4136	ev.scan_environ.once(ev, &scanEnviron); // for PATH
	4137	const PATH = ev.environ.string.PATH orelse default_PATH;
	4138
	4139	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
	4140	defer arena_allocator.deinit();
	4141	const arena = arena_allocator.allocator();
	4142
	4143	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
	4144	for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
	4145
	4146	const env_block = env_block: {
	4147	const prog_fd: i32 = -1;
	4148	if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
	4149	.zig_progress_fd = prog_fd,
	4150	});
	4151	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
	4152	.zig_progress_fd = prog_fd,
	4153	});
	4154	};
	4155
	4156	return ev.execv(options.expand_arg0, argv_buf.ptr[0].?, argv_buf.ptr, env_block, PATH);
	4157	}
	4158
	4159	fn processReplacePath(
	4160	userdata: ?*anyopaque,
	4161	dir: Dir,
	4162	options: process.ReplaceOptions,
	4163	) process.ReplaceError {
	4164	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4165	_ = ev;
	4166	_ = dir;
	4167	_ = options;
	4168	@panic("TODO processReplacePath");
	4169	}
	4170
	4171	fn processSpawn(userdata: ?*anyopaque, options: process.SpawnOptions) process.SpawnError!process.Child {
	4172	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4173	const spawned = try ev.spawn(options);
	4174	defer fileClose(ev, &.{spawned.err_pipe});
	4175
	4176	// Wait for the child to report any errors in or before `execvpe`.
	4177	var child_err: ForkBailError = undefined;
	4178	ev.readAll(spawned.err_pipe, @ptrCast(&child_err)) catch \|read_err\| {
	4179	switch (read_err) {
	4180	error.Canceled => unreachable, // blocked
	4181	error.EndOfStream => {
	4182	// Write end closed by CLOEXEC at the time of the `execvpe` call,
	4183	// indicating success.
	4184	},
	4185	else => {
	4186	// Problem reading the error from the error reporting pipe. We
	4187	// don't know if the child is alive or dead. Better to assume it is
	4188	// alive so the resource does not risk being leaked.
	4189	},
	4190	}
	4191	return .{
	4192	.id = spawned.pid,
	4193	.thread_handle = {},
	4194	.stdin = spawned.stdin,
	4195	.stdout = spawned.stdout,
	4196	.stderr = spawned.stderr,
	4197	.request_resource_usage_statistics = options.request_resource_usage_statistics,
	4198	};
	4199	};
	4200	return child_err;
	4201	}
	4202
	4203	fn processSpawnPath(
	4204	userdata: ?*anyopaque,
	4205	dir: Dir,
	4206	options: process.SpawnOptions,
	4207	) process.SpawnError!process.Child {
	4208	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4209	_ = ev;
	4210	_ = dir;
	4211	_ = options;
	4212	@panic("TODO processSpawnPath");
	4213	}
	4214
	4215	const prog_fileno = @max(c.STDIN_FILENO, c.STDOUT_FILENO, c.STDERR_FILENO) + 1;
	4216
	4217	const Spawned = struct {
	4218	pid: c.pid_t,
	4219	err_pipe: File,
	4220	stdin: ?File,
	4221	stdout: ?File,
	4222	stderr: ?File,
	4223	};
	4224	fn spawn(ev: *Evented, options: process.SpawnOptions) process.SpawnError!Spawned {
	4225	// The child process does need to access (one end of) these pipes. However,
	4226	// we must initially set CLOEXEC to avoid a race condition. If another thread
	4227	// is racing to spawn a different child process, we don't want it to inherit
	4228	// these FDs in any scenario; that would mean that, for instance, calls to
	4229	// `poll` from the parent would not report the child's stdout as closing when
	4230	// expected, since the other child may retain a reference to the write end of
	4231	// the pipe. So, we create the pipes with CLOEXEC initially. After fork, we
	4232	// need to do something in the new child to make sure we preserve the reference
	4233	// we want. We could use `fcntl` to remove CLOEXEC from the FD, but as it
	4234	// turns out, we `dup2` everything anyway, so there's no need!
	4235	const pipe_flags: c.O = .{ .CLOEXEC = true };
	4236
	4237	const stdin_pipe = if (options.stdin == .pipe) try pipe2(pipe_flags) else undefined;
	4238	errdefer if (options.stdin == .pipe) {
	4239	destroyPipe(stdin_pipe);
	4240	};
	4241
	4242	const stdout_pipe = if (options.stdout == .pipe) try pipe2(pipe_flags) else undefined;
	4243	errdefer if (options.stdout == .pipe) {
	4244	destroyPipe(stdout_pipe);
	4245	};
	4246
	4247	const stderr_pipe = if (options.stderr == .pipe) try pipe2(pipe_flags) else undefined;
	4248	errdefer if (options.stderr == .pipe) {
	4249	destroyPipe(stderr_pipe);
	4250	};
	4251
	4252	const any_ignore =
	4253	options.stdin == .ignore or options.stdout == .ignore or options.stderr == .ignore;
	4254	const dev_null_file = if (any_ignore) dev_null_file: {
	4255	ev.open_dev_null.once(ev, &openDevNullFile);
	4256	break :dev_null_file try ev.dev_null_file;
	4257	} else undefined;
	4258
	4259	const prog_pipe: [2]c.fd_t = if (options.progress_node.index != .none)
	4260	// We use CLOEXEC for the same reason as in `pipe_flags`.
	4261	try pipe2(.{ .NONBLOCK = true, .CLOEXEC = true })
	4262	else
	4263	.{ -1, -1 };
	4264	errdefer destroyPipe(prog_pipe);
	4265
	4266	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
	4267	defer arena_allocator.deinit();
	4268	const arena = arena_allocator.allocator();
	4269
	4270	// The POSIX standard does not allow malloc() between fork() and execve(),
	4271	// and this allocator may be a libc allocator.
	4272	// I have personally observed the child process deadlocking when it tries
	4273	// to call malloc() due to a heap allocation between fork() and execve(),
	4274	// in musl v1.1.24.
	4275	// Additionally, we want to reduce the number of possible ways things
	4276	// can fail between fork() and execve().
	4277	// Therefore, we do all the allocation for the execve() before the fork().
	4278	// This means we must do the null-termination of argv and env vars here.
	4279	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
	4280	for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
	4281
	4282	const env_block = env_block: {
	4283	const prog_fd: i32 = if (prog_pipe[1] == -1) -1 else prog_fileno;
	4284	if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
	4285	.zig_progress_fd = prog_fd,
	4286	});
	4287	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
	4288	.zig_progress_fd = prog_fd,
	4289	});
	4290	};
	4291
	4292	// This pipe communicates to the parent errors in the child between `fork` and `execvpe`.
	4293	// It is closed by the child (via CLOEXEC) without writing if `execvpe` succeeds.
	4294	const err_pipe: [2]File = err_pipe: {
	4295	const err_pipe = try pipe2(.{ .CLOEXEC = true });
	4296	break :err_pipe .{
	4297	.{ .handle = err_pipe[0], .flags = .{ .nonblocking = false } },
	4298	.{ .handle = err_pipe[1], .flags = .{ .nonblocking = false } },
	4299	};
	4300	};
	4301	errdefer fileClose(ev, &err_pipe);
	4302
	4303	ev.scan_environ.once(ev, &scanEnviron); // for PATH
	4304	const PATH = ev.environ.string.PATH orelse default_PATH;
	4305
	4306	const pid_result: c.pid_t = fork: {
	4307	const rc = c.fork();
	4308	switch (c.errno(rc)) {
	4309	.SUCCESS => break :fork @intCast(rc),
	4310	.AGAIN => return error.SystemResources,
	4311	.NOMEM => return error.SystemResources,
	4312	.NOSYS => return error.OperationUnsupported,
	4313	else => \|err\| return unexpectedErrno(err),
	4314	}
	4315	};
	4316
	4317	if (pid_result == 0) {
	4318	defer comptime unreachable; // We are the child.
	4319	const err = ev.setUpChild(.{
	4320	.stdin_pipe = stdin_pipe[0],
	4321	.stdout_pipe = stdout_pipe[1],
	4322	.stderr_pipe = stderr_pipe[1],
	4323	.dev_null_fd = dev_null_file.handle,
	4324	.prog_pipe = prog_pipe[1],
	4325	.argv_buf = argv_buf,
	4326	.env_block = env_block,
	4327	.PATH = PATH,
	4328	.spawn = options,
	4329	});
	4330	ev.writeAll(err_pipe[1], @ptrCast(&err)) catch {};
	4331	c.exit(1);
	4332	}
	4333
	4334	const pid: c.pid_t = @intCast(pid_result); // We are the parent.
	4335	errdefer comptime unreachable; // The child is forked; we must not error from now on
	4336
	4337	fileClose(ev, err_pipe[1..2]); // make sure only the child holds the write end open
	4338
	4339	if (options.stdin == .pipe) closeFd(stdin_pipe[0]);
	4340	if (options.stdout == .pipe) closeFd(stdout_pipe[1]);
	4341	if (options.stderr == .pipe) closeFd(stderr_pipe[1]);
	4342
	4343	if (prog_pipe[1] != -1) closeFd(prog_pipe[1]);
	4344
	4345	options.progress_node.setIpcFile(ev, .{ .handle = prog_pipe[0], .flags = .{ .nonblocking = true } });
	4346
	4347	return .{
	4348	.pid = pid,
	4349	.err_pipe = err_pipe[0],
	4350	.stdin = switch (options.stdin) {
	4351	.pipe => .{ .handle = stdin_pipe[1], .flags = .{ .nonblocking = false } },
	4352	else => null,
	4353	},
	4354	.stdout = switch (options.stdout) {
	4355	.pipe => .{ .handle = stdout_pipe[0], .flags = .{ .nonblocking = false } },
	4356	else => null,
	4357	},
	4358	.stderr = switch (options.stderr) {
	4359	.pipe => .{ .handle = stderr_pipe[0], .flags = .{ .nonblocking = false } },
	4360	else => null,
	4361	},
	4362	};
	4363	}
	4364
	4365	fn openDevNullFile(context: ?*anyopaque) callconv(.c) void {
	4366	const ev: *Evented = @ptrCast(@alignCast(context));
	4367	ev.dev_null_file = dirOpenFile(ev, .cwd(), "/dev/null", .{ .mode = .read_write });
	4368	}
	4369
	4370	/// Errors that can occur between fork() and execv()
	4371	const ForkBailError = process.SetCurrentDirError \|\| ChdirError \|\|
	4372	process.SpawnError \|\| process.ReplaceError;
	4373	fn setUpChild(ev: *Evented, options: struct {
	4374	stdin_pipe: c.fd_t,
	4375	stdout_pipe: c.fd_t,
	4376	stderr_pipe: c.fd_t,
	4377	dev_null_fd: c.fd_t,
	4378	prog_pipe: c.fd_t,
	4379	argv_buf: [:null]?[*:0]const u8,
	4380	env_block: process.Environ.Block,
	4381	PATH: []const u8,
	4382	spawn: process.SpawnOptions,
	4383	}) ForkBailError {
	4384	try ev.setUpChildIo(
	4385	options.spawn.stdin,
	4386	options.stdin_pipe,
	4387	c.STDIN_FILENO,
	4388	options.dev_null_fd,
	4389	);
	4390	try ev.setUpChildIo(
	4391	options.spawn.stdout,
	4392	options.stdout_pipe,
	4393	c.STDOUT_FILENO,
	4394	options.dev_null_fd,
	4395	);
	4396	try ev.setUpChildIo(
	4397	options.spawn.stderr,
	4398	options.stderr_pipe,
	4399	c.STDERR_FILENO,
	4400	options.dev_null_fd,
	4401	);
	4402
	4403	switch (options.spawn.cwd) {
	4404	.inherit => {},
	4405	.dir => \|cwd_dir\| try processSetCurrentDir(ev, cwd_dir),
	4406	.path => \|cwd_path\| try processSetCurrentPath(ev, cwd_path),
	4407	}
	4408
	4409	// Must happen after fchdir above, the cwd file descriptor might be
	4410	// equal to prog_fileno and be clobbered by this dup2 call.
	4411	if (options.prog_pipe != -1) try ev.dup2(options.prog_pipe, prog_fileno);
	4412
	4413	if (options.spawn.gid) \|gid\| while (true) switch (c.errno(c.setregid(gid, gid))) {
	4414	.SUCCESS => break,
	4415	.INTR => {},
	4416	.AGAIN => return error.ResourceLimitReached,
	4417	.INVAL => return error.InvalidUserId,
	4418	.PERM => return error.PermissionDenied,
	4419	else => return error.Unexpected,
	4420	};
	4421
	4422	if (options.spawn.uid) \|uid\| while (true) switch (c.errno(c.setreuid(uid, uid))) {
	4423	.SUCCESS => break,
	4424	.INTR => {},
	4425	.AGAIN => return error.ResourceLimitReached,
	4426	.INVAL => return error.InvalidUserId,
	4427	.PERM => return error.PermissionDenied,
	4428	else => return error.Unexpected,
	4429	};
	4430
	4431	if (options.spawn.pgid) \|pid\| while (true) switch (c.errno(c.setpgid(0, pid))) {
	4432	.SUCCESS => break,
	4433	.INTR => {},
	4434	.ACCES => return error.ProcessAlreadyExec,
	4435	.INVAL => return error.InvalidProcessGroupId,
	4436	.PERM => return error.PermissionDenied,
	4437	else => return error.Unexpected,
	4438	};
	4439
	4440	if (options.spawn.start_suspended) while (true) switch (c.errno(c.kill(0, .STOP))) {
	4441	.SUCCESS => break,
	4442	.INTR => {},
	4443	.PERM => return error.PermissionDenied,
	4444	else => return error.Unexpected,
	4445	};
	4446
	4447	return ev.execv(
	4448	options.spawn.expand_arg0,
	4449	options.argv_buf.ptr[0].?,
	4450	options.argv_buf.ptr,
	4451	options.env_block,
	4452	options.PATH,
	4453	);
	4454	}
	4455
	4456	fn setUpChildIo(
	4457	ev: *Evented,
	4458	stdio: process.SpawnOptions.StdIo,
	4459	pipe_fd: c.fd_t,
	4460	std_fileno: i32,
	4461	dev_null_fd: c.fd_t,
	4462	) !void {
	4463	switch (stdio) {
	4464	.pipe => try ev.dup2(pipe_fd, std_fileno),
	4465	.close => closeFd(std_fileno),
	4466	.inherit => {},
	4467	.ignore => try ev.dup2(dev_null_fd, std_fileno),
	4468	.file => \|file\| {
	4469	if (file.flags.nonblocking) @panic("TODO implement setUpChildIo when nonblocking file is used");
	4470	try ev.dup2(file.handle, std_fileno);
	4471	},
	4472	}
	4473	}
	4474
	4475	const PipeError = error{
	4476	SystemFdQuotaExceeded,
	4477	ProcessFdQuotaExceeded,
	4478	} \|\| Io.UnexpectedError;
	4479
	4480	fn pipe2(flags: c.O) PipeError![2]c.fd_t {
	4481	var fds: [2]c.fd_t = undefined;
	4482
	4483	while (true) switch (c.errno(c.pipe(&fds))) {
	4484	.SUCCESS => break,
	4485	.INTR => {},
	4486	.NFILE => return error.SystemFdQuotaExceeded,
	4487	.MFILE => return error.ProcessFdQuotaExceeded,
	4488	else => \|err\| return unexpectedErrno(err),
	4489	};
	4490	errdefer {
	4491	closeFd(fds[0]);
	4492	closeFd(fds[1]);
	4493	}
	4494
	4495	// https://github.com/ziglang/zig/issues/18882
	4496	if (@as(u32, @bitCast(flags)) == 0) return fds;
	4497
	4498	// CLOEXEC is special, it's a file descriptor flag and must be set using
	4499	// F.SETFD.
	4500	if (flags.CLOEXEC) for (fds) \|fd\| while (true) switch (c.errno(c.fcntl(fd, c.F.SETFD, @as(u32, c.FD_CLOEXEC)))) {
	4501	.SUCCESS => break,
	4502	.INTR => {},
	4503	else => \|err\| return unexpectedErrno(err),
	4504	};
	4505
	4506	const new_flags: u32 = f: {
	4507	var new_flags = flags;
	4508	new_flags.CLOEXEC = false;
	4509	break :f @bitCast(new_flags);
	4510	};
	4511
	4512	// Set every other flag affecting the file status using F.SETFL.
	4513	if (new_flags != 0) for (fds) \|fd\| while (true) switch (c.errno(c.fcntl(fd, c.F.SETFL, new_flags))) {
	4514	.SUCCESS => break,
	4515	.INTR => {},
	4516	.INVAL => \|err\| return errnoBug(err),
	4517	else => \|err\| return unexpectedErrno(err),
	4518	};
	4519
	4520	return fds;
	4521	}
	4522
	4523	fn destroyPipe(pipe: [2]c.fd_t) void {
	4524	if (pipe[0] != -1) closeFd(pipe[0]);
	4525	if (pipe[0] != pipe[1]) closeFd(pipe[1]);
	4526	}
	4527
	4528	const DupError = error{
	4529	ProcessFdQuotaExceeded,
	4530	SystemResources,
	4531	} \|\| Io.UnexpectedError \|\| Io.Cancelable;
	4532	fn dup2(ev: *Evented, old_fd: c.fd_t, new_fd: c.fd_t) DupError!void {
	4533	_ = ev;
	4534	while (true) switch (c.errno(c.dup2(old_fd, new_fd))) {
	4535	.SUCCESS => return,
	4536	.BUSY, .INTR => {},
	4537	.INVAL => \|err\| return errnoBug(err), // invalid parameters
	4538	.BADF => \|err\| return errnoBug(err), // use after free
	4539	.MFILE => return error.ProcessFdQuotaExceeded,
	4540	.NOMEM => return error.SystemResources,
	4541	else => \|err\| return unexpectedErrno(err),
	4542	};
	4543	}
	4544
	4545	fn execv(
	4546	ev: *Evented,
	4547	arg0_expand: process.ArgExpansion,
	4548	file: [*:0]const u8,
	4549	child_argv: [:null]?[:0]const u8,
	4550	env_block: process.Environ.PosixBlock,
	4551	PATH: []const u8,
	4552	) process.ReplaceError {
	4553	const file_slice = std.mem.sliceTo(file, 0);
	4554	if (std.mem.findScalar(u8, file_slice, '/') != null) return ev.execvPath(file, child_argv, env_block);
	4555
	4556	// Use of PATH_MAX here is valid as the path_buf will be passed
	4557	// directly to the operating system in posixExecvPath.
	4558	var path_buf: [c.PATH_MAX]u8 = undefined;
	4559	var it = std.mem.tokenizeScalar(u8, PATH, ':');
	4560	var seen_eacces = false;
	4561	var err: process.ReplaceError = error.FileNotFound;
	4562
	4563	// In case of expanding arg0 we must put it back if we return with an error.
	4564	const prev_arg0 = child_argv[0];
	4565	defer switch (arg0_expand) {
	4566	.expand => child_argv[0] = prev_arg0,
	4567	.no_expand => {},
	4568	};
	4569
	4570	while (it.next()) \|search_path\| {
	4571	const path_len = search_path.len + file_slice.len + 1;
	4572	if (path_buf.len < path_len + 1) return error.NameTooLong;
	4573	@memcpy(path_buf[0..search_path.len], search_path);
	4574	path_buf[search_path.len] = '/';
	4575	@memcpy(path_buf[search_path.len + 1 ..][0..file_slice.len], file_slice);
	4576	path_buf[path_len] = 0;
	4577	const full_path = path_buf[0..path_len :0].ptr;
	4578	switch (arg0_expand) {
	4579	.expand => child_argv[0] = full_path,
	4580	.no_expand => {},
	4581	}
	4582	err = ev.execvPath(full_path, child_argv, env_block);
	4583	switch (err) {
	4584	error.AccessDenied => seen_eacces = true,
	4585	error.FileNotFound, error.NotDir => {},
	4586	else => \|e\| return e,
	4587	}
	4588	}
	4589	if (seen_eacces) return error.AccessDenied;
	4590	return err;
	4591	}
	4592	/// This function ignores PATH environment variable.
	4593	fn execvPath(
	4594	ev: *Evented,
	4595	path: [*:0]const u8,
	4596	child_argv: [:null]const ?[:0]const u8,
	4597	env_block: process.Environ.PosixBlock,
	4598	) process.ReplaceError {
	4599	_ = ev;
	4600	switch (c.errno(c.execve(path, child_argv, env_block.slice.ptr))) {
	4601	.FAULT => \|err\| return errnoBug(err), // Bad pointer parameter.
	4602	.@"2BIG" => return error.SystemResources,
	4603	.MFILE => return error.ProcessFdQuotaExceeded,
	4604	.NAMETOOLONG => return error.NameTooLong,
	4605	.NFILE => return error.SystemFdQuotaExceeded,
	4606	.NOMEM => return error.SystemResources,
	4607	.ACCES => return error.AccessDenied,
	4608	.PERM => return error.PermissionDenied,
	4609	.INVAL => return error.InvalidExe,
	4610	.NOEXEC => return error.InvalidExe,
	4611	.IO => return error.FileSystem,
	4612	.LOOP => return error.FileSystem,
	4613	.ISDIR => return error.IsDir,
	4614	.NOENT => return error.FileNotFound,
	4615	.NOTDIR => return error.NotDir,
	4616	.TXTBSY => return error.FileBusy,
	4617	.BADEXEC => return error.InvalidExe,
	4618	.BADARCH => return error.InvalidExe,
	4619	else => \|err\| return unexpectedErrno(err),
	4620	}
	4621	}
	4622
	4623	fn childWait(userdata: ?anyopaque, child: process.Child) process.Child.WaitError!process.Child.Term {
	4624	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4625	defer ev.childCleanup(child);
	4626	const pid = child.id.?;
	4627	const source = c.dispatch.source_create(
	4628	.PROC,
	4629	@bitCast(@as(isize, pid)),
	4630	.{ .PROC = .{ .EXIT = true } },
	4631	ev.queue,
	4632	) orelse return error.Unexpected;
	4633	source.as_object().set_context(Thread.current().currentFiber());
	4634	source.set_event_handler(&Fiber.@"resume");
	4635	ev.yield(.{ .activate = source.as_object() });
	4636	source.as_object().release();
	4637	var status: c_int = undefined;
	4638	var ru: c.rusage = undefined;
	4639	const ru_ptr = if (child.request_resource_usage_statistics) &ru else null;
	4640	while (true) switch (c.errno(c.wait4(pid, &status, 0, ru_ptr))) {
	4641	.SUCCESS => {
	4642	if (ru_ptr) \|p\| child.resource_usage_statistics.rusage = p.*;
	4643	return statusToTerm(@bitCast(status));
	4644	},
	4645	.INTR => {},
	4646	.CHILD => \|err\| return errnoBug(err), // Double-free.
	4647	else => \|err\| return unexpectedErrno(err),
	4648	};
	4649	}
	4650
	4651	fn childKill(userdata: ?anyopaque, child: process.Child) void {
	4652	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4653	defer ev.childCleanup(child);
	4654	const pid = child.id.?;
	4655	while (true) switch (c.errno(c.kill(pid, .TERM))) {
	4656	.SUCCESS => break,
	4657	.INTR => {},
	4658	.PERM => return,
	4659	.INVAL => \|err\| errnoBug(err) catch return,
	4660	.SRCH => \|err\| errnoBug(err) catch return,
	4661	else => \|err\| unexpectedErrno(err) catch return,
	4662	};
	4663	var status: c_int = undefined;
	4664	while (true) switch (c.errno(c.wait4(pid, &status, 0, null))) {
	4665	.SUCCESS => return,
	4666	.INTR => {},
	4667	.CHILD => \|err\| errnoBug(err) catch return, // Double-free.
	4668	else => \|err\| unexpectedErrno(err) catch return,
	4669	};
	4670	}
	4671
	4672	fn childCleanup(ev: Evented, child: process.Child) void {
	4673	if (child.stdin) \|stdin\| {
	4674	fileClose(ev, &.{stdin});
	4675	child.stdin = null;
	4676	}
	4677	if (child.stdout) \|stdout\| {
	4678	fileClose(ev, &.{stdout});
	4679	child.stdout = null;
	4680	}
	4681	if (child.stderr) \|stderr\| {
	4682	fileClose(ev, &.{stderr});
	4683	child.stderr = null;
	4684	}
	4685	child.id = null;
	4686	}
	4687
	4688	fn progressParentFile(userdata: ?*anyopaque) std.Progress.ParentFileError!File {
	4689	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4690	ev.scan_environ.once(ev, &scanEnviron);
	4691	return ev.environ.zig_progress_file;
	4692	}
	4693
	4694	fn scanEnviron(context: ?*anyopaque) callconv(.c) void {
	4695	const ev: *Evented = @ptrCast(@alignCast(context));
	4696	ev.environ.scan(ev.allocator());
	4697	}
	4698
	4699	fn now(userdata: ?*anyopaque, clock: Io.Clock) Io.Timestamp {
	4700	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4701	_ = ev;
	4702	const clock_id: c.clockid_t = clockToPosix(clock);
	4703	var timespec: c.timespec = undefined;
	4704	switch (c.errno(c.clock_gettime(clock_id, &timespec))) {
	4705	.SUCCESS => return timestampFromPosix(&timespec),
	4706	else => return .zero,
	4707	}
	4708	}
	4709
	4710	fn clockResolution(userdata: ?*anyopaque, clock: Io.Clock) Io.Clock.ResolutionError!Io.Duration {
	4711	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4712	_ = ev;
	4713	const clock_id: c.clockid_t = clockToPosix(clock);
	4714	var timespec: c.timespec = undefined;
	4715	return switch (c.errno(c.clock_getres(clock_id, &timespec))) {
	4716	.SUCCESS => .fromNanoseconds(nanosecondsFromPosix(&timespec)),
	4717	.INVAL => return error.ClockUnavailable,
	4718	else => \|err\| return unexpectedErrno(err),
	4719	};
	4720	}
	4721
	4722	fn sleep(userdata: ?*anyopaque, timeout: Io.Timeout) Io.Cancelable!void {
	4723	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4724	ev.yield(.{ .sleep = ev.timeFromTimeout(timeout) });
	4725	}
	4726
	4727	fn timeFromTimeout(ev: *Evented, timeout: Io.Timeout) c.dispatch.time_t {
	4728	return timeout: switch (timeout) {
	4729	.none => .FOREVER,
	4730	.duration => \|duration\| .time(switch (duration.clock) {
	4731	.real => .WALL_NOW,
	4732	else => .NOW,
	4733	}, std.math.lossyCast(i64, duration.raw.toNanoseconds())),
	4734	.deadline => \|deadline\| switch (deadline.clock) {
	4735	.real => .walltime(&.{
	4736	.sec = @intCast(@divFloor(deadline.raw.toNanoseconds(), std.time.ns_per_s)),
	4737	.nsec = @intCast(@mod(deadline.raw.toNanoseconds(), std.time.ns_per_s)),
	4738	}, 0),
	4739	else => continue :timeout .{ .duration = deadline.durationFromNow(ev.io()) },
	4740	},
	4741	};
	4742	}
	4743
	4744	const Random = struct {
	4745	evented: *Evented,
	4746	thread: *Thread,
	4747	buffer: []u8,
	4748
	4749	fn seed(context: ?*anyopaque) callconv(.c) void {
	4750	const rand: *Random = @ptrCast(@alignCast(context));
	4751	const ev = rand.evented;
	4752	ev.csprng_mutex.lockUncancelable(ev);
	4753	defer ev.csprng_mutex.unlock();
	4754	var buffer: [Csprng.seed_len]u8 = undefined;
	4755	if (!ev.csprng.isInitialized()) {
	4756	@branchHint(.unlikely);
	4757	const cancel_protection = swapCancelProtection(ev, .blocked);
	4758	defer assert(swapCancelProtection(ev, cancel_protection) == .blocked);
	4759	randomSecure(ev, &buffer) catch \|err\| switch (err) {
	4760	error.Canceled => unreachable, // blocked
	4761	error.EntropyUnavailable => fallbackSeed(ev, &buffer),
	4762	};
	4763	ev.csprng.rng = .init(buffer);
	4764	}
	4765	ev.csprng.rng.fill(&buffer);
	4766	rand.thread.csprng.rng = .init(buffer);
	4767	rand.thread.csprng.rng.fill(rand.buffer);
	4768	rand.buffer.len = 0;
	4769	}
	4770	};
	4771
	4772	fn random(userdata: ?*anyopaque, buffer: []u8) void {
	4773	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4774	if (buffer.len == 0) return;
	4775	const thread: *Thread = .current();
	4776	var rand: Random = .{ .evented = ev, .thread = thread, .buffer = buffer };
	4777	thread.seed_csprng.once(&rand, &Random.seed);
	4778	if (rand.buffer.len > 0) thread.csprng.rng.fill(buffer);
	4779	}
	4780
	4781	fn randomSecure(userdata: ?*anyopaque, buffer: []u8) Io.RandomSecureError!void {
	4782	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4783	_ = ev;
	4784	if (buffer.len > 0) c.arc4random_buf(buffer.ptr, buffer.len);
	4785	}
	4786
	4787	fn netListenIpUnavailable(
	4788	userdata: ?*anyopaque,
	4789	address: net.IpAddress,
	4790	options: net.IpAddress.ListenOptions,
	4791	) net.IpAddress.ListenError!net.Server {
	4792	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4793	_ = ev;
	4794	_ = address;
	4795	_ = options;
	4796	return error.NetworkDown;
	4797	}
	4798
	4799	fn netAcceptUnavailable(
	4800	userdata: ?*anyopaque,
	4801	listen_handle: net.Socket.Handle,
	4802	) net.Server.AcceptError!net.Stream {
	4803	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4804	_ = ev;
	4805	_ = listen_handle;
	4806	return error.NetworkDown;
	4807	}
	4808
	4809	fn netBindIpUnavailable(
	4810	userdata: ?*anyopaque,
	4811	address: *const net.IpAddress,
	4812	options: net.IpAddress.BindOptions,
	4813	) net.IpAddress.BindError!net.Socket {
	4814	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4815	_ = ev;
	4816	_ = address;
	4817	_ = options;
	4818	return error.NetworkDown;
	4819	}
	4820
	4821	fn netConnectIpUnavailable(
	4822	userdata: ?*anyopaque,
	4823	address: *const net.IpAddress,
	4824	options: net.IpAddress.ConnectOptions,
	4825	) net.IpAddress.ConnectError!net.Stream {
	4826	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4827	_ = ev;
	4828	_ = address;
	4829	_ = options;
	4830	return error.NetworkDown;
	4831	}
	4832
	4833	fn netListenUnixUnavailable(
	4834	userdata: ?*anyopaque,
	4835	address: *const net.UnixAddress,
	4836	options: net.UnixAddress.ListenOptions,
	4837	) net.UnixAddress.ListenError!net.Socket.Handle {
	4838	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4839	_ = ev;
	4840	_ = address;
	4841	_ = options;
	4842	return error.AddressFamilyUnsupported;
	4843	}
	4844
	4845	fn netConnectUnixUnavailable(
	4846	userdata: ?*anyopaque,
	4847	address: *const net.UnixAddress,
	4848	) net.UnixAddress.ConnectError!net.Socket.Handle {
	4849	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4850	_ = ev;
	4851	_ = address;
	4852	return error.AddressFamilyUnsupported;
	4853	}
	4854
	4855	fn netSocketCreatePairUnavailable(
	4856	userdata: ?*anyopaque,
	4857	options: net.Socket.CreatePairOptions,
	4858	) net.Socket.CreatePairError![2]net.Socket {
	4859	_ = userdata;
	4860	_ = options;
	4861	return error.OperationUnsupported;
	4862	}
	4863
	4864	fn netSendUnavailable(
	4865	userdata: ?*anyopaque,
	4866	handle: net.Socket.Handle,
	4867	messages: []net.OutgoingMessage,
	4868	flags: net.SendFlags,
	4869	) struct { ?net.Socket.SendError, usize } {
	4870	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4871	_ = ev;
	4872	_ = handle;
	4873	_ = messages;
	4874	_ = flags;
	4875	return .{ error.NetworkDown, 0 };
	4876	}
	4877
	4878	fn netReceiveUnavailable(
	4879	userdata: ?*anyopaque,
	4880	handle: net.Socket.Handle,
	4881	message_buffer: []net.IncomingMessage,
	4882	data_buffer: []u8,
	4883	flags: net.ReceiveFlags,
	4884	timeout: Io.Timeout,
	4885	) struct { ?net.Socket.ReceiveTimeoutError, usize } {
	4886	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4887	_ = ev;
	4888	_ = handle;
	4889	_ = message_buffer;
	4890	_ = data_buffer;
	4891	_ = flags;
	4892	_ = timeout;
	4893	return .{ error.NetworkDown, 0 };
	4894	}
	4895
	4896	fn netReadUnavailable(
	4897	userdata: ?*anyopaque,
	4898	fd: net.Socket.Handle,
	4899	data: [][]u8,
	4900	) net.Stream.Reader.Error!usize {
	4901	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4902	_ = ev;
	4903	_ = fd;
	4904	_ = data;
	4905	return error.NetworkDown;
	4906	}
	4907
	4908	fn netWriteUnavailable(
	4909	userdata: ?*anyopaque,
	4910	handle: net.Socket.Handle,
	4911	header: []const u8,
	4912	data: []const []const u8,
	4913	splat: usize,
	4914	) net.Stream.Writer.Error!usize {
	4915	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4916	_ = ev;
	4917	_ = handle;
	4918	_ = header;
	4919	_ = data;
	4920	_ = splat;
	4921	return error.NetworkDown;
	4922	}
	4923
	4924	fn netWriteFileUnavailable(
	4925	userdata: ?*anyopaque,
	4926	socket_handle: net.Socket.Handle,
	4927	header: []const u8,
	4928	file_reader: *File.Reader,
	4929	limit: Io.Limit,
	4930	) net.Stream.Writer.WriteFileError!usize {
	4931	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4932	_ = ev;
	4933	_ = socket_handle;
	4934	_ = header;
	4935	_ = file_reader;
	4936	_ = limit;
	4937	return error.NetworkDown;
	4938	}
	4939
	4940	fn netClose(userdata: ?*anyopaque, handles: []const net.Socket.Handle) void {
	4941	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4942	_ = ev;
	4943	for (handles) \|handle\| closeFd(handle);
	4944	}
	4945
	4946	fn netShutdownUnavailable(
	4947	userdata: ?*anyopaque,
	4948	handle: net.Socket.Handle,
	4949	how: net.ShutdownHow,
	4950	) net.ShutdownError!void {
	4951	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4952	_ = ev;
	4953	_ = handle;
	4954	_ = how;
	4955	unreachable; // How you gonna shutdown something that was impossible to open?
	4956	}
	4957
	4958	fn netInterfaceNameResolveUnavailable(
	4959	userdata: ?*anyopaque,
	4960	name: *const net.Interface.Name,
	4961	) net.Interface.Name.ResolveError!net.Interface {
	4962	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4963	_ = ev;
	4964	_ = name;
	4965	return error.InterfaceNotFound;
	4966	}
	4967
	4968	fn netInterfaceNameUnavailable(
	4969	userdata: ?*anyopaque,
	4970	interface: net.Interface,
	4971	) net.Interface.NameError!net.Interface.Name {
	4972	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4973	_ = ev;
	4974	_ = interface;
	4975	return error.Unexpected;
	4976	}
	4977
	4978	fn netLookupUnavailable(
	4979	userdata: ?*anyopaque,
	4980	host_name: net.HostName,
	4981	resolved: *Io.Queue(net.HostName.LookupResult),
	4982	options: net.HostName.LookupOptions,
	4983	) net.HostName.LookupError!void {
	4984	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4985	_ = host_name;
	4986	_ = options;
	4987	resolved.close(ev.io());
	4988	return error.NetworkDown;
	4989	}
	4990
	4991	fn readAll(ev: *Evented, file: File, buffer: []u8) File.ReadStreamingError!void {
	4992	var index: usize = 0;
	4993	while (buffer.len - index != 0) {
	4994	const len = try ev.fileReadStreaming(file, &.{buffer[index..]});
	4995	if (len == 0) return error.EndOfStream;
	4996	index += len;
	4997	}
	4998	}
	4999
	5000	fn writeAll(ev: *Evented, file: File, buffer: []const u8) (File.Writer.Error \|\| error{EndOfStream})!void {
	5001	var index: usize = 0;
	5002	while (buffer.len - index != 0) {
	5003	const len = try ev.fileWriteStreaming(file, &.{}, &.{buffer[index..]}, 1);
	5004	if (len == 0) return error.EndOfStream;
	5005	index += len;
	5006	}
	5007	}
	5008
	5009	/// This is either usize or u32. Since, either is fine, let's use the same
	5010	/// `addBuf` function for both writing to a file and sending network messages.
	5011	const iovlen_t = @FieldType(c.msghdr_const, "iovlen");
	5012
	5013	fn addConstBuf(v: []iovec_const, i: iovlen_t, remaining: ?usize, bytes: []const u8) void {
	5014	if (v.len - i.* == 0) return;
	5015	const len = @min(remaining.*, bytes.len);
	5016	if (len == 0) return;
	5017	v[i.*] = .{ .base = bytes.ptr, .len = len };
	5018	i.* += 1;
	5019	remaining.* -= len;
	5020	}
	5021	fn addBuf(
	5022	comptime is_const: bool,
	5023	vec: []if (is_const) iovec_const else iovec,
	5024	vec_len: *iovlen_t,
	5025	remaining: *Io.Limit,
	5026	bytes: if (is_const) []const u8 else []u8,
	5027	) void {
	5028	if (vec.len - vec_len.* == 0) return;
	5029	const len = remaining.minInt(bytes.len);
	5030	if (len == 0) return;
	5031	vec[vec_len.*] = .{ .base = bytes.ptr, .len = len };
	5032	vec_len.* += 1;
	5033	remaining.* = remaining.subtract(len).?;
	5034	}
	5035
	5036	test {
	5037	_ = Fiber.CancelProtection;
	5038	}

lib/std/Io/IoUring.zig deleted-6336

...	...	@@ -1,6336 +0,0 @@
1		const addressFromPosix = Io.Threaded.addressFromPosix;
2		const addressToPosix = Io.Threaded.addressToPosix;
3		const Alignment = std.mem.Alignment;
4		const Allocator = std.mem.Allocator;
5		const Argv0 = Io.Threaded.Argv0;
6		const assert = std.debug.assert;
7		const builtin = @import("builtin");
8		const ChdirError = Io.Threaded.ChdirError;
9		const clockToPosix = Io.Threaded.clockToPosix;
10		const Csprng = Io.Threaded.Csprng;
11		const default_PATH = Io.Threaded.default_PATH;
12		const Dir = Io.Dir;
13		const Environ = Io.Threaded.Environ;
14		const errnoBug = Io.Threaded.errnoBug;
15		const Evented = @This();
16		const fallbackSeed = Io.Threaded.fallbackSeed;
17		const fd_t = linux.fd_t;
18		const File = Io.File;
19		const Io = std.Io;
20		const IoUring = linux.IoUring;
21		const iovec = std.posix.iovec;
22		const iovec_const = std.posix.iovec_const;
23		const linux = std.os.linux;
24		const linux_statx_request = Io.Threaded.linux_statx_request;
25		const LOCK = std.posix.LOCK;
26		const log = std.log.scoped(.@"io-uring");
27		const max_iovecs_len = Io.Threaded.max_iovecs_len;
28		const nanosecondsFromPosix = Io.Threaded.nanosecondsFromPosix;
29		const net = Io.net;
30		const PATH_MAX = linux.PATH_MAX;
31		const pathToPosix = Io.Threaded.pathToPosix;
32		const pid_t = linux.pid_t;
33		const PosixAddress = Io.Threaded.PosixAddress;
34		const posixAddressFamily = Io.Threaded.posixAddressFamily;
35		const posixProtocol = Io.Threaded.posixProtocol;
36		const posixSocketMode = Io.Threaded.posixSocketMode;
37		const process = std.process;
38		const recoverableOsBugDetected = Io.Threaded.recoverableOsBugDetected;
39		const setTimestampToPosix = Io.Threaded.setTimestampToPosix;
40		const splat_buffer_size = Io.Threaded.splat_buffer_size;
41		const statFromLinux = Io.Threaded.statFromLinux;
42		const std = @import("../std.zig");
43		const timestampFromPosix = Io.Threaded.timestampFromPosix;
44		const unexpectedErrno = std.posix.unexpectedErrno;
45		const winsize = std.posix.winsize;
46
47		const tracy = if (@hasDecl(@import("root"), "tracy")) @import("root").tracy else struct {
48		const enable = false;
49		inline fn fiberEnter(fiber: [*:0]const u8) void {
50		_ = fiber;
51		}
52		inline fn fiberLeave() void {}
53		};
54
55		backing_allocator_needs_mutex: bool,
56		backing_allocator_mutex: Io.Mutex,
57		/// Does not need to be thread-safe if not used elsewhere.
58		backing_allocator: Allocator,
59		main_fiber_buffer: [
60		std.mem.alignForward(usize, @sizeOf(Fiber), @alignOf(Completion)) + @sizeOf(Completion)
61		]u8 align(@max(@alignOf(Fiber), @alignOf(Completion))),
62		log2_ring_entries: u4,
63		threads: Thread.List,
64		sync_limit: ?Io.Semaphore,
65
66		stderr_mutex: Io.Mutex,
67		stderr_writer: File.Writer = .{
68		.io = undefined,
69		.interface = Io.File.Writer.initInterface(&.{}),
70		.file = .stderr(),
71		.mode = .streaming,
72		},
73		stderr_mode: Io.Terminal.Mode = .no_color,
74		stderr_writer_initialized: bool = false,
75
76		environ_mutex: Io.Mutex,
77		environ: Environ,
78
79		null_fd: CachedFd,
80		random_fd: CachedFd,
81
82		csprng_mutex: Io.Mutex,
83		csprng: Csprng,
84
85		/// Empirically saw >128KB being used by the self-hosted backend to panic.
86		/// Empirically saw glibc complain about 256KB.
87		const idle_stack_size = 512 * 1024;
88
89		const max_idle_search = 1;
90		const max_steal_ready_search = 2;
91		const max_steal_free_search = 4;
92
93		const Thread = struct {
94		required_align: void align(4),
95		thread: std.Thread,
96		idle_context: Context,
97		current_context: *Context,
98		ready_queue: ?*Fiber,
99		free_queue: ?*Fiber,
100		io_uring: IoUring,
101		idle_search_index: u32,
102		steal_ready_search_index: u32,
103		steal_free_search_index: u32,
104		name_arena: if (tracy.enable) std.heap.ArenaAllocator.State else struct {},
105		csprng: Csprng,
106
107		threadlocal var self: ?*Thread = null;
108
109		noinline fn current() *Thread {
110		return self.?;
111		}
112
113		fn deinit(thread: *Thread, gpa: Allocator) void {
114		var next_fiber = thread.free_queue;
115		while (next_fiber) \|free_fiber\| {
116		next_fiber = free_fiber.status.free_next;
117		gpa.free(free_fiber.allocatedSlice());
118		}
119		thread.io_uring.deinit();
120		}
121
122		fn currentFiber(thread: Thread) Fiber {
123		assert(thread.current_context != &thread.idle_context);
124		return @fieldParentPtr("context", thread.current_context);
125		}
126
127		fn enqueue(thread: Thread) linux.io_uring_sqe {
128		while (true) return thread.io_uring.get_sqe() catch {
129		thread.submit();
130		continue;
131		};
132		}
133
134		fn submit(thread: *Thread) void {
135		_ = thread.io_uring.submit() catch \|err\| switch (err) {
136		error.SignalInterrupt => {},
137		else => \|e\| @panic(@errorName(e)),
138		};
139		}
140
141		const List = struct {
142		allocated: []Thread,
143		reserved: u32,
144		active: u32,
145		};
146		};
147
148		const Fiber = struct {
149		required_align: void align(4),
150		context: Context,
151		await_count: i32,
152		link: union {
153		awaiter: ?*Fiber,
154		group: struct { prev: ?Fiber, next: ?Fiber },
155		},
156		status: union(enum) {
157		queue_next: ?*Fiber,
158		awaiting_group: Group,
159		free_next: ?*Fiber,
160		},
161		cancel_status: CancelStatus,
162		cancel_protection: CancelProtection,
163		name: if (tracy.enable) [*:0]const u8 else void,
164
165		var next_name: u64 = 0;
166
167		const CancelStatus = packed struct(u32) {
168		requested: bool,
169		awaiting: Awaiting,
170
171		const unrequested: CancelStatus = .{ .requested = false, .awaiting = .nothing };
172
173		const Awaiting = enum(u31) {
174		nothing = std.math.maxInt(u31),
175		group = std.math.maxInt(u31) - 1,
176		select = std.math.maxInt(u31) - 2,
177		/// An io_uring fd.
178		_,
179
180		fn subWrap(lhs: Awaiting, rhs: Awaiting) Awaiting {
181		return @enumFromInt(@intFromEnum(lhs) -% @intFromEnum(rhs));
182		}
183
184		fn fromIoUringFd(fd: fd_t) Awaiting {
185		const awaiting: Awaiting = @enumFromInt(fd);
186		switch (awaiting) {
187		.nothing, .group, .select => unreachable,
188		_ => return awaiting,
189		}
190		}
191
192		fn toIoUringFd(awaiting: Awaiting) fd_t {
193		switch (awaiting) {
194		.nothing, .group => unreachable,
195		_ => return @intFromEnum(awaiting),
196		}
197		}
198		};
199
200		fn changeAwaiting(
201		cancel_status: *CancelStatus,
202		old_awaiting: Awaiting,
203		new_awaiting: Awaiting,
204		) bool {
205		const old_cancel_status = @atomicRmw(CancelStatus, cancel_status, .Add, .{
206		.requested = false,
207		.awaiting = new_awaiting.subWrap(old_awaiting),
208		}, .monotonic);
209		assert(old_cancel_status.awaiting == old_awaiting);
210		return old_cancel_status.requested;
211		}
212		};
213
214		const CancelProtection = packed struct {
215		user: Io.CancelProtection,
216		acknowledged: bool,
217
218		const unblocked: CancelProtection = .{ .user = .unblocked, .acknowledged = false };
219
220		fn check(cancel_protection: CancelProtection) Io.CancelProtection {
221		return @enumFromInt(@intFromBool(cancel_protection != unblocked));
222		}
223
224		fn acknowledge(cancel_protection: *CancelProtection) void {
225		assert(!cancel_protection.acknowledged);
226		cancel_protection.acknowledged = true;
227		}
228
229		fn recancel(cancel_protection: *CancelProtection) void {
230		assert(cancel_protection.acknowledged);
231		cancel_protection.acknowledged = false;
232		}
233
234		test check {
235		try std.testing.expectEqual(Io.CancelProtection.unblocked, check(.unblocked));
236		try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
237		.user = .unblocked,
238		.acknowledged = true,
239		}));
240		try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
241		.user = .blocked,
242		.acknowledged = false,
243		}));
244		try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
245		.user = .blocked,
246		.acknowledged = true,
247		}));
248		}
249		};
250
251		const finished: ?*Fiber = @ptrFromInt(@alignOf(Fiber));
252
253		const max_result_align: Alignment = .@"16";
254		const max_result_size = max_result_align.forward(512);
255		/// This includes any stack realignments that need to happen, and also the
256		/// initial frame return address slot and argument frame, depending on target.
257		const min_stack_size = 60 * 1024 * 1024;
258		const max_context_align: Alignment = .@"16";
259		const max_context_size = max_context_align.forward(1024);
260		const max_closure_size: usize = @sizeOf(AsyncClosure);
261		const max_closure_align: Alignment = .of(AsyncClosure);
262		const allocation_size = std.mem.alignForward(
263		usize,
264		max_closure_align.max(max_context_align).forward(
265		max_result_align.forward(@sizeOf(Fiber)) + max_result_size + min_stack_size,
266		) + max_closure_size + max_context_size,
267		std.heap.page_size_max,
268		);
269		comptime {
270		assert(max_result_align.compare(.gte, .of(Completion)));
271		assert(max_result_size >= @sizeOf(Completion));
272		}
273
274		fn create(ev: Evented) error{OutOfMemory}!Fiber {
275		const thread: *Thread = .current();
276		if (@atomicRmw(?*Fiber, &thread.free_queue, .Xchg, finished, .acquire)) \|free_fiber\| {
277		assert(free_fiber != finished);
278		@atomicStore(?*Fiber, &thread.free_queue, free_fiber.status.free_next, .release);
279		return free_fiber;
280		}
281		const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
282		for (0..@min(max_steal_free_search, active_threads)) \|_\| {
283		defer thread.steal_free_search_index += 1;
284		if (thread.steal_free_search_index == active_threads) thread.steal_free_search_index = 0;
285		const steal_free_search_thread =
286		&ev.threads.allocated[0..active_threads][thread.steal_free_search_index];
287		if (steal_free_search_thread == thread) continue;
288		const free_fiber =
289		@atomicLoad(?*Fiber, &steal_free_search_thread.free_queue, .monotonic) orelse continue;
290		if (free_fiber == finished) continue;
291		if (@cmpxchgWeak(
292		?*Fiber,
293		&steal_free_search_thread.free_queue,
294		free_fiber,
295		null,
296		.acquire,
297		.monotonic,
298		)) \|_\| continue;
299		@atomicStore(?*Fiber, &thread.free_queue, free_fiber.status.free_next, .release);
300		return free_fiber;
301		}
302		@atomicStore(?*Fiber, &thread.free_queue, null, .monotonic);
303		return @ptrCast(try ev.allocator().alignedAlloc(u8, .of(Fiber), allocation_size));
304		}
305
306		fn destroy(fiber: *Fiber) void {
307		const thread: *Thread = .current();
308		assert(fiber.status.queue_next == null);
309		fiber.status = .{ .free_next = @atomicLoad(?*Fiber, &thread.free_queue, .acquire) };
310		while (true) fiber.status.free_next = @cmpxchgWeak(
311		?*Fiber,
312		&thread.free_queue,
313		fiber.status.free_next,
314		fiber,
315		.acq_rel,
316		.acquire,
317		) orelse break;
318		}
319
320		fn allocatedSlice(f: *Fiber) []align(@alignOf(Fiber)) u8 {
321		return @as([*]align(@alignOf(Fiber)) u8, @ptrCast(f))[0..allocation_size];
322		}
323
324		fn allocatedEnd(f: Fiber) []u8 {
325		const allocated_slice = f.allocatedSlice();
326		return allocated_slice[allocated_slice.len..].ptr;
327		}
328
329		fn resultPointer(f: Fiber, comptime Result: type) Result {
330		return @ptrCast(@alignCast(f.resultBytes(.of(Result))));
331		}
332
333		fn resultBytes(f: Fiber, alignment: Alignment) []u8 {
334		return @ptrFromInt(alignment.forward(@intFromPtr(f) + @sizeOf(Fiber)));
335		}
336
337		const Queue = struct { head: Fiber, tail: Fiber };
338
339		/// Like a `*Fiber`, but 2 bits smaller than a pointer (because the LSBs are always 0 due to
340		/// alignment) so that those two bits can be used in a `packed struct`.
341		const PackedPtr = enum(@Int(.unsigned, @bitSizeOf(usize) - 2)) {
342		null = 0,
343		all_ones = std.math.maxInt(@Int(.unsigned, @bitSizeOf(usize) - 2)),
344		_,
345
346		const Split = packed struct(usize) { low: u2, high: PackedPtr };
347		fn pack(ptr: ?*Fiber) PackedPtr {
348		const split: Split = @bitCast(@intFromPtr(ptr));
349		assert(split.low == 0);
350		return split.high;
351		}
352		fn unpack(ptr: PackedPtr) ?*Fiber {
353		const split: Split = .{ .low = 0, .high = ptr };
354		return @ptrFromInt(@as(usize, @bitCast(split)));
355		}
356		};
357
358		fn requestCancel(fiber: Fiber, ev: Evented) void {
359		const cancel_status = @atomicRmw(
360		Fiber.CancelStatus,
361		&fiber.cancel_status,
362		.Or,
363		.{ .requested = true, .awaiting = @enumFromInt(0) },
364		.acq_rel,
365		);
366		assert(!cancel_status.requested);
367		switch (cancel_status.awaiting) {
368		.nothing => {},
369		.group => {
370		// The awaiter received a cancelation request while awaiting a group,
371		// so propagate the cancelation to the group.
372		if (fiber.status.awaiting_group.cancel(ev, null)) {
373		fiber.status = .{ .queue_next = null };
374		_ = ev.schedule(.current(), .{ .head = fiber, .tail = fiber });
375		}
376		},
377		.select => if (@atomicRmw(i32, &fiber.await_count, .Add, 1, .monotonic) == -1) {
378		_ = ev.schedule(.current(), .{ .head = fiber, .tail = fiber });
379		},
380		_ => \|cancel_io_uring_fd\| {
381		const thread: *Thread = .current();
382		thread.enqueue().* = if (thread.io_uring.fd == @intFromEnum(cancel_io_uring_fd)) .{
383		.opcode = .ASYNC_CANCEL,
384		.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
385		.ioprio = 0,
386		.fd = 0,
387		.off = 0,
388		.addr = @intFromPtr(fiber),
389		.len = 0,
390		.rw_flags = 0,
391		.user_data = @intFromEnum(Completion.UserData.wakeup),
392		.buf_index = 0,
393		.personality = 0,
394		.splice_fd_in = 0,
395		.addr3 = 0,
396		.resv = 0,
397		} else .{
398		.opcode = .MSG_RING,
399		.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
400		.ioprio = 0,
401		.fd = @intFromEnum(cancel_io_uring_fd),
402		.off = @intFromPtr(fiber) \| 0b01,
403		.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
404		.len = 0,
405		.rw_flags = 0,
406		.user_data = @intFromEnum(Completion.UserData.cleanup),
407		.buf_index = 0,
408		.personality = 0,
409		.splice_fd_in = 0,
410		.addr3 = 0,
411		.resv = 0,
412		};
413		},
414		}
415		}
416		};
417
418		const CancelRegion = struct {
419		fiber: *Fiber,
420		status: Fiber.CancelStatus,
421		fn init() CancelRegion {
422		const fiber = Thread.current().currentFiber();
423		return .{
424		.fiber = fiber,
425		.status = .{
426		.requested = fiber.cancel_protection.check() == .unblocked,
427		.awaiting = .nothing,
428		},
429		};
430		}
431		fn initBlocked() CancelRegion {
432		return .{
433		.fiber = Thread.current().currentFiber(),
434		.status = .{ .requested = false, .awaiting = .nothing },
435		};
436		}
437		fn deinit(cancel_region: *CancelRegion) void {
438		if (cancel_region.status.requested) _ = cancel_region.fiber.cancel_status.changeAwaiting(
439		cancel_region.status.awaiting,
440		.nothing,
441		);
442		cancel_region.* = undefined;
443		}
444		fn await(cancel_region: *CancelRegion, awaiting: Fiber.CancelStatus.Awaiting) Io.Cancelable!void {
445		if (!cancel_region.status.requested) return;
446		const status: Fiber.CancelStatus = .{ .requested = true, .awaiting = awaiting };
447		if (cancel_region.fiber.cancel_status.changeAwaiting(
448		cancel_region.status.awaiting,
449		status.awaiting,
450		)) {
451		cancel_region.fiber.cancel_protection.acknowledge();
452		cancel_region.status = .unrequested;
453		return error.Canceled;
454		}
455		cancel_region.status = status;
456		}
457		fn awaitIoUring(cancel_region: CancelRegion) Io.Cancelable!Thread {
458		const thread: *Thread = .current();
459		try cancel_region.await(.fromIoUringFd(thread.io_uring.fd));
460		return thread;
461		}
462		fn completion(cancel_region: *const CancelRegion) Completion {
463		return cancel_region.fiber.resultPointer(Completion).*;
464		}
465		fn errno(cancel_region: *const CancelRegion) linux.E {
466		return cancel_region.completion().errno();
467		}
468
469		const Sync = struct {
470		cancel_region: CancelRegion,
471		fn init(ev: *Evented) Io.Cancelable!Sync {
472		if (ev.sync_limit) \|*sync_limit\| try sync_limit.wait(ev.io());
473		return .{ .cancel_region = .init() };
474		}
475		fn initBlocked(ev: *Evented) Sync {
476		if (ev.sync_limit) \|*sync_limit\| sync_limit.waitUncancelable(ev.io());
477		return .{ .cancel_region = .initBlocked() };
478		}
479		fn deinit(sync: Sync, ev: Evented) void {
480		sync.cancel_region.deinit();
481		if (ev.sync_limit) \|*sync_limit\| sync_limit.post(ev.io());
482		}
483
484		const Maybe = union(enum) {
485		cancel_region: CancelRegion,
486		sync: Sync,
487
488		fn deinit(maybe: Maybe, ev: Evented) void {
489		switch (maybe.*) {
490		.cancel_region => \|*cancel_region\| cancel_region.deinit(),
491		.sync => \|*sync\| sync.deinit(ev),
492		}
493		}
494
495		fn enterSync(maybe: Maybe, ev: Evented) Io.Cancelable!*Sync {
496		switch (maybe.*) {
497		.cancel_region => \|cancel_region\| {
498		if (ev.sync_limit) \|*sync_limit\| try sync_limit.wait(ev.io());
499		maybe.* = .{ .sync = .{ .cancel_region = cancel_region } };
500		},
501		.sync => {},
502		}
503		return &maybe.sync;
504		}
505
506		fn leaveSync(maybe: Maybe, ev: Evented) void {
507		switch (maybe.*) {
508		.cancel_region => {},
509		.sync => \|sync\| {
510		if (ev.sync_limit) \|*sync_limit\| sync_limit.post(ev.io());
511		maybe.* = .{ .cancel_region = sync.cancel_region };
512		},
513		}
514		}
515
516		fn cancelRegion(maybe: Maybe) CancelRegion {
517		return switch (maybe.*) {
518		.cancel_region => \|*cancel_region\| cancel_region,
519		.sync => \|*sync\| &sync.cancel_region,
520		};
521		}
522		};
523		};
524		};
525
526		const CachedFd = struct {
527		once: Once,
528
529		const Once = enum(fd_t) {
530		uninitialized = -1,
531		initializing = -2,
532		/// fd
533		_,
534
535		fn fromFd(fd: fd_t) Once {
536		return @enumFromInt(@as(u31, @intCast(fd)));
537		}
538
539		fn toFd(once: Once) fd_t {
540		return @as(u31, @intCast(@intFromEnum(once)));
541		}
542		};
543
544		const init: CachedFd = .{ .once = .uninitialized };
545
546		fn close(cached_fd: *CachedFd) void {
547		switch (cached_fd.once) {
548		.uninitialized => {},
549		.initializing => unreachable,
550		_ => \|fd\| {
551		assert(@intFromEnum(fd) >= 0);
552		_ = std.os.linux.close(@intFromEnum(fd));
553		cached_fd.* = .init;
554		},
555		}
556		}
557
558		fn open(
559		cached_fd: *CachedFd,
560		ev: *Evented,
561		cancel_region: *CancelRegion,
562		path: [*:0]const u8,
563		flags: linux.O,
564		) File.OpenError!fd_t {
565		var once = @atomicLoad(Once, &cached_fd.once, .monotonic);
566		while (true) {
567		switch (once) {
568		.uninitialized => {},
569		.initializing => try futexWait(
570		ev,
571		@ptrCast(&cached_fd.once),
572		@bitCast(@intFromEnum(once)),
573		.none,
574		),
575		_ => \|fd\| {
576		@branchHint(.likely);
577		return fd.toFd();
578		},
579		}
580		once = @cmpxchgWeak(
581		Once,
582		&cached_fd.once,
583		.uninitialized,
584		.initializing,
585		.monotonic,
586		.monotonic,
587		) orelse {
588		errdefer {
589		@atomicStore(Once, &cached_fd.once, .uninitialized, .monotonic);
590		futexWake(ev, @ptrCast(&cached_fd.once), 1);
591		}
592		const fd = try ev.openat(cancel_region, linux.AT.FDCWD, path, flags, 0);
593		@atomicStore(Once, &cached_fd.once, .fromFd(fd), .monotonic);
594		futexWake(ev, @ptrCast(&cached_fd.once), std.math.maxInt(u32));
595		return fd;
596		};
597		}
598		}
599		};
600
601		pub fn allocator(ev: *Evented) std.mem.Allocator {
602		return if (ev.backing_allocator_needs_mutex) .{
603		.ptr = ev,
604		.vtable = &.{
605		.alloc = alloc,
606		.resize = resize,
607		.remap = remap,
608		.free = free,
609		},
610		} else ev.backing_allocator;
611		}
612
613		fn alloc(userdata: anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[]u8 {
614		const ev: *Evented = @ptrCast(@alignCast(userdata));
615		const ev_io = ev.io();
616		ev.backing_allocator_mutex.lockUncancelable(ev_io);
617		defer ev.backing_allocator_mutex.unlock(ev_io);
618		return ev.backing_allocator.rawAlloc(len, alignment, ret_addr);
619		}
620
621		fn resize(
622		userdata: *anyopaque,
623		memory: []u8,
624		alignment: std.mem.Alignment,
625		new_len: usize,
626		ret_addr: usize,
627		) bool {
628		const ev: *Evented = @ptrCast(@alignCast(userdata));
629		const ev_io = ev.io();
630		ev.backing_allocator_mutex.lockUncancelable(ev_io);
631		defer ev.backing_allocator_mutex.unlock(ev_io);
632		return ev.backing_allocator.rawResize(memory, alignment, new_len, ret_addr);
633		}
634
635		fn remap(
636		userdata: *anyopaque,
637		memory: []u8,
638		alignment: Alignment,
639		new_len: usize,
640		ret_addr: usize,
641		) ?[*]u8 {
642		const ev: *Evented = @ptrCast(@alignCast(userdata));
643		const ev_io = ev.io();
644		ev.backing_allocator_mutex.lockUncancelable(ev_io);
645		defer ev.backing_allocator_mutex.unlock(ev_io);
646		return ev.backing_allocator.rawRemap(memory, alignment, new_len, ret_addr);
647		}
648
649		fn free(userdata: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void {
650		const ev: *Evented = @ptrCast(@alignCast(userdata));
651		const ev_io = ev.io();
652		ev.backing_allocator_mutex.lockUncancelable(ev_io);
653		defer ev.backing_allocator_mutex.unlock(ev_io);
654		return ev.backing_allocator.rawFree(memory, alignment, ret_addr);
655		}
656
657		pub fn io(ev: *Evented) Io {
658		return .{
659		.userdata = ev,
660		.vtable = &.{
661		.async = async,
662		.concurrent = concurrent,
663		.await = await,
664		.cancel = cancel,
665
666		.groupAsync = groupAsync,
667		.groupConcurrent = groupConcurrent,
668		.groupAwait = groupAwait,
669		.groupCancel = groupCancel,
670
671		.recancel = recancel,
672		.swapCancelProtection = swapCancelProtection,
673		.checkCancel = checkCancel,
674
675		.select = select,
676
677		.futexWait = futexWait,
678		.futexWaitUncancelable = futexWaitUncancelable,
679		.futexWake = futexWake,
680
681		.operate = operate,
682		.batchAwaitAsync = batchAwaitAsync,
683		.batchAwaitConcurrent = batchAwaitConcurrent,
684		.batchCancel = batchCancel,
685
686		.dirCreateDir = dirCreateDir,
687		.dirCreateDirPath = dirCreateDirPath,
688		.dirCreateDirPathOpen = dirCreateDirPathOpen,
689		.dirOpenDir = dirOpenDir,
690		.dirStat = dirStat,
691		.dirStatFile = dirStatFile,
692		.dirAccess = dirAccess,
693		.dirCreateFile = dirCreateFile,
694		.dirCreateFileAtomic = dirCreateFileAtomic,
695		.dirOpenFile = dirOpenFile,
696		.dirClose = dirClose,
697		.dirRead = dirRead,
698		.dirRealPath = dirRealPath,
699		.dirRealPathFile = dirRealPathFile,
700		.dirDeleteFile = dirDeleteFile,
701		.dirDeleteDir = dirDeleteDir,
702		.dirRename = dirRename,
703		.dirRenamePreserve = dirRenamePreserve,
704		.dirSymLink = dirSymLink,
705		.dirReadLink = dirReadLink,
706		.dirSetOwner = dirSetOwner,
707		.dirSetFileOwner = dirSetFileOwner,
708		.dirSetPermissions = dirSetPermissions,
709		.dirSetFilePermissions = dirSetFilePermissions,
710		.dirSetTimestamps = dirSetTimestamps,
711		.dirHardLink = dirHardLink,
712
713		.fileStat = fileStat,
714		.fileLength = fileLength,
715		.fileClose = fileClose,
716		.fileWritePositional = fileWritePositional,
717		.fileWriteFileStreaming = fileWriteFileStreaming,
718		.fileWriteFilePositional = fileWriteFilePositional,
719		.fileReadPositional = fileReadPositional,
720		.fileSeekBy = fileSeekBy,
721		.fileSeekTo = fileSeekTo,
722		.fileSync = fileSync,
723		.fileIsTty = fileIsTty,
724		.fileEnableAnsiEscapeCodes = fileEnableAnsiEscapeCodes,
725		.fileSupportsAnsiEscapeCodes = fileIsTty,
726		.fileSetLength = fileSetLength,
727		.fileSetOwner = fileSetOwner,
728		.fileSetPermissions = fileSetPermissions,
729		.fileSetTimestamps = fileSetTimestamps,
730		.fileLock = fileLock,
731		.fileTryLock = fileTryLock,
732		.fileUnlock = fileUnlock,
733		.fileDowngradeLock = fileDowngradeLock,
734		.fileRealPath = fileRealPath,
735		.fileHardLink = fileHardLink,
736
737		.fileMemoryMapCreate = fileMemoryMapCreate,
738		.fileMemoryMapDestroy = fileMemoryMapDestroy,
739		.fileMemoryMapSetLength = fileMemoryMapSetLength,
740		.fileMemoryMapRead = fileMemoryMapRead,
741		.fileMemoryMapWrite = fileMemoryMapWrite,
742
743		.processExecutableOpen = processExecutableOpen,
744		.processExecutablePath = processExecutablePath,
745		.lockStderr = lockStderr,
746		.tryLockStderr = tryLockStderr,
747		.unlockStderr = unlockStderr,
748		.processCurrentPath = processCurrentPath,
749		.processSetCurrentDir = processSetCurrentDir,
750		.processReplace = processReplace,
751		.processReplacePath = processReplacePath,
752		.processSpawn = processSpawn,
753		.processSpawnPath = processSpawnPath,
754		.childWait = childWait,
755		.childKill = childKill,
756
757		.progressParentFile = progressParentFile,
758
759		.now = now,
760		.clockResolution = clockResolution,
761		.sleep = sleep,
762
763		.random = random,
764		.randomSecure = randomSecure,
765
766		.netListenIp = netListenIpUnavailable,
767		.netAccept = netAcceptUnavailable,
768		.netBindIp = netBindIp,
769		.netConnectIp = netConnectIpUnavailable,
770		.netListenUnix = netListenUnixUnavailable,
771		.netConnectUnix = netConnectUnixUnavailable,
772		.netSocketCreatePair = netSocketCreatePairUnavailable,
773		.netSend = netSendUnavailable,
774		.netReceive = netReceive,
775		.netRead = netReadUnavailable,
776		.netWrite = netWriteUnavailable,
777		.netWriteFile = netWriteFileUnavailable,
778		.netClose = netClose,
779		.netShutdown = netShutdown,
780		.netInterfaceNameResolve = netInterfaceNameResolveUnavailable,
781		.netInterfaceName = netInterfaceNameUnavailable,
782		.netLookup = netLookupUnavailable,
783		},
784		};
785		}
786
787		fn fileMemoryMapSetLength(
788		userdata: ?*anyopaque,
789		mm: *File.MemoryMap,
790		new_len: usize,
791		) File.MemoryMap.SetLengthError!void {
792		const ev: *Evented = @ptrCast(@alignCast(userdata));
793
794		const page_size = std.heap.pageSize();
795		const alignment: Alignment = .fromByteUnits(page_size);
796		const page_align = std.heap.page_size_min;
797		const old_memory = mm.memory;
798
799		if (alignment.forward(new_len) == alignment.forward(old_memory.len)) {
800		mm.memory.len = new_len;
801		return;
802		}
803		const flags: linux.MREMAP = .{ .MAYMOVE = true };
804		const addr_hint: ?[*]const u8 = null;
805		var sync: CancelRegion.Sync = try .init(ev);
806		defer sync.deinit(ev);
807		const new_memory = while (true) {
808		try sync.cancel_region.await(.nothing);
809		const rc = linux.mremap(old_memory.ptr, old_memory.len, new_len, flags, addr_hint);
810		switch (linux.errno(rc)) {
811		.SUCCESS => break @as([*]align(page_align) u8, @ptrFromInt(rc))[0..new_len],
812		.INTR => continue,
813		.AGAIN => return error.LockedMemoryLimitExceeded,
814		.NOMEM => return error.OutOfMemory,
815		.INVAL => \|err\| return errnoBug(err),
816		.FAULT => \|err\| return errnoBug(err),
817		else => \|err\| return unexpectedErrno(err),
818		}
819		};
820		mm.memory = new_memory;
821		}
822
823		fn fileMemoryMapRead(userdata: ?anyopaque, mm: File.MemoryMap) File.ReadPositionalError!void {
824		const ev: *Evented = @ptrCast(@alignCast(userdata));
825		_ = ev;
826		_ = mm;
827		}
828
829		fn fileMemoryMapWrite(userdata: ?anyopaque, mm: File.MemoryMap) File.WritePositionalError!void {
830		const ev: *Evented = @ptrCast(@alignCast(userdata));
831		_ = ev;
832		_ = mm;
833		}
834
835		pub const InitOptions = struct {
836		backing_allocator_needs_mutex: bool = true,
837
838		/// Maximum thread pool size (excluding the main thread).
839		/// Defaults to one less than the number of logical CPU cores.
840		thread_limit: ?usize = null,
841		/// Maximum number of threads that may perform synchronous syscalls.
842		sync_limit: Io.Limit = .unlimited,
843
844		log2_ring_entries: u4 = 3,
845
846		/// Affects the following operations:
847		/// * `processExecutablePath` on OpenBSD and Haiku.
848		argv0: Argv0 = .empty,
849		/// Affects the following operations:
850		/// * `fileIsTty`
851		/// * `processSpawn`, `processSpawnPath`, `processReplace`, `processReplacePath`
852		environ: process.Environ = .empty,
853		};
854
855		pub fn init(ev: *Evented, backing_allocator: Allocator, options: InitOptions) !void {
856		const threads_size = @sizeOf(Thread) * if (options.thread_limit) \|thread_limit\|
857		1 + thread_limit
858		else
859		@max(std.Thread.getCpuCount() catch 1, 1);
860		const idle_stack_end_offset =
861		std.mem.alignForward(usize, threads_size + idle_stack_size, std.heap.page_size_max);
862		const allocated_slice = try backing_allocator.alignedAlloc(u8, .of(Thread), idle_stack_end_offset);
863		errdefer backing_allocator.free(allocated_slice);
864		ev.* = .{
865		.backing_allocator_needs_mutex = options.backing_allocator_needs_mutex,
866		.backing_allocator_mutex = .init,
867		.backing_allocator = backing_allocator,
868		.main_fiber_buffer = undefined,
869		.log2_ring_entries = options.log2_ring_entries,
870		.threads = .{
871		.allocated = @ptrCast(allocated_slice[0..threads_size]),
872		.reserved = 1,
873		.active = 1,
874		},
875		.sync_limit = if (options.sync_limit.toInt()) \|sync_limit\| .{ .permits = sync_limit } else null,
876
877		.stderr_mutex = .init,
878		.stderr_writer = .{
879		.io = ev.io(),
880		.interface = Io.File.Writer.initInterface(&.{}),
881		.file = .stderr(),
882		.mode = .streaming,
883		},
884		.stderr_mode = .no_color,
885		.stderr_writer_initialized = false,
886
887		.environ_mutex = .init,
888		.environ = .{ .process_environ = options.environ },
889
890		.null_fd = .init,
891		.random_fd = .init,
892
893		.csprng_mutex = .init,
894		.csprng = .uninitialized,
895		};
896		const main_fiber: *Fiber = @ptrCast(&ev.main_fiber_buffer);
897		main_fiber.* = .{
898		.required_align = {},
899		.context = undefined,
900		.await_count = 0,
901		.link = .{ .awaiter = null },
902		.status = .{ .queue_next = null },
903		.cancel_status = .unrequested,
904		.cancel_protection = .unblocked,
905		.name = if (tracy.enable) "main task",
906		};
907		const main_thread = &ev.threads.allocated[0];
908		Thread.self = main_thread;
909		const idle_stack_end: [*]align(16) usize =
910		@ptrCast(@alignCast(allocated_slice[idle_stack_end_offset..].ptr));
911		(idle_stack_end - 1)[0..1].* = .{@intFromPtr(ev)};
912		main_thread.* = .{
913		.required_align = {},
914		.thread = undefined,
915		.idle_context = switch (builtin.cpu.arch) {
916		.aarch64 => .{
917		.sp = @intFromPtr(idle_stack_end),
918		.fp = 0,
919		.pc = @intFromPtr(&mainIdleEntry),
920		},
921		.x86_64 => .{
922		.rsp = @intFromPtr(idle_stack_end - 1),
923		.rbp = 0,
924		.rip = @intFromPtr(&mainIdleEntry),
925		},
926		else => @compileError("unimplemented architecture"),
927		},
928		.current_context = &main_fiber.context,
929		.ready_queue = null,
930		.free_queue = null,
931		.io_uring = try .init(
932		@as(u16, 1) << ev.log2_ring_entries,
933		linux.IORING_SETUP_COOP_TASKRUN \| linux.IORING_SETUP_SINGLE_ISSUER,
934		),
935		.idle_search_index = 1,
936		.steal_ready_search_index = 1,
937		.steal_free_search_index = 1,
938		.name_arena = .{},
939		.csprng = .uninitialized,
940		};
941		errdefer main_thread.io_uring.deinit();
942		if (tracy.enable) tracy.fiberEnter(main_fiber.name);
943		}
944
945		pub fn deinit(ev: *Evented) void {
946		const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
947		for (ev.threads.allocated[0..active_threads]) \|*thread\| {
948		const ready_fiber = @atomicLoad(?*Fiber, &thread.ready_queue, .monotonic);
949		assert(ready_fiber == null or ready_fiber == Fiber.finished); // pending async
950		}
951		ev.yield(null, .exit);
952		ev.threads.allocated[0].deinit(ev.allocator());
953		ev.null_fd.close();
954		ev.random_fd.close();
955		const allocated_ptr: [*]align(@alignOf(Thread)) u8 = @ptrCast(@alignCast(ev.threads.allocated.ptr));
956		const idle_stack_end_offset = std.mem.alignForward(
957		usize,
958		ev.threads.allocated.len * @sizeOf(Thread) + idle_stack_size,
959		std.heap.page_size_max,
960		);
961		for (ev.threads.allocated[1..active_threads]) \|*thread\| thread.thread.join();
962		assert(active_threads == ev.threads.active); // spawned threads while there was no pending async?
963		ev.backing_allocator.free(allocated_ptr[0..idle_stack_end_offset]);
964		ev.* = undefined;
965		}
966
967		fn findReadyFiber(ev: Evented, thread: Thread) ?*Fiber {
968		if (@atomicRmw(?*Fiber, &thread.ready_queue, .Xchg, Fiber.finished, .acquire)) \|ready_fiber\| {
969		assert(ready_fiber != Fiber.finished);
970		@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.status.queue_next, .release);
971		ready_fiber.status.queue_next = null;
972		return ready_fiber;
973		}
974		const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
975		for (0..@min(max_steal_ready_search, active_threads)) \|_\| {
976		defer thread.steal_ready_search_index += 1;
977		if (thread.steal_ready_search_index == active_threads) thread.steal_ready_search_index = 0;
978		const steal_ready_search_thread =
979		&ev.threads.allocated[0..active_threads][thread.steal_ready_search_index];
980		if (steal_ready_search_thread == thread) continue;
981		const ready_fiber =
982		@atomicLoad(?*Fiber, &steal_ready_search_thread.ready_queue, .monotonic) orelse continue;
983		if (ready_fiber == Fiber.finished) continue;
984		if (@cmpxchgWeak(
985		?*Fiber,
986		&steal_ready_search_thread.ready_queue,
987		ready_fiber,
988		null,
989		.acquire,
990		.monotonic,
991		)) \|_\| continue;
992		@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.status.queue_next, .release);
993		ready_fiber.status.queue_next = null;
994		return ready_fiber;
995		}
996		// couldn't find anything to do, so we are now open for business
997		@atomicStore(?*Fiber, &thread.ready_queue, null, .monotonic);
998		return null;
999		}
1000
1001		fn yield(ev: Evented, maybe_ready_fiber: ?Fiber, pending_task: SwitchMessage.PendingTask) void {
1002		const thread: *Thread = .current();
1003		const ready_context = if (maybe_ready_fiber orelse ev.findReadyFiber(thread)) \|ready_fiber\|
1004		&ready_fiber.context
1005		else
1006		&thread.idle_context;
1007		const message: SwitchMessage = .{
1008		.contexts = .{
1009		.prev = thread.current_context,
1010		.ready = ready_context,
1011		},
1012		.pending_task = pending_task,
1013		};
1014		contextSwitch(&message).handle(ev);
1015		}
1016
1017		fn schedule(ev: Evented, thread: Thread, ready_queue: Fiber.Queue) bool {
1018		// shared fields of previous `Thread` must be initialized before later ones are marked as active
1019		const new_thread_index = @atomicLoad(u32, &ev.threads.active, .acquire);
1020		for (0..@min(max_idle_search, new_thread_index)) \|_\| {
1021		defer thread.idle_search_index += 1;
1022		if (thread.idle_search_index == new_thread_index) thread.idle_search_index = 0;
1023		const idle_search_thread = &ev.threads.allocated[0..new_thread_index][thread.idle_search_index];
1024		if (idle_search_thread == thread) continue;
1025		if (@cmpxchgWeak(
1026		?*Fiber,
1027		&idle_search_thread.ready_queue,
1028		null,
1029		ready_queue.head,
1030		.release,
1031		.monotonic,
1032		)) \|_\| continue;
1033		thread.enqueue().* = .{
1034		.opcode = .MSG_RING,
1035		.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1036		.ioprio = 0,
1037		.fd = idle_search_thread.io_uring.fd,
1038		.off = @intFromEnum(Completion.UserData.wakeup),
1039		.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
1040		.len = 0,
1041		.rw_flags = 0,
1042		.user_data = @intFromEnum(Completion.UserData.wakeup),
1043		.buf_index = 0,
1044		.personality = 0,
1045		.splice_fd_in = 0,
1046		.addr3 = 0,
1047		.resv = 0,
1048		};
1049		return true;
1050		}
1051		spawn_thread: {
1052		// previous failed reservations must have completed before retrying
1053		if (new_thread_index == ev.threads.allocated.len or @cmpxchgWeak(
1054		u32,
1055		&ev.threads.reserved,
1056		new_thread_index,
1057		new_thread_index + 1,
1058		.acquire,
1059		.monotonic,
1060		) != null) break :spawn_thread;
1061		const new_thread = &ev.threads.allocated[new_thread_index];
1062		const next_thread_index = new_thread_index + 1;
1063		var params = std.mem.zeroInit(linux.io_uring_params, .{
1064		.flags = linux.IORING_SETUP_ATTACH_WQ \|
1065		linux.IORING_SETUP_R_DISABLED \|
1066		linux.IORING_SETUP_COOP_TASKRUN \|
1067		linux.IORING_SETUP_SINGLE_ISSUER,
1068		.wq_fd = @as(u32, @intCast(ev.threads.allocated[0].io_uring.fd)),
1069		});
1070		new_thread.* = .{
1071		.required_align = {},
1072		.thread = undefined,
1073		.idle_context = undefined,
1074		.current_context = &new_thread.idle_context,
1075		.ready_queue = ready_queue.head,
1076		.free_queue = null,
1077		.io_uring = IoUring.init_params(@as(u16, 1) << ev.log2_ring_entries, &params) catch \|err\| {
1078		@atomicStore(u32, &ev.threads.reserved, new_thread_index, .release);
1079		// no more access to `thread` after giving up reservation
1080		log.warn("unable to create worker thread due to io_uring init failure: {s}", .{
1081		@errorName(err),
1082		});
1083		break :spawn_thread;
1084		},
1085		.idle_search_index = 0,
1086		.steal_ready_search_index = 0,
1087		.steal_free_search_index = 0,
1088		.name_arena = .{},
1089		.csprng = .uninitialized,
1090		};
1091		new_thread.thread = std.Thread.spawn(.{
1092		.stack_size = idle_stack_size,
1093		.allocator = ev.allocator(),
1094		}, threadEntry, .{ ev, new_thread_index }) catch \|err\| {
1095		new_thread.io_uring.deinit();
1096		@atomicStore(u32, &ev.threads.reserved, new_thread_index, .release);
1097		// no more access to `thread` after giving up reservation
1098		log.warn("unable to create worker thread due spawn failure: {s}", .{@errorName(err)});
1099		break :spawn_thread;
1100		};
1101		// shared fields of `Thread` must be initialized before being marked active
1102		@atomicStore(u32, &ev.threads.active, next_thread_index, .release);
1103		return false;
1104		}
1105		// nobody wanted it, so just queue it on ourselves
1106		while (true) ready_queue.tail.status.queue_next = @cmpxchgWeak(
1107		?*Fiber,
1108		&thread.ready_queue,
1109		ready_queue.tail.status.queue_next,
1110		ready_queue.head,
1111		.acq_rel,
1112		.acquire,
1113		) orelse break;
1114		return false;
1115		}
1116
1117		fn mainIdle(
1118		ev: *Evented,
1119		message: *const SwitchMessage,
1120		) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Context)))) noreturn {
1121		message.handle(ev);
1122		ev.idle(&ev.threads.allocated[0]);
1123		ev.yield(@ptrCast(&ev.main_fiber_buffer), .nothing);
1124		unreachable; // switched to dead fiber
1125		}
1126
1127		fn threadEntry(ev: *Evented, index: u32) void {
1128		const thread: *Thread = &ev.threads.allocated[index];
1129		Thread.self = thread;
1130		defer thread.deinit(ev.allocator());
1131		switch (linux.errno(linux.io_uring_register(thread.io_uring.fd, .REGISTER_ENABLE_RINGS, null, 0))) {
1132		.SUCCESS => ev.idle(thread),
1133		else => \|err\| @panic(@tagName(err)),
1134		}
1135		}
1136
1137		const Completion = struct {
1138		result: i32,
1139		flags: u32,
1140
1141		const UserData = enum(usize) {
1142		unused,
1143		wakeup,
1144		futex_wake,
1145		cleanup,
1146		exit,
1147		/// If bit 0 is 1, a pointer to the `context` field of `Io.Batch.Storage.Pending`.
1148		/// If bits 0 and 1 are 0, a `*Fiber`.
1149		_,
1150		};
1151
1152		fn errno(completion: Completion) linux.E {
1153		return linux.errno(@bitCast(@as(isize, completion.result)));
1154		}
1155		};
1156
1157		fn idle(ev: Evented, thread: Thread) void {
1158		var maybe_ready_fiber: ?*Fiber = null;
1159		while (true) {
1160		while (maybe_ready_fiber orelse ev.findReadyFiber(thread)) \|ready_fiber\| {
1161		ev.yield(ready_fiber, .nothing);
1162		maybe_ready_fiber = null;
1163		}
1164		_ = thread.io_uring.submit_and_wait(1) catch \|err\| switch (err) {
1165		error.SignalInterrupt => {},
1166		else => \|e\| @panic(@errorName(e)),
1167		};
1168		var maybe_ready_queue: ?Fiber.Queue = null;
1169		while (true) {
1170		var cqes_buffer: [1 << 8]linux.io_uring_cqe = undefined;
1171		const cqes = cqes_buffer[0 .. thread.io_uring.copy_cqes(&cqes_buffer, 0) catch \|err\| switch (err) {
1172		error.SignalInterrupt => 0,
1173		else => \|e\| @panic(@errorName(e)),
1174		}];
1175		if (cqes.len == 0) break;
1176		for (cqes) \|cqe\| if (cqe.flags & linux.IORING_CQE_F_SKIP == 0) switch (@as(
1177		Completion.UserData,
1178		@enumFromInt(cqe.user_data),
1179		)) {
1180		.unused => unreachable, // bad submission queued?
1181		.wakeup => {},
1182		.futex_wake => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
1183		.SUCCESS => recoverableOsBugDetected(), // success is skipped
1184		.INVAL => {}, // invalid futex_wait() on ptr done elsewhere
1185		.INTR, .CANCELED => recoverableOsBugDetected(), // `Completion.UserData.futex_wake` is not cancelable
1186		.FAULT => {}, // pointer became invalid while doing the wake
1187		else => recoverableOsBugDetected(), // deadlock due to operating system bug
1188		},
1189		.cleanup => @panic("failed to notify other threads that we are exiting"),
1190		.exit => {
1191		assert(maybe_ready_fiber == null and maybe_ready_queue == null); // pending async
1192		return;
1193		},
1194		_ => if (@as(?*Fiber, ready_fiber: switch (@as(u2, @truncate(cqe.user_data))) {
1195		0b00 => {
1196		const ready_fiber: *Fiber = @ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
1197		ready_fiber.resultPointer(Completion).* = .{
1198		.result = cqe.res,
1199		.flags = cqe.flags,
1200		};
1201		break :ready_fiber ready_fiber;
1202		},
1203		0b01 => {
1204		thread.enqueue().* = .{
1205		.opcode = .ASYNC_CANCEL,
1206		.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1207		.ioprio = 0,
1208		.fd = 0,
1209		.off = 0,
1210		.addr = cqe.user_data & ~@as(usize, 0b11),
1211		.len = 0,
1212		.rw_flags = 0,
1213		.user_data = @intFromEnum(Completion.UserData.wakeup),
1214		.buf_index = 0,
1215		.personality = 0,
1216		.splice_fd_in = 0,
1217		.addr3 = 0,
1218		.resv = 0,
1219		};
1220		break :ready_fiber null;
1221		},
1222		0b10 => {
1223		const context: *Io.Operation.Storage.Pending.Context =
1224		@ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
1225		const batch: *Io.Batch = @ptrFromInt(context[0]);
1226		var next: usize = 0b00;
1227		context[0..3].* = .{ next, @as(u32, @bitCast(cqe.res)), cqe.flags };
1228		while (true) {
1229		next = @cmpxchgWeak(
1230		usize,
1231		@as(*usize, @ptrCast(&batch.context)),
1232		next,
1233		cqe.user_data,
1234		.release,
1235		.acquire,
1236		) orelse break;
1237		context[0] = next;
1238		}
1239		break :ready_fiber switch (@as(u2, @truncate(next))) {
1240		0b00, 0b01 => @ptrFromInt(next & ~@as(usize, 0b11)),
1241		0b10, 0b11 => null,
1242		};
1243		},
1244		0b11 => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
1245		.SUCCESS => unreachable, // no event count specified
1246		.TIME => {
1247		const context: *usize = @ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
1248		const fiber = @atomicRmw(usize, context, .Add, 0b01, .acquire);
1249		break :ready_fiber switch (@as(u2, @truncate(fiber))) {
1250		else => unreachable, // timeout completed multiple times
1251		0b00 => @ptrFromInt(fiber & ~@as(usize, 0b11)),
1252		0b10 => null,
1253		};
1254		},
1255		.CANCELED => null, // user data may have been invalidated
1256		else => \|err\| unexpectedErrno(err) catch null,
1257		},
1258		})) \|ready_fiber\| {
1259		assert(ready_fiber.status.queue_next == null);
1260		if (maybe_ready_fiber == null) {
1261		maybe_ready_fiber = ready_fiber;
1262		} else if (maybe_ready_queue) \|*ready_queue\| {
1263		ready_queue.tail.status.queue_next = ready_fiber;
1264		ready_queue.tail = ready_fiber;
1265		} else maybe_ready_queue = .{ .head = ready_fiber, .tail = ready_fiber };
1266		},
1267		};
1268		}
1269		if (maybe_ready_queue) \|ready_queue\| _ = ev.schedule(thread, ready_queue);
1270		}
1271		}
1272
1273		const SwitchMessage = struct {
1274		contexts: extern struct {
1275		prev: *Context,
1276		ready: *Context,
1277		},
1278		pending_task: PendingTask,
1279
1280		const PendingTask = union(enum) {
1281		nothing,
1282		reschedule,
1283		await: u31,
1284		group_await: Group,
1285		group_cancel: Group,
1286		batch_await: *Io.Batch,
1287		destroy,
1288		exit,
1289		};
1290
1291		fn handle(message: const SwitchMessage, ev: Evented) void {
1292		const thread: *Thread = .current();
1293		thread.current_context = message.contexts.ready;
1294		if (tracy.enable) {
1295		if (message.contexts.ready != &thread.idle_context) {
1296		const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.ready));
1297		tracy.fiberEnter(fiber.name);
1298		} else tracy.fiberLeave();
1299		}
1300		switch (message.pending_task) {
1301		.nothing => {},
1302		.reschedule => if (message.contexts.prev != &thread.idle_context) {
1303		const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
1304		assert(fiber.status.queue_next == null);
1305		_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1306		},
1307		.await => \|count\| {
1308		const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
1309		if (@atomicRmw(i32, &fiber.await_count, .Sub, count, .monotonic) > 0)
1310		_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1311		},
1312		.group_await => \|group\| {
1313		const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
1314		if (group.await(ev, fiber))
1315		_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1316		},
1317		.group_cancel => \|group\| {
1318		const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
1319		if (group.cancel(ev, fiber))
1320		_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1321		},
1322		.batch_await => \|batch\| {
1323		const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
1324		if (@cmpxchgStrong(
1325		?*anyopaque,
1326		&batch.context,
1327		null,
1328		fiber,
1329		.release,
1330		.monotonic,
1331		)) \|head\| {
1332		assert(@as(u2, @truncate(@intFromPtr(head))) != 0b00);
1333		_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1334		}
1335		},
1336		.destroy => {
1337		const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
1338		fiber.destroy();
1339		},
1340		.exit => for (
1341		ev.threads.allocated[0..@atomicLoad(u32, &ev.threads.active, .acquire)],
1342		) \|*each_thread\| {
1343		thread.enqueue().* = .{
1344		.opcode = .MSG_RING,
1345		.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1346		.ioprio = 0,
1347		.fd = each_thread.io_uring.fd,
1348		.off = @intFromEnum(Completion.UserData.exit),
1349		.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
1350		.len = 0,
1351		.rw_flags = 0,
1352		.user_data = @intFromEnum(Completion.UserData.cleanup),
1353		.buf_index = 0,
1354		.personality = 0,
1355		.splice_fd_in = 0,
1356		.addr3 = 0,
1357		.resv = 0,
1358		};
1359		},
1360		}
1361		}
1362		};
1363
1364		const Context = switch (builtin.cpu.arch) {
1365		.aarch64 => extern struct {
1366		sp: u64,
1367		fp: u64,
1368		pc: u64,
1369		},
1370		.x86_64 => extern struct {
1371		rsp: u64,
1372		rbp: u64,
1373		rip: u64,
1374		},
1375		else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1376		};
1377
1378		inline fn contextSwitch(message: const SwitchMessage) const SwitchMessage {
1379		return @fieldParentPtr("contexts", switch (builtin.cpu.arch) {
1380		.aarch64 => asm volatile (
1381		\\ ldp x0, x2, [x1]
1382		\\ ldr x3, [x2, #16]
1383		\\ mov x4, sp
1384		\\ stp x4, fp, [x0]
1385		\\ adr x5, 0f
1386		\\ ldp x4, fp, [x2]
1387		\\ str x5, [x0, #16]
1388		\\ mov sp, x4
1389		\\ br x3
1390		\\0:
1391		: [received_message] "={x1}" (-> *const @FieldType(SwitchMessage, "contexts")),
1392		: [message_to_send] "{x1}" (&message.contexts),
1393		: .{
1394		.x0 = true,
1395		.x1 = true,
1396		.x2 = true,
1397		.x3 = true,
1398		.x4 = true,
1399		.x5 = true,
1400		.x6 = true,
1401		.x7 = true,
1402		.x8 = true,
1403		.x9 = true,
1404		.x10 = true,
1405		.x11 = true,
1406		.x12 = true,
1407		.x13 = true,
1408		.x14 = true,
1409		.x15 = true,
1410		.x16 = true,
1411		.x17 = true,
1412		.x19 = true,
1413		.x20 = true,
1414		.x21 = true,
1415		.x22 = true,
1416		.x23 = true,
1417		.x24 = true,
1418		.x25 = true,
1419		.x26 = true,
1420		.x27 = true,
1421		.x28 = true,
1422		.x30 = true,
1423		.z0 = true,
1424		.z1 = true,
1425		.z2 = true,
1426		.z3 = true,
1427		.z4 = true,
1428		.z5 = true,
1429		.z6 = true,
1430		.z7 = true,
1431		.z8 = true,
1432		.z9 = true,
1433		.z10 = true,
1434		.z11 = true,
1435		.z12 = true,
1436		.z13 = true,
1437		.z14 = true,
1438		.z15 = true,
1439		.z16 = true,
1440		.z17 = true,
1441		.z18 = true,
1442		.z19 = true,
1443		.z20 = true,
1444		.z21 = true,
1445		.z22 = true,
1446		.z23 = true,
1447		.z24 = true,
1448		.z25 = true,
1449		.z26 = true,
1450		.z27 = true,
1451		.z28 = true,
1452		.z29 = true,
1453		.z30 = true,
1454		.z31 = true,
1455		.p0 = true,
1456		.p1 = true,
1457		.p2 = true,
1458		.p3 = true,
1459		.p4 = true,
1460		.p5 = true,
1461		.p6 = true,
1462		.p7 = true,
1463		.p8 = true,
1464		.p9 = true,
1465		.p10 = true,
1466		.p11 = true,
1467		.p12 = true,
1468		.p13 = true,
1469		.p14 = true,
1470		.p15 = true,
1471		.fpcr = true,
1472		.fpsr = true,
1473		.ffr = true,
1474		.memory = true,
1475		}),
1476		.x86_64 => asm volatile (
1477		\\ movq 0(%%rsi), %%rax
1478		\\ movq 8(%%rsi), %%rcx
1479		\\ leaq 0f(%%rip), %%rdx
1480		\\ movq %%rsp, 0(%%rax)
1481		\\ movq %%rbp, 8(%%rax)
1482		\\ movq %%rdx, 16(%%rax)
1483		\\ movq 0(%%rcx), %%rsp
1484		\\ movq 8(%%rcx), %%rbp
1485		\\ jmpq *16(%%rcx)
1486		\\0:
1487		: [received_message] "={rsi}" (-> *const @FieldType(SwitchMessage, "contexts")),
1488		: [message_to_send] "{rsi}" (&message.contexts),
1489		: .{
1490		.rax = true,
1491		.rcx = true,
1492		.rdx = true,
1493		.rbx = true,
1494		.rsi = true,
1495		.rdi = true,
1496		.r8 = true,
1497		.r9 = true,
1498		.r10 = true,
1499		.r11 = true,
1500		.r12 = true,
1501		.r13 = true,
1502		.r14 = true,
1503		.r15 = true,
1504		.mm0 = true,
1505		.mm1 = true,
1506		.mm2 = true,
1507		.mm3 = true,
1508		.mm4 = true,
1509		.mm5 = true,
1510		.mm6 = true,
1511		.mm7 = true,
1512		.zmm0 = true,
1513		.zmm1 = true,
1514		.zmm2 = true,
1515		.zmm3 = true,
1516		.zmm4 = true,
1517		.zmm5 = true,
1518		.zmm6 = true,
1519		.zmm7 = true,
1520		.zmm8 = true,
1521		.zmm9 = true,
1522		.zmm10 = true,
1523		.zmm11 = true,
1524		.zmm12 = true,
1525		.zmm13 = true,
1526		.zmm14 = true,
1527		.zmm15 = true,
1528		.zmm16 = true,
1529		.zmm17 = true,
1530		.zmm18 = true,
1531		.zmm19 = true,
1532		.zmm20 = true,
1533		.zmm21 = true,
1534		.zmm22 = true,
1535		.zmm23 = true,
1536		.zmm24 = true,
1537		.zmm25 = true,
1538		.zmm26 = true,
1539		.zmm27 = true,
1540		.zmm28 = true,
1541		.zmm29 = true,
1542		.zmm30 = true,
1543		.zmm31 = true,
1544		.fpsr = true,
1545		.fpcr = true,
1546		.mxcsr = true,
1547		.rflags = true,
1548		.dirflag = true,
1549		.memory = true,
1550		}),
1551		else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1552		});
1553		}
1554
1555		fn mainIdleEntry() callconv(.naked) void {
1556		switch (builtin.cpu.arch) {
1557		.aarch64 => asm volatile (
1558		\\ ldr x0, [sp, #-8]
1559		\\ b %[mainIdle]
1560		:
1561		: [mainIdle] "X" (&mainIdle),
1562		),
1563		.x86_64 => asm volatile (
1564		\\ movq (%%rsp), %%rdi
1565		\\ jmp %[mainIdle:P]
1566		:
1567		: [mainIdle] "X" (&mainIdle),
1568		),
1569		else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1570		}
1571		}
1572
1573		const AsyncClosure = struct {
1574		ev: *Evented,
1575		fiber: *Fiber,
1576		start: const fn (context: const anyopaque, result: *anyopaque) void,
1577		result_align: Alignment,
1578
1579		fn fromFiber(fiber: Fiber) AsyncClosure {
1580		return @ptrFromInt(Fiber.max_context_align.max(.of(AsyncClosure)).backward(
1581		@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
1582		) - @sizeOf(AsyncClosure));
1583		}
1584
1585		fn contextPointer(closure: AsyncClosure) []align(Fiber.max_context_align.toByteUnits()) u8 {
1586		return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(AsyncClosure));
1587		}
1588
1589		fn entry() callconv(.naked) void {
1590		switch (builtin.cpu.arch) {
1591		.aarch64 => asm volatile (
1592		\\ mov x0, sp
1593		\\ b %[call]
1594		:
1595		: [call] "X" (&call),
1596		),
1597		.x86_64 => asm volatile (
1598		\\ leaq 8(%%rsp), %%rdi
1599		\\ jmp %[call:P]
1600		:
1601		: [call] "X" (&call),
1602		),
1603		else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1604		}
1605		}
1606
1607		fn call(
1608		closure: *AsyncClosure,
1609		message: *const SwitchMessage,
1610		) callconv(.withStackAlign(.c, @alignOf(AsyncClosure))) noreturn {
1611		message.handle(closure.ev);
1612		const fiber = closure.fiber;
1613		closure.start(closure.contextPointer(), fiber.resultBytes(closure.result_align));
1614		closure.ev.yield(
1615		if (@atomicRmw(?*Fiber, &fiber.link.awaiter, .Xchg, Fiber.finished, .acq_rel)) \|awaiter\|
1616		if (@atomicRmw(i32, &awaiter.await_count, .Add, 1, .monotonic) == -1) awaiter else null
1617		else
1618		null,
1619		.nothing,
1620		);
1621		unreachable; // switched to dead fiber
1622		}
1623		};
1624
1625		fn async(
1626		userdata: ?*anyopaque,
1627		result: []u8,
1628		result_alignment: Alignment,
1629		context: []const u8,
1630		context_alignment: Alignment,
1631		start: const fn (context: const anyopaque, result: *anyopaque) void,
1632		) ?*std.Io.AnyFuture {
1633		const ev: *Evented = @ptrCast(@alignCast(userdata));
1634		return concurrent(ev, result.len, result_alignment, context, context_alignment, start) catch {
1635		start(context.ptr, result.ptr);
1636		return null;
1637		};
1638		}
1639
1640		fn concurrent(
1641		userdata: ?*anyopaque,
1642		result_len: usize,
1643		result_alignment: Alignment,
1644		context: []const u8,
1645		context_alignment: Alignment,
1646		start: const fn (context: const anyopaque, result: *anyopaque) void,
1647		) Io.ConcurrentError!*std.Io.AnyFuture {
1648		assert(result_alignment.compare(.lte, Fiber.max_result_align)); // TODO
1649		assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
1650		assert(result_len <= Fiber.max_result_size); // TODO
1651		assert(context.len <= Fiber.max_context_size); // TODO
1652
1653		const ev: *Evented = @ptrCast(@alignCast(userdata));
1654		const fiber = Fiber.create(ev) catch \|err\| switch (err) {
1655		error.OutOfMemory => return error.ConcurrencyUnavailable,
1656		};
1657
1658		const closure: *AsyncClosure = .fromFiber(fiber);
1659		fiber.* = .{
1660		.required_align = {},
1661		.context = switch (builtin.cpu.arch) {
1662		.aarch64 => .{
1663		.sp = @intFromPtr(closure),
1664		.fp = 0,
1665		.pc = @intFromPtr(&AsyncClosure.entry),
1666		},
1667		.x86_64 => .{
1668		.rsp = @intFromPtr(closure) - @sizeOf(usize),
1669		.rbp = 0,
1670		.rip = @intFromPtr(&AsyncClosure.entry),
1671		},
1672		else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1673		},
1674		.await_count = 0,
1675		.link = .{ .awaiter = null },
1676		.status = .{ .queue_next = null },
1677		.cancel_status = .unrequested,
1678		.cancel_protection = .unblocked,
1679		.name = if (tracy.enable) name: {
1680		const thread: *Thread = .current();
1681		var name_arena = thread.name_arena.promote(std.heap.page_allocator);
1682		defer thread.name_arena = name_arena.state;
1683		break :name std.fmt.allocPrintSentinel(
1684		name_arena.allocator(),
1685		"task {d}",
1686		.{@atomicRmw(u64, &Fiber.next_name, .Add, 1, .monotonic)},
1687		0,
1688		) catch return error.ConcurrencyUnavailable;
1689		},
1690		};
1691		closure.* = .{
1692		.ev = ev,
1693		.fiber = fiber,
1694		.start = start,
1695		.result_align = result_alignment,
1696		};
1697		@memcpy(closure.contextPointer(), context);
1698
1699		const thread: *Thread = .current();
1700		if (ev.schedule(thread, .{ .head = fiber, .tail = fiber })) thread.submit();
1701		return @ptrCast(fiber);
1702		}
1703
1704		fn await(
1705		userdata: ?*anyopaque,
1706		future: *std.Io.AnyFuture,
1707		result: []u8,
1708		result_alignment: Alignment,
1709		) void {
1710		const ev: *Evented = @ptrCast(@alignCast(userdata));
1711		const fiber = Thread.current().currentFiber();
1712		const future_fiber: *Fiber = @ptrCast(@alignCast(future));
1713		if (@atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, fiber, .acq_rel)) \|awaiter\| {
1714		assert(awaiter == Fiber.finished);
1715		} else while (true) {
1716		ev.yield(null, .{ .await = 1 });
1717		const awaiter = @atomicLoad(?*Fiber, &future_fiber.link.awaiter, .acquire);
1718		if (awaiter == Fiber.finished) break;
1719		assert(awaiter == fiber); // spurious wakeup
1720		}
1721		@memcpy(result, future_fiber.resultBytes(result_alignment));
1722		future_fiber.destroy();
1723		}
1724
1725		fn cancel(
1726		userdata: ?*anyopaque,
1727		future: *std.Io.AnyFuture,
1728		result: []u8,
1729		result_alignment: Alignment,
1730		) void {
1731		const ev: *Evented = @ptrCast(@alignCast(userdata));
1732		const future_fiber: *Fiber = @ptrCast(@alignCast(future));
1733		future_fiber.requestCancel(ev);
1734		await(ev, future, result, result_alignment);
1735		}
1736
1737		const Group = struct {
1738		ptr: *Io.Group,
1739
1740		const List = packed struct(usize) {
1741		cancel_requested: bool,
1742		awaiter_delayed: bool,
1743		fibers: Fiber.PackedPtr,
1744		};
1745		fn listPtr(group: Group) *List {
1746		return @ptrCast(&group.ptr.token);
1747		}
1748
1749		const Mutex = packed struct(u32) {
1750		locked: bool,
1751		contended: bool,
1752		shared2: u30,
1753		};
1754		fn mutexPtr(group: Group) *Mutex {
1755		return switch (comptime builtin.cpu.arch.endian()) {
1756		.little => @ptrCast(&group.ptr.state),
1757		.big => @ptrCast(@alignCast(
1758		@as([*]u8, @ptrCast(&group.ptr.state)) + @sizeOf(usize) - @sizeOf(u32),
1759		)),
1760		};
1761		}
1762
1763		const Awaiter = packed struct(usize) {
1764		locked: bool,
1765		contended: bool,
1766		awaiter: Fiber.PackedPtr,
1767		};
1768		fn awaiterPtr(group: Group) *Awaiter {
1769		return @ptrCast(&group.ptr.state);
1770		}
1771
1772		fn lock(group: Group, ev: *Evented) void {
1773		const mutex = group.mutexPtr();
1774		{
1775		const old_state = @atomicRmw(
1776		Mutex,
1777		mutex,
1778		.Or,
1779		.{ .locked = true, .contended = false, .shared2 = 0 },
1780		.acquire,
1781		);
1782		if (!old_state.locked) {
1783		@branchHint(.likely);
1784		return;
1785		}
1786		if (old_state.contended) {
1787		futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
1788		}
1789		}
1790		while (true) {
1791		var old_state = @atomicRmw(
1792		Mutex,
1793		mutex,
1794		.Or,
1795		.{ .locked = true, .contended = true, .shared2 = 0 },
1796		.acquire,
1797		);
1798		if (!old_state.locked) {
1799		@branchHint(.likely);
1800		return;
1801		}
1802		old_state.contended = true;
1803		futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
1804		}
1805		}
1806
1807		fn unlock(group: Group, ev: *Evented) void {
1808		const mutex = group.mutexPtr();
1809		const old_state = @atomicRmw(
1810		Mutex,
1811		mutex,
1812		.And,
1813		.{ .locked = false, .contended = false, .shared2 = std.math.maxInt(u30) },
1814		.release,
1815		);
1816		assert(old_state.locked);
1817		if (old_state.contended) futexWake(ev, @ptrCast(mutex), 1);
1818		}
1819
1820		fn addFiber(group: Group, ev: Evented, fiber: Fiber) void {
1821		group.lock(ev);
1822		defer group.unlock(ev);
1823		const list_ptr = group.listPtr();
1824		const list = @atomicLoad(List, list_ptr, .monotonic);
1825		if (list.cancel_requested) fiber.cancel_status = .{ .requested = true, .awaiting = .nothing };
1826		const old_head = list.fibers.unpack();
1827		if (old_head) \|head\| head.link.group.prev = fiber;
1828		fiber.link.group.next = old_head;
1829		@atomicStore(List, list_ptr, .{
1830		.cancel_requested = list.cancel_requested,
1831		.awaiter_delayed = list.awaiter_delayed,
1832		.fibers = .pack(fiber),
1833		}, .monotonic);
1834		}
1835
1836		fn removeFiber(group: Group, ev: Evented, fiber: Fiber) ?*Fiber {
1837		group.lock(ev);
1838		defer group.unlock(ev);
1839		const list_ptr = group.listPtr();
1840		const list = @atomicLoad(List, list_ptr, .monotonic);
1841		if (fiber.link.group.next) \|next\| next.link.group.prev = fiber.link.group.prev;
1842		if (fiber.link.group.prev) \|prev\| {
1843		prev.link.group.next = fiber.link.group.next;
1844		} else if (fiber.link.group.next) \|new_head\| {
1845		@atomicStore(List, list_ptr, .{
1846		.cancel_requested = list.cancel_requested,
1847		.awaiter_delayed = list.awaiter_delayed,
1848		.fibers = .pack(new_head),
1849		}, .monotonic);
1850		} else if (@atomicLoad(Awaiter, group.awaiterPtr(), .monotonic).awaiter.unpack()) \|awaiter\| {
1851		if (!awaiter.cancel_status.changeAwaiting(.group, .nothing) or list.cancel_requested) {
1852		@atomicStore(List, list_ptr, .{
1853		.cancel_requested = false,
1854		.awaiter_delayed = false,
1855		.fibers = .null,
1856		}, .release);
1857		assert(awaiter.status.awaiting_group.ptr == group.ptr);
1858		awaiter.status = .{ .queue_next = null };
1859		return awaiter;
1860		}
1861		// Race with `Fiber.requestCancel`
1862		@atomicStore(List, list_ptr, .{
1863		.cancel_requested = false,
1864		.awaiter_delayed = true,
1865		.fibers = .null,
1866		}, .monotonic);
1867		} else @atomicStore(List, list_ptr, .{
1868		.cancel_requested = false,
1869		.awaiter_delayed = false,
1870		.fibers = .null,
1871		}, .release);
1872		return null;
1873		}
1874
1875		fn await(group: Group, ev: Evented, awaiter: Fiber) bool {
1876		group.lock(ev);
1877		defer group.unlock(ev);
1878		if (@atomicLoad(List, group.listPtr(), .monotonic).fibers.unpack()) \|_\| {
1879		if (group.registerAwaiter(awaiter) and awaiter.cancel_protection.check() == .unblocked) {
1880		// The awaiter already had an unacknowledged cancelation request before
1881		// attempting to await a group, so propagate the cancelation to the group.
1882		assert(!group.cancelLocked(ev, null));
1883		}
1884		return false;
1885		}
1886		return true;
1887		}
1888
1889		fn cancel(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
1890		group.lock(ev);
1891		defer group.unlock(ev);
1892		return group.cancelLocked(ev, maybe_awaiter);
1893		}
1894
1895		/// Assumes the mutex is held.
1896		fn cancelLocked(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
1897		const list_ptr = group.listPtr();
1898		const list = @atomicRmw(
1899		List,
1900		list_ptr,
1901		.Add,
1902		.{ .cancel_requested = true, .awaiter_delayed = false, .fibers = .null },
1903		.monotonic,
1904		);
1905		assert(!list.cancel_requested);
1906		if (list.fibers.unpack()) \|head\| {
1907		var maybe_fiber: ?*Fiber = head;
1908		while (maybe_fiber) \|fiber\| {
1909		fiber.requestCancel(ev);
1910		maybe_fiber = fiber.link.group.next;
1911		}
1912		if (maybe_awaiter) \|awaiter\| _ = group.registerAwaiter(awaiter);
1913		return false;
1914		}
1915		@atomicStore(
1916		List,
1917		list_ptr,
1918		.{ .cancel_requested = false, .awaiter_delayed = false, .fibers = .null },
1919		.release,
1920		);
1921		return if (maybe_awaiter) \|_\| true else list.awaiter_delayed;
1922		}
1923
1924		/// Assumes the mutex is held.
1925		fn registerAwaiter(group: Group, awaiter: *Fiber) bool {
1926		assert(awaiter.status.queue_next == null);
1927		awaiter.status = .{ .awaiting_group = group };
1928		assert(@atomicRmw(
1929		Awaiter,
1930		group.awaiterPtr(),
1931		.Add,
1932		.{ .locked = false, .contended = false, .awaiter = .pack(awaiter) },
1933		.monotonic,
1934		).awaiter == .null);
1935		return awaiter.cancel_status.changeAwaiting(.nothing, .group);
1936		}
1937
1938		const AsyncClosure = struct {
1939		ev: *Evented,
1940		group: Group,
1941		fiber: *Fiber,
1942		start: const fn (context: const anyopaque) Io.Cancelable!void,
1943
1944		fn fromFiber(fiber: Fiber) Group.AsyncClosure {
1945		return @ptrFromInt(Fiber.max_context_align.max(.of(Group.AsyncClosure)).backward(
1946		@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
1947		) - @sizeOf(Group.AsyncClosure));
1948		}
1949
1950		fn contextPointer(
1951		closure: *Group.AsyncClosure,
1952		) [*]align(Fiber.max_context_align.toByteUnits()) u8 {
1953		return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(Group.AsyncClosure));
1954		}
1955
1956		fn entry() callconv(.naked) void {
1957		switch (builtin.cpu.arch) {
1958		.aarch64 => asm volatile (
1959		\\ mov x0, sp
1960		\\ b %[call]
1961		:
1962		: [call] "X" (&call),
1963		),
1964		.x86_64 => asm volatile (
1965		\\ leaq 8(%%rsp), %%rdi
1966		\\ jmp %[call:P]
1967		:
1968		: [call] "X" (&call),
1969		),
1970		else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1971		}
1972		}
1973
1974		fn call(
1975		closure: *Group.AsyncClosure,
1976		message: *const SwitchMessage,
1977		) callconv(.withStackAlign(.c, @alignOf(Group.AsyncClosure))) noreturn {
1978		message.handle(closure.ev);
1979		assert(closure.fiber.status.queue_next == null);
1980		const result = closure.start(closure.contextPointer());
1981		const ev = closure.ev;
1982		const group = closure.group;
1983		const fiber = closure.fiber;
1984		const cancel_acknowledged = fiber.cancel_protection.acknowledged;
1985		if (result) {
1986		assert(!cancel_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
1987		} else \|err\| switch (err) {
1988		error.Canceled => assert(cancel_acknowledged), // group task returned `error.Canceled` but was never canceled
1989		}
1990		ev.yield(group.removeFiber(ev, fiber), .destroy);
1991		unreachable; // switched to dead fiber
1992		}
1993		};
1994		};
1995
1996		fn groupAsync(
1997		userdata: ?*anyopaque,
1998		type_erased: *Io.Group,
1999		context: []const u8,
2000		context_alignment: Alignment,
2001		start: const fn (context: const anyopaque) Io.Cancelable!void,
2002		) void {
2003		const ev: *Evented = @ptrCast(@alignCast(userdata));
2004		return groupConcurrent(ev, type_erased, context, context_alignment, start) catch {
2005		const fiber = Thread.current().currentFiber();
2006		const pre_acknowledged = fiber.cancel_protection.acknowledged;
2007		const result = start(context.ptr);
2008		const post_acknowledged = fiber.cancel_protection.acknowledged;
2009		if (result) {
2010		if (pre_acknowledged) {
2011		assert(post_acknowledged); // group task called `recancel` but was not canceled
2012		} else {
2013		assert(!post_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
2014		}
2015		} else \|err\| switch (err) {
2016		// Don't swallow the cancelation: make it visible to the `Group.async` caller.
2017		error.Canceled => {
2018		assert(!pre_acknowledged); // group task called `recancel` but was not canceled
2019		assert(post_acknowledged); // group task returned `error.Canceled` but was never canceled
2020		recancel(userdata);
2021		},
2022		}
2023		};
2024		}
2025
2026		fn groupConcurrent(
2027		userdata: ?*anyopaque,
2028		type_erased: *Io.Group,
2029		context: []const u8,
2030		context_alignment: Alignment,
2031		start: const fn (context: const anyopaque) Io.Cancelable!void,
2032		) Io.ConcurrentError!void {
2033		assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
2034		assert(context.len <= Fiber.max_context_size); // TODO
2035
2036		const ev: *Evented = @ptrCast(@alignCast(userdata));
2037		const group: Group = .{ .ptr = type_erased };
2038		const fiber = Fiber.create(ev) catch \|err\| switch (err) {
2039		error.OutOfMemory => return error.ConcurrencyUnavailable,
2040		};
2041
2042		const closure: *Group.AsyncClosure = .fromFiber(fiber);
2043		fiber.* = .{
2044		.required_align = {},
2045		.context = switch (builtin.cpu.arch) {
2046		.aarch64 => .{
2047		.sp = @intFromPtr(closure),
2048		.fp = 0,
2049		.pc = @intFromPtr(&Group.AsyncClosure.entry),
2050		},
2051		.x86_64 => .{
2052		.rsp = @intFromPtr(closure) - @sizeOf(usize),
2053		.rbp = 0,
2054		.rip = @intFromPtr(&Group.AsyncClosure.entry),
2055		},
2056		else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
2057		},
2058		.await_count = 0,
2059		.link = .{ .group = .{ .prev = null, .next = null } },
2060		.status = .{ .queue_next = null },
2061		.cancel_status = .unrequested,
2062		.cancel_protection = .unblocked,
2063		.name = if (tracy.enable) name: {
2064		const thread: *Thread = .current();
2065		var name_arena = thread.name_arena.promote(std.heap.page_allocator);
2066		defer thread.name_arena = name_arena.state;
2067		break :name std.fmt.allocPrintSentinel(
2068		name_arena.allocator(),
2069		"group task {d}",
2070		.{@atomicRmw(u64, &Fiber.next_name, .Add, 1, .monotonic)},
2071		0,
2072		) catch return error.ConcurrencyUnavailable;
2073		},
2074		};
2075		closure.* = .{
2076		.ev = ev,
2077		.group = group,
2078		.fiber = fiber,
2079		.start = start,
2080		};
2081		@memcpy(closure.contextPointer(), context);
2082		group.addFiber(ev, fiber);
2083		const thread: *Thread = .current();
2084		if (ev.schedule(thread, .{ .head = fiber, .tail = fiber })) thread.submit();
2085		}
2086
2087		fn groupAwait(
2088		userdata: ?*anyopaque,
2089		type_erased: *Io.Group,
2090		initial_token: *anyopaque,
2091		) Io.Cancelable!void {
2092		const ev: *Evented = @ptrCast(@alignCast(userdata));
2093		_ = initial_token;
2094		ev.yield(null, .{ .group_await = .{ .ptr = type_erased } });
2095		}
2096
2097		fn groupCancel(userdata: ?anyopaque, type_erased: Io.Group, initial_token: *anyopaque) void {
2098		const ev: *Evented = @ptrCast(@alignCast(userdata));
2099		_ = initial_token;
2100		ev.yield(null, .{ .group_cancel = .{ .ptr = type_erased } });
2101		}
2102
2103		fn recancel(userdata: ?*anyopaque) void {
2104		const ev: *Evented = @ptrCast(@alignCast(userdata));
2105		_ = ev;
2106		const cancel_protection = &Thread.current().currentFiber().cancel_protection;
2107		assert(cancel_protection.acknowledged);
2108		cancel_protection.acknowledged = false;
2109		}
2110
2111		fn swapCancelProtection(userdata: ?*anyopaque, new: Io.CancelProtection) Io.CancelProtection {
2112		const ev: *Evented = @ptrCast(@alignCast(userdata));
2113		_ = ev;
2114		const cancel_protection = &Thread.current().currentFiber().cancel_protection;
2115		defer cancel_protection.user = new;
2116		return cancel_protection.user;
2117		}
2118
2119		fn checkCancel(userdata: ?*anyopaque) Io.Cancelable!void {
2120		const ev: *Evented = @ptrCast(@alignCast(userdata));
2121		_ = ev;
2122		const fiber = Thread.current().currentFiber();
2123		switch (fiber.cancel_protection.check()) {
2124		.blocked => {},
2125		.unblocked => if (@atomicLoad(Fiber.CancelStatus, &fiber.cancel_status, .monotonic).requested) {
2126		fiber.cancel_protection.acknowledge();
2127		return error.Canceled;
2128		},
2129		}
2130		}
2131
2132		fn select(userdata: ?anyopaque, futures: []const Io.AnyFuture) Io.Cancelable!usize {
2133		const ev: *Evented = @ptrCast(@alignCast(userdata));
2134		var cancel_region: CancelRegion = .init();
2135		defer cancel_region.deinit();
2136		var await_count: u31, var result = for (futures, 0..) \|future, future_index\| {
2137		const future_fiber: *Fiber = @ptrCast(@alignCast(future));
2138		if (@atomicRmw(
2139		?*Fiber,
2140		&future_fiber.link.awaiter,
2141		.Xchg,
2142		cancel_region.fiber,
2143		.acq_rel,
2144		)) \|awaiter\| {
2145		assert(awaiter == Fiber.finished);
2146		break .{ @intCast(future_index), future_index };
2147		}
2148		} else result: {
2149		const await_count: u31 = @intCast(futures.len);
2150		cancel_region.await(.select) catch \|err\| switch (err) {
2151		error.Canceled => \|e\| break :result .{ await_count + 1, e },
2152		};
2153		ev.yield(null, .{ .await = 1 });
2154		cancel_region.await(.nothing) catch \|err\| switch (err) {
2155		error.Canceled => \|e\| break :result .{ await_count, e },
2156		};
2157		break :result .{ await_count - 1, futures.len };
2158		};
2159		for (futures[0 .. result catch futures.len], 0..) \|future, future_index\| {
2160		const future_fiber: *Fiber = @ptrCast(@alignCast(future));
2161		const awaiter = @atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, null, .monotonic);
2162		if (awaiter == Fiber.finished) {
2163		@atomicStore(?*Fiber, &future_fiber.link.awaiter, Fiber.finished, .monotonic);
2164		result = if (result) \|finished_index\| @min(future_index, finished_index) else \|e\| e;
2165		} else {
2166		assert(awaiter == cancel_region.fiber);
2167		await_count -= 1;
2168		}
2169		}
2170		// Equivalent to `ev.yield(null, .{ .await = await_count });`,
2171		// but avoiding a context switch in the common case.
2172		switch (std.math.order(
2173		@atomicRmw(i32, &cancel_region.fiber.await_count, .Sub, await_count, .monotonic),
2174		await_count,
2175		)) {
2176		.lt => ev.yield(null, .{ .await = 0 }),
2177		.eq => {},
2178		.gt => unreachable,
2179		}
2180		return result;
2181		}
2182
2183		fn futexWait(
2184		userdata: ?*anyopaque,
2185		ptr: *const u32,
2186		expected: u32,
2187		timeout: Io.Timeout,
2188		) Io.Cancelable!void {
2189		const ev: *Evented = @ptrCast(@alignCast(userdata));
2190		const timespec: ?linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = timespec: switch (timeout) {
2191		.none => .{
2192		null,
2193		.awake,
2194		linux.IORING_TIMEOUT_ABS,
2195		},
2196		.duration => \|duration\| {
2197		const ns = duration.raw.toNanoseconds();
2198		break :timespec .{
2199		.{
2200		.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
2201		.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
2202		},
2203		duration.clock,
2204		0,
2205		};
2206		},
2207		.deadline => \|deadline\| {
2208		const ns = deadline.raw.toNanoseconds();
2209		break :timespec .{
2210		.{
2211		.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
2212		.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
2213		},
2214		deadline.clock,
2215		linux.IORING_TIMEOUT_ABS,
2216		};
2217		},
2218		};
2219		var cancel_region: CancelRegion = .init();
2220		defer cancel_region.deinit();
2221		const thread = try cancel_region.awaitIoUring();
2222		thread.enqueue().* = .{
2223		.opcode = .FUTEX_WAIT,
2224		.flags = if (timespec) \|_\| linux.IOSQE_IO_LINK else 0,
2225		.ioprio = 0,
2226		.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
2227		.off = expected,
2228		.addr = @intFromPtr(ptr),
2229		.len = 0,
2230		.rw_flags = 0,
2231		.user_data = @intFromPtr(cancel_region.fiber),
2232		.buf_index = 0,
2233		.personality = 0,
2234		.splice_fd_in = 0,
2235		.addr3 = std.math.maxInt(u32),
2236		.resv = 0,
2237		};
2238		if (timespec) \|timespec_ptr\| thread.enqueue(). = .{
2239		.opcode = .LINK_TIMEOUT,
2240		.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
2241		.ioprio = 0,
2242		.fd = 0,
2243		.off = 0,
2244		.addr = @intFromPtr(timespec_ptr),
2245		.len = 1,
2246		.rw_flags = timeout_flags \| @as(u32, switch (clock) {
2247		.real => linux.IORING_TIMEOUT_REALTIME,
2248		else => 0,
2249		.boot => linux.IORING_TIMEOUT_BOOTTIME,
2250		}),
2251		.user_data = @intFromEnum(Completion.UserData.wakeup),
2252		.buf_index = 0,
2253		.personality = 0,
2254		.splice_fd_in = 0,
2255		.addr3 = 0,
2256		.resv = 0,
2257		};
2258		ev.yield(null, .nothing);
2259		switch (cancel_region.errno()) {
2260		.SUCCESS => {}, // notified by `wake()`
2261		.INTR, .CANCELED => {}, // caller's responsibility to retry
2262		.AGAIN => {}, // ptr.* != expect
2263		.INVAL => {}, // possibly timeout overflow
2264		.TIMEDOUT => unreachable,
2265		.FAULT => recoverableOsBugDetected(), // ptr was invalid
2266		else => recoverableOsBugDetected(),
2267		}
2268		}
2269
2270		fn futexWaitUncancelable(userdata: ?anyopaque, ptr: const u32, expected: u32) void {
2271		const ev: *Evented = @ptrCast(@alignCast(userdata));
2272		var cancel_region: CancelRegion = .initBlocked();
2273		defer cancel_region.deinit();
2274		const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
2275		error.Canceled => unreachable, // blocked
2276		};
2277		thread.enqueue().* = .{
2278		.opcode = .FUTEX_WAIT,
2279		.flags = 0,
2280		.ioprio = 0,
2281		.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
2282		.off = expected,
2283		.addr = @intFromPtr(ptr),
2284		.len = 0,
2285		.rw_flags = 0,
2286		.user_data = @intFromPtr(cancel_region.fiber),
2287		.buf_index = 0,
2288		.personality = 0,
2289		.splice_fd_in = 0,
2290		.addr3 = std.math.maxInt(u32),
2291		.resv = 0,
2292		};
2293		ev.yield(null, .nothing);
2294		switch (cancel_region.errno()) {
2295		.SUCCESS => {}, // notified by `wake()`
2296		.INTR, .CANCELED => {}, // caller's responsibility to retry
2297		.AGAIN => {}, // ptr.* != expect
2298		.INVAL => {}, // possibly timeout overflow
2299		.FAULT => recoverableOsBugDetected(), // ptr was invalid
2300		else => recoverableOsBugDetected(),
2301		}
2302		}
2303
2304		fn futexWake(userdata: ?anyopaque, ptr: const u32, max_waiters: u32) void {
2305		const ev: *Evented = @ptrCast(@alignCast(userdata));
2306		_ = ev;
2307		const thread: *Thread = .current();
2308		thread.enqueue().* = .{
2309		.opcode = .FUTEX_WAKE,
2310		.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
2311		.ioprio = 0,
2312		.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
2313		.off = max_waiters,
2314		.addr = @intFromPtr(ptr),
2315		.len = 0,
2316		.rw_flags = 0,
2317		.user_data = @intFromEnum(Completion.UserData.futex_wake),
2318		.buf_index = 0,
2319		.personality = 0,
2320		.splice_fd_in = 0,
2321		.addr3 = std.math.maxInt(u32),
2322		.resv = 0,
2323		};
2324		thread.submit();
2325		}
2326
2327		fn operate(userdata: ?*anyopaque, operation: Io.Operation) Io.Cancelable!Io.Operation.Result {
2328		const ev: *Evented = @ptrCast(@alignCast(userdata));
2329		var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
2330		defer maybe_sync.deinit(ev);
2331		return switch (operation) {
2332		.file_read_streaming => \|o\| .{
2333		.file_read_streaming = ev.fileReadStreaming(
2334		&maybe_sync.cancel_region,
2335		o.file,
2336		o.data,
2337		) catch \|err\| switch (err) {
2338		error.Canceled => \|e\| return e,
2339		else => \|e\| e,
2340		},
2341		},
2342		.file_write_streaming => \|o\| .{
2343		.file_write_streaming = ev.fileWriteStreaming(
2344		&maybe_sync.cancel_region,
2345		o.file,
2346		o.header,
2347		o.data,
2348		o.splat,
2349		) catch \|err\| switch (err) {
2350		error.Canceled => \|e\| return e,
2351		else => \|e\| e,
2352		},
2353		},
2354		.device_io_control => \|o\| .{
2355		.device_io_control = try ev.deviceIoControl(try maybe_sync.enterSync(ev), o),
2356		},
2357		};
2358		}
2359
2360		fn fileReadStreaming(
2361		ev: *Evented,
2362		cancel_region: *CancelRegion,
2363		file: File,
2364		data: []const []u8,
2365		) File.ReadStreamingError!usize {
2366		var iovecs_buffer: [max_iovecs_len]iovec = undefined;
2367		var i: usize = 0;
2368		for (data) \|buf\| {
2369		if (iovecs_buffer.len - i == 0) break;
2370		if (buf.len != 0) {
2371		iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len };
2372		i += 1;
2373		}
2374		}
2375		const dest = iovecs_buffer[0..i];
2376		assert(dest[0].len > 0);
2377
2378		const n = try ev.preadv(cancel_region, file.handle, dest, null);
2379		return if (n == 0) error.EndOfStream else n;
2380		}
2381
2382		fn fileWriteStreaming(
2383		ev: *Evented,
2384		cancel_region: *CancelRegion,
2385		file: File,
2386		header: []const u8,
2387		data: []const []const u8,
2388		splat: usize,
2389		) File.Writer.Error!usize {
2390		var iovecs: [max_iovecs_len]iovec_const = undefined;
2391		var iovlen: iovlen_t = 0;
2392		addBuf(&iovecs, &iovlen, header);
2393		for (data[0 .. data.len - 1]) \|bytes\| addBuf(&iovecs, &iovlen, bytes);
2394		const pattern = data[data.len - 1];
2395		if (iovecs.len - iovlen != 0) switch (splat) {
2396		0 => {},
2397		1 => addBuf(&iovecs, &iovlen, pattern),
2398		else => switch (pattern.len) {
2399		0 => {},
2400		1 => {
2401		var backup_buffer: [splat_buffer_size]u8 = undefined;
2402		const splat_buffer = &backup_buffer;
2403		const memset_len = @min(splat_buffer.len, splat);
2404		const buf = splat_buffer[0..memset_len];
2405		@memset(buf, pattern[0]);
2406		addBuf(&iovecs, &iovlen, buf);
2407		var remaining_splat = splat - buf.len;
2408		while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0) {
2409		assert(buf.len == splat_buffer.len);
2410		addBuf(&iovecs, &iovlen, splat_buffer);
2411		remaining_splat -= splat_buffer.len;
2412		}
2413		addBuf(&iovecs, &iovlen, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
2414		},
2415		else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
2416		addBuf(&iovecs, &iovlen, pattern);
2417		},
2418		},
2419		};
2420		return ev.pwritev(cancel_region, file.handle, iovecs[0..iovlen], null);
2421		}
2422
2423		fn deviceIoControl(
2424		ev: *Evented,
2425		sync: *CancelRegion.Sync,
2426		o: Io.Operation.DeviceIoControl,
2427		) Io.Cancelable!i32 {
2428		_ = ev;
2429		while (true) {
2430		try sync.cancel_region.await(.nothing);
2431		const rc = linux.ioctl(o.file.handle, @bitCast(o.code), @intFromPtr(o.arg));
2432		switch (linux.errno(rc)) {
2433		.SUCCESS => return @bitCast(@as(u32, @truncate(rc))),
2434		.INTR => continue,
2435		else => \|err\| return -@as(i32, @intFromEnum(err)),
2436		}
2437		}
2438		}
2439
2440		fn batchAwaitAsync(userdata: ?anyopaque, batch: Io.Batch) Io.Cancelable!void {
2441		const ev: *Evented = @ptrCast(@alignCast(userdata));
2442		var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
2443		defer maybe_sync.deinit(ev);
2444		ev.batchDrainSubmitted(&maybe_sync, batch, false) catch \|err\| switch (err) {
2445		error.ConcurrencyUnavailable => unreachable, // passed concurrency=false
2446		else => \|e\| return e,
2447		};
2448		maybe_sync.leaveSync(ev);
2449		while (true) {
2450		batchDrainReady(batch) catch \|err\| switch (err) {
2451		error.Timeout => unreachable, // no timeout
2452		};
2453		if (batch.completed.head != .none) return;
2454		ev.yield(null, .{ .batch_await = batch });
2455		}
2456		}
2457
2458		fn batchAwaitConcurrent(
2459		userdata: ?*anyopaque,
2460		batch: *Io.Batch,
2461		timeout: Io.Timeout,
2462		) Io.Batch.AwaitConcurrentError!void {
2463		const ev: *Evented = @ptrCast(@alignCast(userdata));
2464		var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
2465		defer maybe_sync.deinit(ev);
2466		try ev.batchDrainSubmitted(&maybe_sync, batch, true);
2467		maybe_sync.leaveSync(ev);
2468		const timespec: linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = while (true) {
2469		batchDrainReady(batch) catch \|err\| switch (err) {
2470		error.Timeout => unreachable, // no timeout
2471		};
2472		if (batch.completed.head != .none) return;
2473		switch (timeout) {
2474		.none => ev.yield(null, .{ .batch_await = batch }),
2475		.duration => \|duration\| {
2476		const ns = duration.raw.toNanoseconds();
2477		break .{
2478		.{
2479		.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
2480		.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
2481		},
2482		duration.clock,
2483		0,
2484		};
2485		},
2486		.deadline => \|deadline\| {
2487		const ns = deadline.raw.toNanoseconds();
2488		break .{
2489		.{
2490		.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
2491		.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
2492		},
2493		deadline.clock,
2494		linux.IORING_TIMEOUT_ABS,
2495		};
2496		},
2497		}
2498		};
2499		{
2500		const thread = try maybe_sync.cancel_region.awaitIoUring();
2501		thread.enqueue().* = .{
2502		.opcode = .TIMEOUT,
2503		.flags = 0,
2504		.ioprio = 0,
2505		.fd = 0,
2506		.off = 0,
2507		.addr = @intFromPtr(&timespec),
2508		.len = 1,
2509		.rw_flags = timeout_flags \| @as(u32, switch (clock) {
2510		.real => linux.IORING_TIMEOUT_REALTIME,
2511		else => 0,
2512		.boot => linux.IORING_TIMEOUT_BOOTTIME,
2513		}),
2514		.user_data = @intFromPtr(&batch.context) \| 0b11,
2515		.buf_index = 0,
2516		.personality = 0,
2517		.splice_fd_in = 0,
2518		.addr3 = 0,
2519		.resv = 0,
2520		};
2521		}
2522		while (batch.completed.head == .none) {
2523		ev.yield(null, .{ .batch_await = batch });
2524		batchDrainReady(batch) catch \|err\| switch (err) {
2525		error.Timeout => \|e\| return if (batch.completed.head == .none) e,
2526		};
2527		if (batch.completed.head == .none) continue;
2528		}
2529		const thread = try maybe_sync.cancel_region.awaitIoUring();
2530		thread.enqueue().* = .{
2531		.opcode = .TIMEOUT_REMOVE,
2532		.flags = 0,
2533		.ioprio = 0,
2534		.fd = 0,
2535		.off = 0,
2536		.addr = @intFromPtr(&batch.context) \| 0b11,
2537		.len = 0,
2538		.rw_flags = 0,
2539		.user_data = @intFromPtr(maybe_sync.cancel_region.fiber),
2540		.buf_index = 0,
2541		.personality = 0,
2542		.splice_fd_in = 0,
2543		.addr3 = 0,
2544		.resv = 0,
2545		};
2546		ev.yield(null, .nothing);
2547		switch (maybe_sync.cancel_region.errno()) {
2548		.SUCCESS => return,
2549		.BUSY, .NOENT => {},
2550		else => \|err\| unexpectedErrno(err) catch {},
2551		}
2552		while (true) {
2553		batchDrainReady(batch) catch \|err\| switch (err) {
2554		error.Timeout => return,
2555		};
2556		ev.yield(null, .{ .batch_await = batch });
2557		}
2558		}
2559
2560		/// If `concurrency` is false, `error.ConcurrencyUnavailable` is unreachable.
2561		fn batchDrainSubmitted(
2562		ev: *Evented,
2563		maybe_sync: *CancelRegion.Sync.Maybe,
2564		batch: *Io.Batch,
2565		concurrency: bool,
2566		) (Io.ConcurrentError \|\| Io.Cancelable)!void {
2567		var index = batch.submitted.head;
2568		if (index == .none) return;
2569		errdefer batch.submitted.head = index;
2570		const thread = try maybe_sync.cancelRegion().awaitIoUring();
2571		while (index != .none) {
2572		const storage = &batch.storage[index.toIndex()];
2573		const next_index = storage.submission.node.next;
2574		if (@as(?Io.Operation.Result, result: switch (storage.submission.operation) {
2575		.file_read_streaming => \|o\| {
2576		const buffer = for (o.data) \|buffer\| {
2577		if (buffer.len != 0) break buffer;
2578		} else break :result .{ .file_read_streaming = 0 };
2579		const fd = o.file.handle;
2580		storage.* = .{ .pending = .{
2581		.node = .{ .prev = batch.pending.tail, .next = .none },
2582		.tag = .file_read_streaming,
2583		.context = undefined,
2584		} };
2585		thread.enqueue().* = .{
2586		.opcode = .READ,
2587		.flags = 0,
2588		.ioprio = 0,
2589		.fd = fd,
2590		.off = std.math.maxInt(u64),
2591		.addr = @intFromPtr(buffer.ptr),
2592		.len = @min(buffer.len, 0xfffff000),
2593		.rw_flags = 0,
2594		.user_data = @intFromPtr(&storage.pending.context) \| 0b10,
2595		.buf_index = 0,
2596		.personality = 0,
2597		.splice_fd_in = 0,
2598		.addr3 = 0,
2599		.resv = 0,
2600		};
2601		break :result null;
2602		},
2603		.file_write_streaming => \|o\| {
2604		const buffer = buffer: {
2605		if (o.header.len != 0) break :buffer o.header;
2606		for (o.data[0 .. o.data.len - 1]) \|buffer\| {
2607		if (buffer.len != 0) break :buffer buffer;
2608		}
2609		if (o.splat > 0) break :buffer o.data[o.data.len - 1];
2610		break :result .{ .file_write_streaming = 0 };
2611		};
2612		const fd = o.file.handle;
2613		storage.* = .{ .pending = .{
2614		.node = .{ .prev = batch.pending.tail, .next = .none },
2615		.tag = .file_write_streaming,
2616		.context = undefined,
2617		} };
2618		thread.enqueue().* = .{
2619		.opcode = .WRITE,
2620		.flags = 0,
2621		.ioprio = 0,
2622		.fd = fd,
2623		.off = std.math.maxInt(u64),
2624		.addr = @intFromPtr(buffer.ptr),
2625		.len = @min(buffer.len, 0xfffff000),
2626		.rw_flags = 0,
2627		.user_data = @intFromPtr(&storage.pending.context) \| 0b10,
2628		.buf_index = 0,
2629		.personality = 0,
2630		.splice_fd_in = 0,
2631		.addr3 = 0,
2632		.resv = 0,
2633		};
2634		break :result null;
2635		},
2636		.device_io_control => \|o\| if (concurrency)
2637		return error.ConcurrencyUnavailable
2638		else
2639		.{ .device_io_control = try ev.deviceIoControl(try maybe_sync.enterSync(ev), o) },
2640		})) \|result\| {
2641		switch (batch.completed.tail) {
2642		.none => batch.completed.head = index,
2643		else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next = index,
2644		}
2645		batch.completed.tail = index;
2646		storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2647		} else {
2648		switch (batch.pending.tail) {
2649		.none => batch.pending.head = index,
2650		else => \|tail_index\| batch.storage[tail_index.toIndex()].pending.node.next = index,
2651		}
2652		batch.pending.tail = index;
2653		storage.pending.context[0] = @intFromPtr(batch);
2654		}
2655		index = next_index;
2656		}
2657		batch.submitted = .{ .head = .none, .tail = .none };
2658		}
2659
2660		fn batchDrainReady(batch: *Io.Batch) Io.Timeout.Error!void {
2661		while (@atomicRmw(?*anyopaque, &batch.context, .Xchg, null, .acquire)) \|head\| {
2662		var next: usize = @intFromPtr(head);
2663		var timeout = false;
2664		while (cond: switch (@as(u2, @truncate(next))) {
2665		0b00 => if (timeout) return error.Timeout else false,
2666		0b01 => {
2667		assert(!timeout);
2668		return error.Timeout;
2669		},
2670		0b10 => true,
2671		0b11 => {
2672		assert(!timeout);
2673		timeout = true;
2674		break :cond true;
2675		},
2676		}) {
2677		var context: *Io.Operation.Storage.Pending.Context = @ptrFromInt(next & ~@as(usize, 0b11));
2678		next = context[0];
2679		const completion: Completion = .{
2680		.result = @bitCast(@as(u32, @intCast(context[1]))),
2681		.flags = @intCast(context[2]),
2682		};
2683		const pending: *Io.Operation.Storage.Pending = @fieldParentPtr("context", context);
2684		const storage: *Io.Operation.Storage = @fieldParentPtr("pending", pending);
2685		const index: Io.Operation.OptionalIndex = .fromIndex(storage - batch.storage.ptr);
2686		assert(completion.flags & linux.IORING_CQE_F_SKIP == 0);
2687		switch (pending.node.prev) {
2688		.none => batch.pending.head = pending.node.next,
2689		else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.next =
2690		pending.node.next,
2691		}
2692		switch (pending.node.next) {
2693		.none => batch.pending.tail = pending.node.prev,
2694		else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.prev =
2695		pending.node.prev,
2696		}
2697		if (@as(?Io.Operation.Result, result: switch (pending.tag) {
2698		.file_read_streaming => .{
2699		.file_read_streaming = switch (completion.errno()) {
2700		.SUCCESS => @as(u32, @bitCast(completion.result)),
2701		.INTR => 0,
2702		.CANCELED => break :result null,
2703		.INVAL => \|err\| errnoBug(err),
2704		.FAULT => \|err\| errnoBug(err),
2705		.AGAIN => error.WouldBlock,
2706		.BADF => \|err\| errnoBug(err), // File descriptor used after closed
2707		.IO => error.InputOutput,
2708		.ISDIR => error.IsDir,
2709		.NOBUFS => error.SystemResources,
2710		.NOMEM => error.SystemResources,
2711		.NOTCONN => error.SocketUnconnected,
2712		.CONNRESET => error.ConnectionResetByPeer,
2713		else => \|err\| unexpectedErrno(err),
2714		},
2715		},
2716		.file_write_streaming => .{
2717		.file_write_streaming = switch (completion.errno()) {
2718		.SUCCESS => @as(u32, @bitCast(completion.result)),
2719		.INTR => 0,
2720		.CANCELED => break :result null,
2721		.INVAL => \|err\| errnoBug(err),
2722		.FAULT => \|err\| errnoBug(err),
2723		.AGAIN => error.WouldBlock,
2724		.BADF => error.NotOpenForWriting, // Can be a race condition.
2725		.DESTADDRREQ => \|err\| errnoBug(err), // `connect` was never called.
2726		.DQUOT => error.DiskQuota,
2727		.FBIG => error.FileTooBig,
2728		.IO => error.InputOutput,
2729		.NOSPC => error.NoSpaceLeft,
2730		.PERM => error.PermissionDenied,
2731		.PIPE => error.BrokenPipe,
2732		.CONNRESET => \|err\| errnoBug(err), // Not a socket handle.
2733		.BUSY => error.DeviceBusy,
2734		else => \|err\| unexpectedErrno(err),
2735		},
2736		},
2737		.device_io_control => unreachable,
2738		})) \|result\| {
2739		switch (batch.completed.tail) {
2740		.none => batch.completed.head = index,
2741		else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next =
2742		index,
2743		}
2744		storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2745		batch.completed.tail = index;
2746		} else {
2747		switch (batch.unused.tail) {
2748		.none => batch.unused.head = index,
2749		else => \|tail_index\| batch.storage[tail_index.toIndex()].unused.next = index,
2750		}
2751		storage.* = .{ .unused = .{ .prev = batch.unused.tail, .next = .none } };
2752		batch.unused.tail = index;
2753		}
2754		}
2755		}
2756		}
2757
2758		fn batchCancel(userdata: ?anyopaque, batch: Io.Batch) void {
2759		const ev: *Evented = @ptrCast(@alignCast(userdata));
2760		_ = ev;
2761		batchDrainReady(batch) catch \|err\| switch (err) {
2762		error.Timeout => unreachable, // no timeout
2763		};
2764		var index = batch.pending.head;
2765		if (index == .none) return;
2766		var cancel_region: CancelRegion = .initBlocked();
2767		defer cancel_region.deinit();
2768		const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
2769		error.Canceled => unreachable, // blocked
2770		};
2771		while (index != .none) {
2772		const pending = &batch.storage[index.toIndex()].pending;
2773		thread.enqueue().* = .{
2774		.opcode = .ASYNC_CANCEL,
2775		.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
2776		.ioprio = 0,
2777		.fd = 0,
2778		.off = 0,
2779		.addr = @intFromPtr(&pending.context) \| 0b10,
2780		.len = 0,
2781		.rw_flags = 0,
2782		.user_data = @intFromEnum(Completion.UserData.wakeup),
2783		.buf_index = 0,
2784		.personality = 0,
2785		.splice_fd_in = 0,
2786		.addr3 = 0,
2787		.resv = 0,
2788		};
2789		index = pending.node.next;
2790		}
2791		while (batch.pending.head != .none) batchDrainReady(batch) catch \|err\| switch (err) {
2792		error.Timeout => unreachable, // no timeout
2793		};
2794		}
2795
2796		fn dirCreateDir(
2797		userdata: ?*anyopaque,
2798		dir: Dir,
2799		sub_path: []const u8,
2800		permissions: Dir.Permissions,
2801		) Dir.CreateDirError!void {
2802		const ev: *Evented = @ptrCast(@alignCast(userdata));
2803
2804		var path_buffer: [PATH_MAX]u8 = undefined;
2805		const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2806
2807		var cancel_region: CancelRegion = .init();
2808		defer cancel_region.deinit();
2809		while (true) {
2810		const thread = try cancel_region.awaitIoUring();
2811		thread.enqueue().* = .{
2812		.opcode = .MKDIRAT,
2813		.flags = 0,
2814		.ioprio = 0,
2815		.fd = dir.handle,
2816		.off = 0,
2817		.addr = @intFromPtr(sub_path_posix.ptr),
2818		.len = permissions.toMode(),
2819		.rw_flags = 0,
2820		.user_data = @intFromPtr(cancel_region.fiber),
2821		.buf_index = 0,
2822		.personality = 0,
2823		.splice_fd_in = 0,
2824		.addr3 = 0,
2825		.resv = 0,
2826		};
2827		ev.yield(null, .nothing);
2828		switch (cancel_region.errno()) {
2829		.SUCCESS => return,
2830		.INTR, .CANCELED => continue,
2831		.ACCES => return error.AccessDenied,
2832		.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
2833		.PERM => return error.PermissionDenied,
2834		.DQUOT => return error.DiskQuota,
2835		.EXIST => return error.PathAlreadyExists,
2836		.FAULT => \|err\| return errnoBug(err),
2837		.LOOP => return error.SymLinkLoop,
2838		.MLINK => return error.LinkQuotaExceeded,
2839		.NAMETOOLONG => return error.NameTooLong,
2840		.NOENT => return error.FileNotFound,
2841		.NOMEM => return error.SystemResources,
2842		.NOSPC => return error.NoSpaceLeft,
2843		.NOTDIR => return error.NotDir,
2844		.ROFS => return error.ReadOnlyFileSystem,
2845		// dragonfly: when dir_fd is unlinked from filesystem
2846		.NOTCONN => return error.FileNotFound,
2847		.ILSEQ => return error.BadPathName,
2848		else => \|err\| return unexpectedErrno(err),
2849		}
2850		}
2851		}
2852
2853		fn dirCreateDirPath(
2854		userdata: ?*anyopaque,
2855		dir: Dir,
2856		sub_path: []const u8,
2857		permissions: Dir.Permissions,
2858		) Dir.CreateDirPathError!Dir.CreatePathStatus {
2859		const ev: *Evented = @ptrCast(@alignCast(userdata));
2860
2861		var it = Dir.path.componentIterator(sub_path);
2862		var status: Dir.CreatePathStatus = .existed;
2863		var component = it.last() orelse return error.BadPathName;
2864		while (true) {
2865		if (dirCreateDir(ev, dir, component.path, permissions)) \|_\| {
2866		status = .created;
2867		} else \|err\| switch (err) {
2868		error.PathAlreadyExists => {
2869		// stat the file and return an error if it's not a directory
2870		// this is important because otherwise a dangling symlink
2871		// could cause an infinite loop
2872		const fstat = try dirStatFile(ev, dir, component.path, .{});
2873		if (fstat.kind != .directory) return error.NotDir;
2874		},
2875		error.FileNotFound => \|e\| {
2876		component = it.previous() orelse return e;
2877		continue;
2878		},
2879		else => \|e\| return e,
2880		}
2881		component = it.next() orelse return status;
2882		}
2883		}
2884
2885		fn dirCreateDirPathOpen(
2886		userdata: ?*anyopaque,
2887		dir: Dir,
2888		sub_path: []const u8,
2889		permissions: Dir.Permissions,
2890		options: Dir.OpenOptions,
2891		) Dir.CreateDirPathOpenError!Dir {
2892		const ev: *Evented = @ptrCast(@alignCast(userdata));
2893		return dirOpenDir(ev, dir, sub_path, options) catch \|err\| switch (err) {
2894		error.FileNotFound => {
2895		_ = try dirCreateDirPath(ev, dir, sub_path, permissions);
2896		return dirOpenDir(ev, dir, sub_path, options);
2897		},
2898		else => \|e\| return e,
2899		};
2900		}
2901
2902		fn dirOpenDir(
2903		userdata: ?*anyopaque,
2904		dir: Dir,
2905		sub_path: []const u8,
2906		options: Dir.OpenOptions,
2907		) Dir.OpenError!Dir {
2908		const ev: *Evented = @ptrCast(@alignCast(userdata));
2909
2910		var path_buffer: [PATH_MAX]u8 = undefined;
2911		const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2912
2913		var cancel_region: CancelRegion = .init();
2914		defer cancel_region.deinit();
2915		return .{
2916		.handle = ev.openat(&cancel_region, dir.handle, sub_path_posix, .{
2917		.ACCMODE = .RDONLY,
2918		.DIRECTORY = true,
2919		.NOFOLLOW = !options.follow_symlinks,
2920		.CLOEXEC = true,
2921		.PATH = !options.iterate,
2922		}, 0) catch \|err\| switch (err) {
2923		error.IsDir => return errnoBug(.ISDIR),
2924		error.WouldBlock => return errnoBug(.AGAIN),
2925		error.FileTooBig => return errnoBug(.FBIG),
2926		error.NoSpaceLeft => return errnoBug(.NOSPC),
2927		error.DeviceBusy => return errnoBug(.BUSY), // O_EXCL not passed
2928		error.FileBusy => return errnoBug(.TXTBSY),
2929		error.PathAlreadyExists => return errnoBug(.EXIST), // Not creating.
2930		error.PipeBusy => return error.Unexpected, // Not opening a pipe.
2931		error.AntivirusInterference => unreachable, // Windows-only
2932		error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
2933		else => \|e\| return e,
2934		},
2935		};
2936		}
2937
2938		fn dirStat(userdata: ?*anyopaque, dir: Dir) Dir.StatError!Dir.Stat {
2939		const ev: *Evented = @ptrCast(@alignCast(userdata));
2940		var cancel_region: CancelRegion = .init();
2941		defer cancel_region.deinit();
2942		return ev.stat(&cancel_region, dir.handle);
2943		}
2944
2945		fn dirStatFile(
2946		userdata: ?*anyopaque,
2947		dir: Dir,
2948		sub_path: []const u8,
2949		options: Dir.StatFileOptions,
2950		) Dir.StatFileError!File.Stat {
2951		const ev: *Evented = @ptrCast(@alignCast(userdata));
2952		var path_buffer: [PATH_MAX]u8 = undefined;
2953		const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2954		var cancel_region: CancelRegion = .init();
2955		defer cancel_region.deinit();
2956		return ev.statx(&cancel_region, dir.handle, sub_path_posix, linux.AT.NO_AUTOMOUNT \|
2957		@as(u32, if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW));
2958		}
2959
2960		fn dirAccess(
2961		userdata: ?*anyopaque,
2962		dir: Dir,
2963		sub_path: []const u8,
2964		options: Dir.AccessOptions,
2965		) Dir.AccessError!void {
2966		const ev: *Evented = @ptrCast(@alignCast(userdata));
2967
2968		var path_buffer: [PATH_MAX]u8 = undefined;
2969		const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2970
2971		const mode: u32 =
2972		@as(u32, if (options.read) linux.R_OK else 0) \|
2973		@as(u32, if (options.write) linux.W_OK else 0) \|
2974		@as(u32, if (options.execute) linux.X_OK else 0);
2975		const flags: u32 = if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW;
2976
2977		var sync: CancelRegion.Sync = try .init(ev);
2978		defer sync.deinit(ev);
2979		while (true) {
2980		try sync.cancel_region.await(.nothing);
2981		switch (linux.errno(linux.faccessat(dir.handle, sub_path_posix, mode, flags))) {
2982		.SUCCESS => return,
2983		.INTR => continue,
2984		.ACCES => return error.AccessDenied,
2985		.PERM => return error.PermissionDenied,
2986		.ROFS => return error.ReadOnlyFileSystem,
2987		.LOOP => return error.SymLinkLoop,
2988		.TXTBSY => return error.FileBusy,
2989		.NOTDIR => return error.FileNotFound,
2990		.NOENT => return error.FileNotFound,
2991		.NAMETOOLONG => return error.NameTooLong,
2992		.INVAL => \|err\| return errnoBug(err),
2993		.FAULT => \|err\| return errnoBug(err),
2994		.IO => return error.InputOutput,
2995		.NOMEM => return error.SystemResources,
2996		.ILSEQ => return error.BadPathName,
2997		else => \|err\| return unexpectedErrno(err),
2998		}
2999		}
3000		}
3001
3002		fn dirCreateFile(
3003		userdata: ?*anyopaque,
3004		dir: Dir,
3005		sub_path: []const u8,
3006		flags: File.CreateFlags,
3007		) File.OpenError!File {
3008		const ev: *Evented = @ptrCast(@alignCast(userdata));
3009
3010		var path_buffer: [PATH_MAX]u8 = undefined;
3011		const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3012
3013		var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
3014		defer maybe_sync.deinit(ev);
3015		const fd = try ev.openat(&maybe_sync.cancel_region, dir.handle, sub_path_posix, .{
3016		.ACCMODE = if (flags.read) .RDWR else .WRONLY,
3017		.CREAT = true,
3018		.TRUNC = flags.truncate,
3019		.EXCL = flags.exclusive,
3020		.CLOEXEC = true,
3021		}, flags.permissions.toMode());
3022		errdefer ev.close(maybe_sync.cancelRegion(), fd);
3023
3024		switch (flags.lock) {
3025		.none => {},
3026		.shared, .exclusive => try ev.flock(
3027		try maybe_sync.enterSync(ev),
3028		fd,
3029		flags.lock,
3030		if (flags.lock_nonblocking) .nonblocking else .blocking,
3031		),
3032		}
3033
3034		return .{ .handle = fd, .flags = .{ .nonblocking = false } };
3035		}
3036
3037		fn dirCreateFileAtomic(
3038		userdata: ?*anyopaque,
3039		dir: Dir,
3040		dest_path: []const u8,
3041		options: Dir.CreateFileAtomicOptions,
3042		) Dir.CreateFileAtomicError!File.Atomic {
3043		const ev: *Evented = @ptrCast(@alignCast(userdata));
3044		// Linux has O_TMPFILE, but linkat() does not support AT_REPLACE, so it's
3045		// useless when we have to make up a bogus path name to do the rename()
3046		// anyway.
3047		if (!options.replace) tmpfile: {
3048		const flags: linux.O = if (@hasField(linux.O, "TMPFILE")) .{
3049		.ACCMODE = .RDWR,
3050		.TMPFILE = true,
3051		.DIRECTORY = true,
3052		.CLOEXEC = true,
3053		} else if (@hasField(linux.O, "TMPFILE0") and !@hasField(linux.O, "TMPFILE2")) .{
3054		.ACCMODE = .RDWR,
3055		.TMPFILE0 = true,
3056		.TMPFILE1 = true,
3057		.DIRECTORY = true,
3058		.CLOEXEC = true,
3059		} else break :tmpfile;
3060
3061		const dest_dirname = Dir.path.dirname(dest_path);
3062		if (dest_dirname) \|dirname\| {
3063		// This has a nice side effect of preemptively triggering EISDIR or
3064		// ENOENT, avoiding the ambiguity below.
3065		_ = dirCreateDirPath(ev, dir, dirname, .default_dir) catch \|err\| switch (err) {
3066		// None of these make sense in this context.
3067		error.IsDir,
3068		error.Streaming,
3069		error.DiskQuota,
3070		error.PathAlreadyExists,
3071		error.LinkQuotaExceeded,
3072		error.PipeBusy,
3073		error.FileTooBig,
3074		error.DeviceBusy,
3075		error.FileLocksUnsupported,
3076		error.FileBusy,
3077		=> return error.Unexpected,
3078
3079		else => \|e\| return e,
3080		};
3081		}
3082
3083		var path_buffer: [PATH_MAX]u8 = undefined;
3084		const sub_path_posix = try pathToPosix(dest_dirname orelse ".", &path_buffer);
3085
3086		var cancel_region: CancelRegion = .init();
3087		defer cancel_region.deinit();
3088		return .{
3089		.file = .{
3090		.handle = ev.openat(
3091		&cancel_region,
3092		dir.handle,
3093		sub_path_posix,
3094		flags,
3095		options.permissions.toMode(),
3096		) catch \|err\| switch (err) {
3097		error.IsDir, error.FileNotFound => {
3098		// Ambiguous error code. It might mean the file system
3099		// does not support O_TMPFILE. Therefore, we must fall
3100		// back to not using O_TMPFILE.
3101		break :tmpfile;
3102		},
3103		error.FileTooBig => return errnoBug(.FBIG),
3104		error.DeviceBusy => return errnoBug(.BUSY), // O_EXCL not passed
3105		error.PathAlreadyExists => return errnoBug(.EXIST), // Not creating.
3106		error.PipeBusy => return error.Unexpected, // Not opening a pipe.
3107		error.AntivirusInterference => unreachable, // Windows-only
3108		error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
3109		else => \|e\| return e,
3110		},
3111		.flags = .{ .nonblocking = false },
3112		},
3113		.file_basename_hex = 0,
3114		.dest_sub_path = dest_path,
3115		.file_open = true,
3116		.file_exists = false,
3117		.close_dir_on_deinit = false,
3118		.dir = dir,
3119		};
3120		}
3121
3122		if (Dir.path.dirname(dest_path)) \|dirname\| {
3123		const new_dir = if (options.make_path)
3124		dirCreateDirPathOpen(ev, dir, dirname, .default_dir, .{}) catch \|err\| switch (err) {
3125		// None of these make sense in this context.
3126		error.IsDir,
3127		error.Streaming,
3128		error.DiskQuota,
3129		error.PathAlreadyExists,
3130		error.LinkQuotaExceeded,
3131		error.PipeBusy,
3132		error.FileTooBig,
3133		error.FileLocksUnsupported,
3134		error.DeviceBusy,
3135		=> return error.Unexpected,
3136
3137		else => \|e\| return e,
3138		}
3139		else
3140		try dirOpenDir(ev, dir, dirname, .{});
3141
3142		return ev.atomicFileInit(Dir.path.basename(dest_path), options.permissions, new_dir, true);
3143		}
3144
3145		return ev.atomicFileInit(dest_path, options.permissions, dir, false);
3146		}
3147
3148		fn atomicFileInit(
3149		ev: *Evented,
3150		dest_basename: []const u8,
3151		permissions: File.Permissions,
3152		dir: Dir,
3153		close_dir_on_deinit: bool,
3154		) Dir.CreateFileAtomicError!File.Atomic {
3155		while (true) {
3156		var random_integer: u64 = undefined;
3157		random(ev, @ptrCast(&random_integer));
3158		const tmp_sub_path = std.fmt.hex(random_integer);
3159		const file = dirCreateFile(ev, dir, &tmp_sub_path, .{
3160		.permissions = permissions,
3161		.exclusive = true,
3162		}) catch \|err\| switch (err) {
3163		error.PathAlreadyExists => continue,
3164		error.DeviceBusy => continue,
3165		error.FileBusy => continue,
3166
3167		error.IsDir => return error.Unexpected, // No path components.
3168		error.FileTooBig => return error.Unexpected, // Creating, not opening.
3169		error.FileLocksUnsupported => return error.Unexpected, // Not asking for locks.
3170		error.PipeBusy => return error.Unexpected, // Not opening a pipe.
3171
3172		else => \|e\| return e,
3173		};
3174		return .{
3175		.file = file,
3176		.file_basename_hex = random_integer,
3177		.dest_sub_path = dest_basename,
3178		.file_open = true,
3179		.file_exists = true,
3180		.close_dir_on_deinit = close_dir_on_deinit,
3181		.dir = dir,
3182		};
3183		}
3184		}
3185
3186		fn dirOpenFile(
3187		userdata: ?*anyopaque,
3188		dir: Dir,
3189		sub_path: []const u8,
3190		flags: File.OpenFlags,
3191		) File.OpenError!File {
3192		const ev: *Evented = @ptrCast(@alignCast(userdata));
3193
3194		var path_buffer: [PATH_MAX]u8 = undefined;
3195		const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3196
3197		var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
3198		defer maybe_sync.deinit(ev);
3199		const fd = try ev.openat(&maybe_sync.cancel_region, dir.handle, sub_path_posix, .{
3200		.ACCMODE = switch (flags.mode) {
3201		.read_only => .RDONLY,
3202		.write_only => .WRONLY,
3203		.read_write => .RDWR,
3204		},
3205		.NOCTTY = !flags.allow_ctty,
3206		.NOFOLLOW = !flags.follow_symlinks,
3207		.CLOEXEC = true,
3208		.PATH = flags.path_only,
3209		}, 0);
3210		errdefer ev.close(maybe_sync.cancelRegion(), fd);
3211
3212		if (!flags.allow_directory) {
3213		const is_dir = is_dir: {
3214		const s = ev.stat(&maybe_sync.cancel_region, fd) catch \|err\| switch (err) {
3215		// The directory-ness is either unknown or unknowable
3216		error.Streaming => break :is_dir false,
3217		else => \|e\| return e,
3218		};
3219		break :is_dir s.kind == .directory;
3220		};
3221		if (is_dir) return error.IsDir;
3222		}
3223
3224		switch (flags.lock) {
3225		.none => {},
3226		.shared, .exclusive => try ev.flock(
3227		try maybe_sync.enterSync(ev),
3228		fd,
3229		flags.lock,
3230		if (flags.lock_nonblocking) .nonblocking else .blocking,
3231		),
3232		}
3233
3234		return .{ .handle = fd, .flags = .{ .nonblocking = false } };
3235		}
3236
3237		fn dirClose(userdata: ?*anyopaque, dirs: []const Dir) void {
3238		const ev: *Evented = @ptrCast(@alignCast(userdata));
3239		var cancel_region: CancelRegion = .init();
3240		defer cancel_region.deinit();
3241		for (dirs) \|dir\| ev.close(&cancel_region, dir.handle);
3242		}
3243
3244		fn dirRead(userdata: ?anyopaque, dr: Dir.Reader, buffer: []Dir.Entry) Dir.Reader.Error!usize {
3245		const ev: *Evented = @ptrCast(@alignCast(userdata));
3246		var buffer_index: usize = 0;
3247		while (buffer.len - buffer_index != 0) {
3248		if (dr.end - dr.index == 0) {
3249		// Refill the buffer, unless we've already created references to
3250		// buffered data.
3251		if (buffer_index != 0) break;
3252		var sync: CancelRegion.Sync = try .init(ev);
3253		defer sync.deinit(ev);
3254		if (dr.state == .reset) {
3255		ev.lseek(&sync, dr.dir.handle, 0, linux.SEEK.SET) catch \|err\| switch (err) {
3256		error.Unseekable => return error.Unexpected,
3257		else => \|e\| return e,
3258		};
3259		dr.state = .reading;
3260		}
3261		const n = while (true) {
3262		try sync.cancel_region.await(.nothing);
3263		const rc = linux.getdents64(dr.dir.handle, dr.buffer.ptr, dr.buffer.len);
3264		switch (linux.errno(rc)) {
3265		.SUCCESS => break rc,
3266		.INTR => continue,
3267		.BADF => \|err\| return errnoBug(err), // Dir is invalid or was opened without iteration ability.
3268		.FAULT => \|err\| return errnoBug(err),
3269		.NOTDIR => \|err\| return errnoBug(err),
3270		// To be consistent across platforms, iteration
3271		// ends if the directory being iterated is deleted
3272		// during iteration. This matches the behavior of
3273		// non-Linux, non-WASI UNIX platforms.
3274		.NOENT => {
3275		dr.state = .finished;
3276		return 0;
3277		},
3278		// This can occur when reading /proc/$PID/net, or
3279		// if the provided buffer is too small. Neither
3280		// scenario is intended to be handled by this API.
3281		.INVAL => return error.Unexpected,
3282		.ACCES => return error.AccessDenied, // Lacking permission to iterate this directory.
3283		else => \|err\| return unexpectedErrno(err),
3284		}
3285		};
3286		if (n == 0) {
3287		dr.state = .finished;
3288		return 0;
3289		}
3290		dr.index = 0;
3291		dr.end = n;
3292		}
3293		// Linux aligns the header by padding after the null byte of the name
3294		// to align the next entry. This means we can find the end of the name
3295		// by looking at only the 8 bytes before the next record. However since
3296		// file names are usually short it's better to keep the machine code
3297		// simpler.
3298		//
3299		// Furthermore, I observed qemu user mode to not align this struct, so
3300		// this code makes the conservative choice to not assume alignment.
3301		const linux_entry: *align(1) linux.dirent64 = @ptrCast(&dr.buffer[dr.index]);
3302		const next_index = dr.index + linux_entry.reclen;
3303		dr.index = next_index;
3304		const name_ptr: [*]u8 = &linux_entry.name;
3305		const padded_name = name_ptr[0 .. linux_entry.reclen - @offsetOf(linux.dirent64, "name")];
3306		const name_len = std.mem.findScalar(u8, padded_name, 0).?;
3307		const name = name_ptr[0..name_len :0];
3308
3309		if (std.mem.eql(u8, name, ".") or std.mem.eql(u8, name, "..")) continue;
3310
3311		const entry_kind: File.Kind = switch (linux_entry.type) {
3312		linux.DT.BLK => .block_device,
3313		linux.DT.CHR => .character_device,
3314		linux.DT.DIR => .directory,
3315		linux.DT.FIFO => .named_pipe,
3316		linux.DT.LNK => .sym_link,
3317		linux.DT.REG => .file,
3318		linux.DT.SOCK => .unix_domain_socket,
3319		else => .unknown,
3320		};
3321		buffer[buffer_index] = .{
3322		.name = name,
3323		.kind = entry_kind,
3324		.inode = linux_entry.ino,
3325		};
3326		buffer_index += 1;
3327		}
3328		return buffer_index;
3329		}
3330
3331		fn dirRealPath(userdata: ?*anyopaque, dir: Dir, out_buffer: []u8) Dir.RealPathError!usize {
3332		const ev: *Evented = @ptrCast(@alignCast(userdata));
3333		var sync: CancelRegion.Sync = try .init(ev);
3334		defer sync.deinit(ev);
3335		return ev.realPath(&sync, dir.handle, out_buffer);
3336		}
3337
3338		fn dirRealPathFile(
3339		userdata: ?*anyopaque,
3340		dir: Dir,
3341		sub_path: []const u8,
3342		out_buffer: []u8,
3343		) Dir.RealPathFileError!usize {
3344		const ev: *Evented = @ptrCast(@alignCast(userdata));
3345
3346		var path_buffer: [PATH_MAX]u8 = undefined;
3347		const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3348
3349		var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
3350		defer maybe_sync.deinit(ev);
3351		const fd = ev.openat(&maybe_sync.cancel_region, dir.handle, sub_path_posix, .{
3352		.CLOEXEC = true,
3353		.PATH = true,
3354		}, 0) catch \|err\| switch (err) {
3355		error.WouldBlock => return errnoBug(.AGAIN),
3356		error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
3357		else => \|e\| return e,
3358		};
3359		defer ev.close(maybe_sync.cancelRegion(), fd);
3360		return ev.realPath(try maybe_sync.enterSync(ev), fd, out_buffer);
3361		}
3362
3363		fn dirDeleteFile(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteFileError!void {
3364		const ev: *Evented = @ptrCast(@alignCast(userdata));
3365
3366		var path_buffer: [PATH_MAX]u8 = undefined;
3367		const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3368
3369		var cancel_region: CancelRegion = .init();
3370		defer cancel_region.deinit();
3371		while (true) {
3372		const thread = try cancel_region.awaitIoUring();
3373		thread.enqueue().* = .{
3374		.opcode = .UNLINKAT,
3375		.flags = 0,
3376		.ioprio = 0,
3377		.fd = dir.handle,
3378		.off = 0,
3379		.addr = @intFromPtr(sub_path_posix.ptr),
3380		.len = 0,
3381		.rw_flags = 0,
3382		.user_data = @intFromPtr(cancel_region.fiber),
3383		.buf_index = 0,
3384		.personality = 0,
3385		.splice_fd_in = 0,
3386		.addr3 = 0,
3387		.resv = 0,
3388		};
3389		ev.yield(null, .nothing);
3390		switch (cancel_region.errno()) {
3391		.SUCCESS => return,
3392		.INTR, .CANCELED => continue,
3393		.PERM => return error.PermissionDenied,
3394		.ACCES => return error.AccessDenied,
3395		.BUSY => return error.FileBusy,
3396		.FAULT => \|err\| return errnoBug(err),
3397		.IO => return error.FileSystem,
3398		.ISDIR => return error.IsDir,
3399		.LOOP => return error.SymLinkLoop,
3400		.NAMETOOLONG => return error.NameTooLong,
3401		.NOENT => return error.FileNotFound,
3402		.NOTDIR => return error.NotDir,
3403		.NOMEM => return error.SystemResources,
3404		.ROFS => return error.ReadOnlyFileSystem,
3405		.EXIST => \|err\| return errnoBug(err),
3406		.NOTEMPTY => \|err\| return errnoBug(err), // Not passing AT.REMOVEDIR
3407		.ILSEQ => return error.BadPathName,
3408		.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
3409		.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
3410		else => \|err\| return unexpectedErrno(err),
3411		}
3412		}
3413		}
3414
3415		fn dirDeleteDir(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteDirError!void {
3416		const ev: *Evented = @ptrCast(@alignCast(userdata));
3417
3418		var path_buffer: [PATH_MAX]u8 = undefined;
3419		const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3420
3421		var cancel_region: CancelRegion = .init();
3422		defer cancel_region.deinit();
3423		while (true) {
3424		const thread = try cancel_region.awaitIoUring();
3425		thread.enqueue().* = .{
3426		.opcode = .UNLINKAT,
3427		.flags = 0,
3428		.ioprio = 0,
3429		.fd = dir.handle,
3430		.off = 0,
3431		.addr = @intFromPtr(sub_path_posix.ptr),
3432		.len = 0,
3433		.rw_flags = linux.AT.REMOVEDIR,
3434		.user_data = @intFromPtr(cancel_region.fiber),
3435		.buf_index = 0,
3436		.personality = 0,
3437		.splice_fd_in = 0,
3438		.addr3 = 0,
3439		.resv = 0,
3440		};
3441		ev.yield(null, .nothing);
3442		switch (cancel_region.errno()) {
3443		.SUCCESS => return,
3444		.INTR, .CANCELED => continue,
3445		.ACCES => return error.AccessDenied,
3446		.PERM => return error.PermissionDenied,
3447		.BUSY => return error.FileBusy,
3448		.FAULT => \|err\| return errnoBug(err),
3449		.IO => return error.FileSystem,
3450		.ISDIR => \|err\| return errnoBug(err),
3451		.LOOP => return error.SymLinkLoop,
3452		.NAMETOOLONG => return error.NameTooLong,
3453		.NOENT => return error.FileNotFound,
3454		.NOTDIR => return error.NotDir,
3455		.NOMEM => return error.SystemResources,
3456		.ROFS => return error.ReadOnlyFileSystem,
3457		.EXIST => \|err\| return errnoBug(err),
3458		.NOTEMPTY => return error.DirNotEmpty,
3459		.ILSEQ => return error.BadPathName,
3460		.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
3461		.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
3462		else => \|err\| return unexpectedErrno(err),
3463		}
3464		}
3465		}
3466
3467		fn dirRename(
3468		userdata: ?*anyopaque,
3469		old_dir: Dir,
3470		old_sub_path: []const u8,
3471		new_dir: Dir,
3472		new_sub_path: []const u8,
3473		) Dir.RenameError!void {
3474		const ev: *Evented = @ptrCast(@alignCast(userdata));
3475
3476		var old_path_buffer: [PATH_MAX]u8 = undefined;
3477		var new_path_buffer: [PATH_MAX]u8 = undefined;
3478
3479		const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
3480		const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
3481
3482		var cancel_region: CancelRegion = .init();
3483		defer cancel_region.deinit();
3484		return ev.renameat(
3485		&cancel_region,
3486		old_dir.handle,
3487		old_sub_path_posix,
3488		new_dir.handle,
3489		new_sub_path_posix,
3490		.{},
3491		);
3492		}
3493
3494		fn dirRenamePreserve(
3495		userdata: ?*anyopaque,
3496		old_dir: Dir,
3497		old_sub_path: []const u8,
3498		new_dir: Dir,
3499		new_sub_path: []const u8,
3500		) Dir.RenamePreserveError!void {
3501		const ev: *Evented = @ptrCast(@alignCast(userdata));
3502
3503		var old_path_buffer: [PATH_MAX]u8 = undefined;
3504		var new_path_buffer: [PATH_MAX]u8 = undefined;
3505
3506		const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
3507		const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
3508
3509		var cancel_region: CancelRegion = .init();
3510		defer cancel_region.deinit();
3511		return ev.renameat(
3512		&cancel_region,
3513		old_dir.handle,
3514		old_sub_path_posix,
3515		new_dir.handle,
3516		new_sub_path_posix,
3517		.{ .NOREPLACE = true },
3518		);
3519		}
3520
3521		fn dirSymLink(
3522		userdata: ?*anyopaque,
3523		dir: Dir,
3524		target_path: []const u8,
3525		sym_link_path: []const u8,
3526		flags: Dir.SymLinkFlags,
3527		) Dir.SymLinkError!void {
3528		const ev: *Evented = @ptrCast(@alignCast(userdata));
3529		_ = flags;
3530
3531		var target_path_buffer: [PATH_MAX]u8 = undefined;
3532		var sym_link_path_buffer: [PATH_MAX]u8 = undefined;
3533
3534		const target_path_posix = try pathToPosix(target_path, &target_path_buffer);
3535		const sym_link_path_posix = try pathToPosix(sym_link_path, &sym_link_path_buffer);
3536
3537		var cancel_region: CancelRegion = .init();
3538		defer cancel_region.deinit();
3539		while (true) {
3540		const thread = try cancel_region.awaitIoUring();
3541		thread.enqueue().* = .{
3542		.opcode = .SYMLINKAT,
3543		.flags = 0,
3544		.ioprio = 0,
3545		.fd = dir.handle,
3546		.off = @intFromPtr(sym_link_path_posix.ptr),
3547		.addr = @intFromPtr(target_path_posix.ptr),
3548		.len = 0,
3549		.rw_flags = 0,
3550		.user_data = @intFromPtr(cancel_region.fiber),
3551		.buf_index = 0,
3552		.personality = 0,
3553		.splice_fd_in = 0,
3554		.addr3 = 0,
3555		.resv = 0,
3556		};
3557		ev.yield(null, .nothing);
3558		switch (cancel_region.errno()) {
3559		.SUCCESS => return,
3560		.INTR, .CANCELED => continue,
3561		.FAULT => \|err\| return errnoBug(err),
3562		.INVAL => \|err\| return errnoBug(err),
3563		.ACCES => return error.AccessDenied,
3564		.PERM => return error.PermissionDenied,
3565		.DQUOT => return error.DiskQuota,
3566		.EXIST => return error.PathAlreadyExists,
3567		.IO => return error.FileSystem,
3568		.LOOP => return error.SymLinkLoop,
3569		.NAMETOOLONG => return error.NameTooLong,
3570		.NOENT => return error.FileNotFound,
3571		.NOTDIR => return error.NotDir,
3572		.NOMEM => return error.SystemResources,
3573		.NOSPC => return error.NoSpaceLeft,
3574		.ROFS => return error.ReadOnlyFileSystem,
3575		.ILSEQ => return error.BadPathName,
3576		else => \|err\| return unexpectedErrno(err),
3577		}
3578		}
3579		}
3580
3581		fn dirReadLink(
3582		userdata: ?*anyopaque,
3583		dir: Dir,
3584		sub_path: []const u8,
3585		buffer: []u8,
3586		) Dir.ReadLinkError!usize {
3587		const ev: *Evented = @ptrCast(@alignCast(userdata));
3588
3589		var sub_path_buffer: [PATH_MAX]u8 = undefined;
3590		const sub_path_posix = try pathToPosix(sub_path, &sub_path_buffer);
3591
3592		var sync: CancelRegion.Sync = try .init(ev);
3593		defer sync.deinit(ev);
3594		while (true) {
3595		try sync.cancel_region.await(.nothing);
3596		const rc = linux.readlinkat(dir.handle, sub_path_posix, buffer.ptr, buffer.len);
3597		switch (linux.errno(rc)) {
3598		.SUCCESS => {
3599		const len: usize = @bitCast(rc);
3600		return len;
3601		},
3602		.INTR => continue,
3603		.ACCES => return error.AccessDenied,
3604		.FAULT => \|err\| return errnoBug(err),
3605		.INVAL => return error.NotLink,
3606		.IO => return error.FileSystem,
3607		.LOOP => return error.SymLinkLoop,
3608		.NAMETOOLONG => return error.NameTooLong,
3609		.NOENT => return error.FileNotFound,
3610		.NOMEM => return error.SystemResources,
3611		.NOTDIR => return error.NotDir,
3612		.ILSEQ => return error.BadPathName,
3613		else => \|err\| return unexpectedErrno(err),
3614		}
3615		}
3616		}
3617
3618		fn dirSetOwner(
3619		userdata: ?*anyopaque,
3620		dir: Dir,
3621		owner: ?File.Uid,
3622		group: ?File.Gid,
3623		) Dir.SetOwnerError!void {
3624		const ev: *Evented = @ptrCast(@alignCast(userdata));
3625		var sync: CancelRegion.Sync = try .init(ev);
3626		defer sync.deinit(ev);
3627		try ev.fchownat(
3628		&sync,
3629		dir.handle,
3630		"",
3631		owner orelse std.math.maxInt(linux.uid_t),
3632		group orelse std.math.maxInt(linux.gid_t),
3633		linux.AT.EMPTY_PATH,
3634		);
3635		}
3636
3637		fn dirSetFileOwner(
3638		userdata: ?*anyopaque,
3639		dir: Dir,
3640		sub_path: []const u8,
3641		owner: ?File.Uid,
3642		group: ?File.Gid,
3643		options: Dir.SetFileOwnerOptions,
3644		) Dir.SetFileOwnerError!void {
3645		const ev: *Evented = @ptrCast(@alignCast(userdata));
3646		var path_buffer: [PATH_MAX]u8 = undefined;
3647		const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3648		var sync: CancelRegion.Sync = try .init(ev);
3649		defer sync.deinit(ev);
3650		try ev.fchownat(
3651		&sync,
3652		dir.handle,
3653		sub_path_posix,
3654		owner orelse std.math.maxInt(linux.uid_t),
3655		group orelse std.math.maxInt(linux.gid_t),
3656		if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
3657		);
3658		}
3659
3660		fn dirSetPermissions(
3661		userdata: ?*anyopaque,
3662		dir: Dir,
3663		permissions: Dir.Permissions,
3664		) Dir.SetPermissionsError!void {
3665		const ev: *Evented = @ptrCast(@alignCast(userdata));
3666		var sync: CancelRegion.Sync = try .init(ev);
3667		defer sync.deinit(ev);
3668		ev.fchmodat(
3669		&sync,
3670		dir.handle,
3671		"",
3672		permissions.toMode(),
3673		linux.AT.EMPTY_PATH,
3674		) catch \|err\| switch (err) {
3675		error.NameTooLong => return errnoBug(.NAMETOOLONG),
3676		error.BadPathName => return errnoBug(.ILSEQ),
3677		error.ProcessFdQuotaExceeded => return errnoBug(.MFILE),
3678		error.SystemFdQuotaExceeded => return errnoBug(.NFILE),
3679		error.OperationUnsupported => return errnoBug(.OPNOTSUPP),
3680		else => \|e\| return e,
3681		};
3682		}
3683
3684		fn dirSetFilePermissions(
3685		userdata: ?*anyopaque,
3686		dir: Dir,
3687		sub_path: []const u8,
3688		permissions: Dir.Permissions,
3689		options: Dir.SetFilePermissionsOptions,
3690		) Dir.SetFilePermissionsError!void {
3691		const ev: *Evented = @ptrCast(@alignCast(userdata));
3692		var path_buffer: [PATH_MAX]u8 = undefined;
3693		const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3694		var sync: CancelRegion.Sync = try .init(ev);
3695		defer sync.deinit(ev);
3696		try ev.fchmodat(
3697		&sync,
3698		dir.handle,
3699		sub_path_posix,
3700		permissions.toMode(),
3701		if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
3702		);
3703		}
3704
3705		fn dirSetTimestamps(
3706		userdata: ?*anyopaque,
3707		dir: Dir,
3708		sub_path: []const u8,
3709		options: Dir.SetTimestampsOptions,
3710		) Dir.SetTimestampsError!void {
3711		const ev: *Evented = @ptrCast(@alignCast(userdata));
3712		var path_buffer: [PATH_MAX]u8 = undefined;
3713		const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3714		var cancel_region: CancelRegion.Sync = try .init(ev);
3715		defer cancel_region.deinit(ev);
3716		try ev.utimensat(
3717		&cancel_region,
3718		dir.handle,
3719		sub_path_posix,
3720		if (options.modify_timestamp != .now or options.access_timestamp != .now) &.{
3721		setTimestampToPosix(options.access_timestamp),
3722		setTimestampToPosix(options.modify_timestamp),
3723		} else null,
3724		if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
3725		);
3726		}
3727
3728		fn dirHardLink(
3729		userdata: ?*anyopaque,
3730		old_dir: Dir,
3731		old_sub_path: []const u8,
3732		new_dir: Dir,
3733		new_sub_path: []const u8,
3734		options: Dir.HardLinkOptions,
3735		) Dir.HardLinkError!void {
3736		const ev: *Evented = @ptrCast(@alignCast(userdata));
3737
3738		var old_path_buffer: [PATH_MAX]u8 = undefined;
3739		var new_path_buffer: [PATH_MAX]u8 = undefined;
3740
3741		const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
3742		const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
3743
3744		var cancel_region: CancelRegion = .init();
3745		defer cancel_region.deinit();
3746		return ev.linkat(
3747		&cancel_region,
3748		old_dir.handle,
3749		old_sub_path_posix,
3750		new_dir.handle,
3751		new_sub_path_posix,
3752		if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
3753		);
3754		}
3755
3756		fn fileStat(userdata: ?*anyopaque, file: File) File.StatError!File.Stat {
3757		const ev: *Evented = @ptrCast(@alignCast(userdata));
3758		var cancel_region: CancelRegion = .init();
3759		defer cancel_region.deinit();
3760		return ev.stat(&cancel_region, file.handle);
3761		}
3762
3763		fn fileLength(userdata: ?*anyopaque, file: File) File.LengthError!u64 {
3764		const ev: *Evented = @ptrCast(@alignCast(userdata));
3765		var cancel_region: CancelRegion = .init();
3766		defer cancel_region.deinit();
3767		while (true) {
3768		var statx_buf = std.mem.zeroes(linux.Statx);
3769		const thread = try cancel_region.awaitIoUring();
3770		thread.enqueue().* = .{
3771		.opcode = .STATX,
3772		.flags = 0,
3773		.ioprio = 0,
3774		.fd = file.handle,
3775		.off = @intFromPtr(&statx_buf),
3776		.addr = @intFromPtr(""),
3777		.len = @bitCast(linux.STATX{ .SIZE = true }),
3778		.rw_flags = linux.AT.EMPTY_PATH,
3779		.user_data = @intFromPtr(cancel_region.fiber),
3780		.buf_index = 0,
3781		.personality = 0,
3782		.splice_fd_in = 0,
3783		.addr3 = 0,
3784		.resv = 0,
3785		};
3786		ev.yield(null, .nothing);
3787		switch (cancel_region.errno()) {
3788		.SUCCESS => {
3789		if (!statx_buf.mask.SIZE) return error.Unexpected;
3790		return statx_buf.size;
3791		},
3792		.INTR, .CANCELED => continue,
3793		.ACCES => \|err\| return errnoBug(err),
3794		.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
3795		.FAULT => \|err\| return errnoBug(err),
3796		.INVAL => \|err\| return errnoBug(err),
3797		.LOOP => \|err\| return errnoBug(err),
3798		.NAMETOOLONG => \|err\| return errnoBug(err),
3799		.NOENT => \|err\| return errnoBug(err),
3800		.NOMEM => return error.SystemResources,
3801		.NOTDIR => \|err\| return errnoBug(err),
3802		else => \|err\| return unexpectedErrno(err),
3803		}
3804		}
3805		}
3806
3807		fn fileClose(userdata: ?*anyopaque, files: []const File) void {
3808		const ev: *Evented = @ptrCast(@alignCast(userdata));
3809		var cancel_region: CancelRegion = .init();
3810		defer cancel_region.deinit();
3811		for (files) \|file\| ev.close(&cancel_region, file.handle);
3812		}
3813
3814		fn fileWritePositional(
3815		userdata: ?*anyopaque,
3816		file: File,
3817		header: []const u8,
3818		data: []const []const u8,
3819		splat: usize,
3820		offset: u64,
3821		) File.WritePositionalError!usize {
3822		const ev: *Evented = @ptrCast(@alignCast(userdata));
3823
3824		var iovecs: [max_iovecs_len]iovec_const = undefined;
3825		var iovlen: iovlen_t = 0;
3826		addBuf(&iovecs, &iovlen, header);
3827		for (data[0 .. data.len - 1]) \|bytes\| addBuf(&iovecs, &iovlen, bytes);
3828		const pattern = data[data.len - 1];
3829		if (iovecs.len - iovlen != 0) switch (splat) {
3830		0 => {},
3831		1 => addBuf(&iovecs, &iovlen, pattern),
3832		else => switch (pattern.len) {
3833		0 => {},
3834		1 => {
3835		var backup_buffer: [splat_buffer_size]u8 = undefined;
3836		const splat_buffer = &backup_buffer;
3837		const memset_len = @min(splat_buffer.len, splat);
3838		const buf = splat_buffer[0..memset_len];
3839		@memset(buf, pattern[0]);
3840		addBuf(&iovecs, &iovlen, buf);
3841		var remaining_splat = splat - buf.len;
3842		while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0) {
3843		assert(buf.len == splat_buffer.len);
3844		addBuf(&iovecs, &iovlen, splat_buffer);
3845		remaining_splat -= splat_buffer.len;
3846		}
3847		addBuf(&iovecs, &iovlen, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
3848		},
3849		else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
3850		addBuf(&iovecs, &iovlen, pattern);
3851		},
3852		},
3853		};
3854
3855		var cancel_region: CancelRegion = .init();
3856		defer cancel_region.deinit();
3857		return ev.pwritev(&cancel_region, file.handle, iovecs[0..iovlen], offset);
3858		}
3859
3860		/// This is either usize or u32. Since, either is fine, let's use the same
3861		/// `addBuf` function for both writing to a file and sending network messages.
3862		const iovlen_t = @FieldType(linux.msghdr_const, "iovlen");
3863
3864		fn addBuf(v: []iovec_const, i: *iovlen_t, bytes: []const u8) void {
3865		// OS checks ptr addr before length so zero length vectors must be omitted.
3866		if (bytes.len == 0) return;
3867		if (v.len - i.* == 0) return;
3868		v[i.*] = .{ .base = bytes.ptr, .len = bytes.len };
3869		i.* += 1;
3870		}
3871
3872		fn fileWriteFileStreaming(
3873		userdata: ?*anyopaque,
3874		file: File,
3875		header: []const u8,
3876		file_reader: *File.Reader,
3877		limit: Io.Limit,
3878		) File.Writer.WriteFileError!usize {
3879		const ev: *Evented = @ptrCast(@alignCast(userdata));
3880		_ = ev;
3881		_ = file;
3882		_ = header;
3883		_ = file_reader;
3884		_ = limit;
3885		return error.Unimplemented;
3886		}
3887
3888		fn fileWriteFilePositional(
3889		userdata: ?*anyopaque,
3890		file: File,
3891		header: []const u8,
3892		file_reader: *File.Reader,
3893		limit: Io.Limit,
3894		offset: u64,
3895		) File.WriteFilePositionalError!usize {
3896		const ev: *Evented = @ptrCast(@alignCast(userdata));
3897		_ = ev;
3898		_ = file;
3899		_ = header;
3900		_ = file_reader;
3901		_ = limit;
3902		_ = offset;
3903		return error.Unimplemented;
3904		}
3905
3906		fn fileReadPositional(
3907		userdata: ?*anyopaque,
3908		file: File,
3909		data: []const []u8,
3910		offset: u64,
3911		) File.ReadPositionalError!usize {
3912		const ev: *Evented = @ptrCast(@alignCast(userdata));
3913
3914		var iovecs_buffer: [max_iovecs_len]iovec = undefined;
3915		var i: usize = 0;
3916		for (data) \|buf\| {
3917		if (iovecs_buffer.len - i == 0) break;
3918		if (buf.len != 0) {
3919		iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len };
3920		i += 1;
3921		}
3922		}
3923		if (i == 0) return 0;
3924		const dest = iovecs_buffer[0..i];
3925		assert(dest[0].len > 0);
3926
3927		var cancel_region: CancelRegion = .init();
3928		defer cancel_region.deinit();
3929		return ev.preadv(&cancel_region, file.handle, dest, offset) catch \|err\| switch (err) {
3930		error.SocketUnconnected => errnoBug(.NOTCONN), // not a socket
3931		error.ConnectionResetByPeer => errnoBug(.CONNRESET), // not a socket
3932		else => \|e\| e,
3933		};
3934		}
3935
3936		fn fileSeekBy(userdata: ?*anyopaque, file: File, offset: i64) File.SeekError!void {
3937		const ev: *Evented = @ptrCast(@alignCast(userdata));
3938		var sync: CancelRegion.Sync = try .init(ev);
3939		defer sync.deinit(ev);
3940		try ev.lseek(&sync, file.handle, @bitCast(offset), linux.SEEK.CUR);
3941		}
3942
3943		fn fileSeekTo(userdata: ?*anyopaque, file: File, offset: u64) File.SeekError!void {
3944		const ev: *Evented = @ptrCast(@alignCast(userdata));
3945		var sync: CancelRegion.Sync = try .init(ev);
3946		defer sync.deinit(ev);
3947		try ev.lseek(&sync, file.handle, offset, linux.SEEK.SET);
3948		}
3949
3950		fn fileSync(userdata: ?*anyopaque, file: File) File.SyncError!void {
3951		const ev: *Evented = @ptrCast(@alignCast(userdata));
3952		var cancel_region: CancelRegion = .init();
3953		defer cancel_region.deinit();
3954		while (true) {
3955		const thread = try cancel_region.awaitIoUring();
3956		thread.enqueue().* = .{
3957		.opcode = .FSYNC,
3958		.flags = 0,
3959		.ioprio = 0,
3960		.fd = file.handle,
3961		.off = 0,
3962		.addr = 0,
3963		.len = 0,
3964		.rw_flags = 0,
3965		.user_data = @intFromPtr(cancel_region.fiber),
3966		.buf_index = 0,
3967		.personality = 0,
3968		.splice_fd_in = 0,
3969		.addr3 = 0,
3970		.resv = 0,
3971		};
3972		ev.yield(null, .nothing);
3973		switch (cancel_region.errno()) {
3974		.SUCCESS => return,
3975		.INTR, .CANCELED => continue,
3976		.BADF => \|err\| return errnoBug(err),
3977		.INVAL => \|err\| return errnoBug(err),
3978		.ROFS => \|err\| return errnoBug(err),
3979		.IO => return error.InputOutput,
3980		.NOSPC => return error.NoSpaceLeft,
3981		.DQUOT => return error.DiskQuota,
3982		else => \|err\| return unexpectedErrno(err),
3983		}
3984		}
3985		}
3986
3987		fn fileIsTty(userdata: ?*anyopaque, file: File) Io.Cancelable!bool {
3988		const ev: *Evented = @ptrCast(@alignCast(userdata));
3989		var sync: CancelRegion.Sync = try .init(ev);
3990		defer sync.deinit(ev);
3991		while (true) {
3992		try sync.cancel_region.await(.nothing);
3993		var wsz: winsize = undefined;
3994		const rc = linux.ioctl(file.handle, linux.T.IOCGWINSZ, @intFromPtr(&wsz));
3995		switch (linux.errno(rc)) {
3996		.SUCCESS => return true,
3997		.INTR => continue,
3998		else => return false,
3999		}
4000		}
4001		}
4002
4003		fn fileEnableAnsiEscapeCodes(userdata: ?*anyopaque, file: File) File.EnableAnsiEscapeCodesError!void {
4004		const ev: *Evented = @ptrCast(@alignCast(userdata));
4005		if (!try fileIsTty(ev, file)) return error.NotTerminalDevice;
4006		}
4007
4008		fn fileSetLength(userdata: ?*anyopaque, file: File, length: u64) File.SetLengthError!void {
4009		const ev: *Evented = @ptrCast(@alignCast(userdata));
4010		var cancel_region: CancelRegion = .init();
4011		defer cancel_region.deinit();
4012		while (true) {
4013		const thread = try cancel_region.awaitIoUring();
4014		thread.enqueue().* = .{
4015		.opcode = .FTRUNCATE,
4016		.flags = 0,
4017		.ioprio = 0,
4018		.fd = file.handle,
4019		.off = length,
4020		.addr = 0,
4021		.len = 0,
4022		.rw_flags = 0,
4023		.user_data = @intFromPtr(cancel_region.fiber),
4024		.buf_index = 0,
4025		.personality = 0,
4026		.splice_fd_in = 0,
4027		.addr3 = 0,
4028		.resv = 0,
4029		};
4030		ev.yield(null, .nothing);
4031		switch (cancel_region.errno()) {
4032		.SUCCESS => return,
4033		.INTR, .CANCELED => continue,
4034		.FBIG => return error.FileTooBig,
4035		.IO => return error.InputOutput,
4036		.PERM => return error.PermissionDenied,
4037		.TXTBSY => return error.FileBusy,
4038		.BADF => \|err\| return errnoBug(err), // Handle not open for writing.
4039		.INVAL => return error.NonResizable, // This is returned for /dev/null for example.
4040		else => \|err\| return unexpectedErrno(err),
4041		}
4042		}
4043		}
4044
4045		fn fileSetOwner(
4046		userdata: ?*anyopaque,
4047		file: File,
4048		owner: ?File.Uid,
4049		group: ?File.Gid,
4050		) File.SetOwnerError!void {
4051		const ev: *Evented = @ptrCast(@alignCast(userdata));
4052		var sync: CancelRegion.Sync = try .init(ev);
4053		defer sync.deinit(ev);
4054		try ev.fchownat(
4055		&sync,
4056		file.handle,
4057		"",
4058		owner orelse std.math.maxInt(linux.uid_t),
4059		group orelse std.math.maxInt(linux.gid_t),
4060		linux.AT.EMPTY_PATH,
4061		);
4062		}
4063
4064		fn fileSetPermissions(
4065		userdata: ?*anyopaque,
4066		file: File,
4067		permissions: File.Permissions,
4068		) File.SetPermissionsError!void {
4069		const ev: *Evented = @ptrCast(@alignCast(userdata));
4070		var sync: CancelRegion.Sync = try .init(ev);
4071		defer sync.deinit(ev);
4072		ev.fchmodat(
4073		&sync,
4074		file.handle,
4075		"",
4076		permissions.toMode(),
4077		linux.AT.EMPTY_PATH,
4078		) catch \|err\| switch (err) {
4079		error.NameTooLong => return errnoBug(.NAMETOOLONG),
4080		error.BadPathName => return errnoBug(.ILSEQ),
4081		error.ProcessFdQuotaExceeded => return errnoBug(.MFILE),
4082		error.SystemFdQuotaExceeded => return errnoBug(.NFILE),
4083		error.OperationUnsupported => return errnoBug(.OPNOTSUPP),
4084		else => \|e\| return e,
4085		};
4086		}
4087
4088		fn fileSetTimestamps(
4089		userdata: ?*anyopaque,
4090		file: File,
4091		options: File.SetTimestampsOptions,
4092		) File.SetTimestampsError!void {
4093		const ev: *Evented = @ptrCast(@alignCast(userdata));
4094		var sync: CancelRegion.Sync = try .init(ev);
4095		defer sync.deinit(ev);
4096		try ev.utimensat(
4097		&sync,
4098		file.handle,
4099		"",
4100		if (options.modify_timestamp != .now or options.access_timestamp != .now) &.{
4101		setTimestampToPosix(options.access_timestamp),
4102		setTimestampToPosix(options.modify_timestamp),
4103		} else null,
4104		linux.AT.EMPTY_PATH,
4105		);
4106		}
4107
4108		fn fileLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!void {
4109		const ev: *Evented = @ptrCast(@alignCast(userdata));
4110		var sync: CancelRegion.Sync = try .init(ev);
4111		defer sync.deinit(ev);
4112		ev.flock(&sync, file.handle, lock, .blocking) catch \|err\| switch (err) {
4113		error.WouldBlock => unreachable, // blocking
4114		else => \|e\| return e,
4115		};
4116		}
4117
4118		fn fileTryLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!bool {
4119		const ev: *Evented = @ptrCast(@alignCast(userdata));
4120		var sync: CancelRegion.Sync = try .init(ev);
4121		defer sync.deinit(ev);
4122		ev.flock(&sync, file.handle, lock, switch (lock) {
4123		.none => .blocking,
4124		.shared, .exclusive => .nonblocking,
4125		}) catch \|err\| switch (err) {
4126		error.WouldBlock => return false,
4127		else => \|e\| return e,
4128		};
4129		return true;
4130		}
4131
4132		fn fileUnlock(userdata: ?*anyopaque, file: File) void {
4133		const ev: *Evented = @ptrCast(@alignCast(userdata));
4134		var sync: CancelRegion.Sync = .initBlocked(ev);
4135		defer sync.deinit(ev);
4136		ev.flock(&sync, file.handle, .none, .blocking) catch \|err\| switch (err) {
4137		error.Canceled => unreachable, // blocked
4138		error.WouldBlock => unreachable, // blocking
4139		error.SystemResources => return recoverableOsBugDetected(), // Resource deallocation.
4140		error.FileLocksUnsupported => return recoverableOsBugDetected(), // We already got the lock.
4141		error.Unexpected => return recoverableOsBugDetected(), // Resource deallocation must succeed.
4142		};
4143		}
4144
4145		fn fileDowngradeLock(userdata: ?*anyopaque, file: File) File.DowngradeLockError!void {
4146		const ev: *Evented = @ptrCast(@alignCast(userdata));
4147		var sync: CancelRegion.Sync = try .init(ev);
4148		defer sync.deinit(ev);
4149		ev.flock(&sync, file.handle, .shared, .nonblocking) catch \|err\| switch (err) {
4150		error.WouldBlock => return errnoBug(.AGAIN), // File was not locked in exclusive mode.
4151		error.SystemResources => return errnoBug(.NOLCK), // Lock already obtained.
4152		error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Lock already obtained.
4153		else => \|e\| return e,
4154		};
4155		}
4156
4157		fn fileRealPath(userdata: ?*anyopaque, file: File, out_buffer: []u8) File.RealPathError!usize {
4158		const ev: *Evented = @ptrCast(@alignCast(userdata));
4159		var sync: CancelRegion.Sync = try .init(ev);
4160		defer sync.deinit(ev);
4161		return ev.realPath(&sync, file.handle, out_buffer);
4162		}
4163
4164		fn fileHardLink(
4165		userdata: ?*anyopaque,
4166		file: File,
4167		new_dir: Dir,
4168		new_sub_path: []const u8,
4169		options: File.HardLinkOptions,
4170		) File.HardLinkError!void {
4171		const ev: *Evented = @ptrCast(@alignCast(userdata));
4172
4173		var new_path_buffer: [PATH_MAX]u8 = undefined;
4174		const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
4175
4176		var cancel_region: CancelRegion = .init();
4177		defer cancel_region.deinit();
4178		return ev.linkat(
4179		&cancel_region,
4180		file.handle,
4181		"",
4182		new_dir.handle,
4183		new_sub_path_posix,
4184		linux.AT.EMPTY_PATH \| @as(u32, if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW),
4185		);
4186		}
4187
4188		fn fileMemoryMapCreate(
4189		userdata: ?*anyopaque,
4190		file: File,
4191		options: File.MemoryMap.CreateOptions,
4192		) File.MemoryMap.CreateError!File.MemoryMap {
4193		const ev: *Evented = @ptrCast(@alignCast(userdata));
4194
4195		const prot: linux.PROT = .{
4196		.READ = options.protection.read,
4197		.WRITE = options.protection.write,
4198		.EXEC = options.protection.execute,
4199		};
4200		const flags: linux.MAP = .{
4201		.TYPE = .SHARED_VALIDATE,
4202		.POPULATE = options.populate,
4203		};
4204
4205		const page_align = std.heap.page_size_min;
4206
4207		var sync: CancelRegion.Sync = try .init(ev);
4208		defer sync.deinit(ev);
4209		const contents = while (true) {
4210		try sync.cancel_region.await(.nothing);
4211		const casted_offset = std.math.cast(i64, options.offset) orelse return error.Unseekable;
4212		const rc = linux.mmap(null, options.len, prot, flags, file.handle, casted_offset);
4213		switch (linux.errno(rc)) {
4214		.SUCCESS => break @as([*]align(page_align) u8, @ptrFromInt(rc))[0..options.len],
4215		.INTR => continue,
4216		.ACCES => return error.AccessDenied,
4217		.AGAIN => return error.LockedMemoryLimitExceeded,
4218		.MFILE => return error.ProcessFdQuotaExceeded,
4219		.NFILE => return error.SystemFdQuotaExceeded,
4220		.NOMEM => return error.OutOfMemory,
4221		.PERM => return error.PermissionDenied,
4222		.OVERFLOW => return error.Unseekable,
4223		.BADF => \|err\| return errnoBug(err), // Always a race condition.
4224		.INVAL => \|err\| return errnoBug(err), // Invalid parameters to mmap()
4225		.OPNOTSUPP => \|err\| return errnoBug(err), // Bad flags with MAP.SHARED_VALIDATE on Linux.
4226		else => \|err\| return unexpectedErrno(err),
4227		}
4228		};
4229		return .{
4230		.file = file,
4231		.offset = options.offset,
4232		.memory = contents,
4233		.section = {},
4234		};
4235		}
4236
4237		fn fileMemoryMapDestroy(userdata: ?anyopaque, mm: File.MemoryMap) void {
4238		const ev: *Evented = @ptrCast(@alignCast(userdata));
4239		_ = ev;
4240		const memory = mm.memory;
4241		if (memory.len == 0) return;
4242		switch (linux.errno(linux.munmap(memory.ptr, memory.len))) {
4243		.SUCCESS => {},
4244		else => \|err\| if (builtin.mode == .Debug)
4245		std.log.err("failed to unmap {d} bytes at {*}: {t}", .{ memory.len, memory.ptr, err }),
4246		}
4247		mm.* = undefined;
4248		}
4249
4250		fn processExecutableOpen(
4251		userdata: ?*anyopaque,
4252		flags: File.OpenFlags,
4253		) process.OpenExecutableError!File {
4254		const ev: *Evented = @ptrCast(@alignCast(userdata));
4255		return dirOpenFile(ev, .{ .handle = linux.AT.FDCWD }, "/proc/self/exe", flags);
4256		}
4257
4258		fn processExecutablePath(userdata: ?*anyopaque, out_buffer: []u8) process.ExecutablePathError!usize {
4259		const ev: *Evented = @ptrCast(@alignCast(userdata));
4260		return dirReadLink(ev, .cwd(), "/proc/self/exe", out_buffer) catch \|err\| switch (err) {
4261		error.UnsupportedReparsePointType => unreachable, // Windows-only
4262		error.NetworkNotFound => unreachable, // Windows-only
4263		error.FileBusy => unreachable, // Windows-only
4264		else => \|e\| return e,
4265		};
4266		}
4267
4268		fn lockStderr(userdata: ?*anyopaque, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
4269		const ev: *Evented = @ptrCast(@alignCast(userdata));
4270		const ev_io = ev.io();
4271		ev.stderr_mutex.lockUncancelable(ev_io);
4272		errdefer ev.stderr_mutex.unlock(ev_io);
4273		return ev.initLockedStderr(terminal_mode);
4274		}
4275
4276		fn tryLockStderr(
4277		userdata: ?*anyopaque,
4278		terminal_mode: ?Io.Terminal.Mode,
4279		) Io.Cancelable!?Io.LockedStderr {
4280		const ev: *Evented = @ptrCast(@alignCast(userdata));
4281		const ev_io = ev.io();
4282		if (!ev.stderr_mutex.tryLock()) return null;
4283		errdefer ev.stderr_mutex.unlock(ev_io);
4284		return try ev.initLockedStderr(terminal_mode);
4285		}
4286
4287		fn initLockedStderr(ev: *Evented, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
4288		if (!ev.stderr_writer_initialized) {
4289		const ev_io = ev.io();
4290		try ev.scanEnviron();
4291		const NO_COLOR = ev.environ.exist.NO_COLOR;
4292		const CLICOLOR_FORCE = ev.environ.exist.CLICOLOR_FORCE;
4293		ev.stderr_mode = terminal_mode orelse
4294		try .detect(ev_io, ev.stderr_writer.file, NO_COLOR, CLICOLOR_FORCE);
4295		ev.stderr_writer_initialized = true;
4296		}
4297		return .{
4298		.file_writer = &ev.stderr_writer,
4299		.terminal_mode = terminal_mode orelse ev.stderr_mode,
4300		};
4301		}
4302
4303		fn unlockStderr(userdata: ?*anyopaque) void {
4304		const ev: *Evented = @ptrCast(@alignCast(userdata));
4305		ev.stderr_writer.interface.flush() catch \|err\| switch (err) {
4306		error.WriteFailed => switch (ev.stderr_writer.err.?) {
4307		error.Canceled => recancel(ev),
4308		else => {},
4309		},
4310		};
4311		ev.stderr_writer.interface.end = 0;
4312		ev.stderr_writer.interface.buffer = &.{};
4313		ev.stderr_mutex.unlock(ev.io());
4314		}
4315
4316		fn processCurrentPath(userdata: ?*anyopaque, buffer: []u8) process.CurrentPathError!usize {
4317		const ev: *Evented = @ptrCast(@alignCast(userdata));
4318		var sync: CancelRegion.Sync = try .init(ev);
4319		defer sync.deinit(ev);
4320		while (true) {
4321		try sync.cancel_region.await(.nothing);
4322		switch (linux.errno(linux.getcwd(buffer.ptr, buffer.len))) {
4323		.SUCCESS => return std.mem.findScalar(u8, buffer, 0).?,
4324		.INTR => continue,
4325		.NOENT => return error.CurrentDirUnlinked,
4326		.RANGE => return error.NameTooLong,
4327		.FAULT => \|err\| return errnoBug(err),
4328		.INVAL => \|err\| return errnoBug(err),
4329		else => \|err\| return unexpectedErrno(err),
4330		}
4331		}
4332		}
4333
4334		fn processSetCurrentDir(userdata: ?*anyopaque, dir: Dir) process.SetCurrentDirError!void {
4335		const ev: *Evented = @ptrCast(@alignCast(userdata));
4336		if (dir.handle == linux.AT.FDCWD) return;
4337		var sync: CancelRegion.Sync = try .init(ev);
4338		defer sync.deinit(ev);
4339		return ev.fchdir(&sync, dir.handle);
4340		}
4341
4342		fn processSetCurrentPath(userdata: ?*anyopaque, dir_path: []const u8) ChdirError!void {
4343		const ev: *Evented = @ptrCast(@alignCast(userdata));
4344		var path_buffer: [PATH_MAX]u8 = undefined;
4345		const dir_path_posix = try pathToPosix(dir_path, &path_buffer);
4346		var sync: CancelRegion.Sync = try .init(ev);
4347		defer sync.deinit(ev);
4348		return ev.chdir(&sync, dir_path_posix);
4349		}
4350
4351		fn processReplace(userdata: ?*anyopaque, options: process.ReplaceOptions) process.ReplaceError {
4352		const ev: *Evented = @ptrCast(@alignCast(userdata));
4353
4354		try ev.scanEnviron(); // for PATH
4355		const PATH = ev.environ.string.PATH orelse default_PATH;
4356
4357		var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
4358		defer arena_allocator.deinit();
4359		const arena = arena_allocator.allocator();
4360
4361		const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
4362		for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
4363
4364		const env_block = env_block: {
4365		const prog_fd: i32 = -1;
4366		if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
4367		.zig_progress_fd = prog_fd,
4368		});
4369		break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
4370		.zig_progress_fd = prog_fd,
4371		});
4372		};
4373
4374		var sync: CancelRegion.Sync = try .init(ev);
4375		defer sync.deinit(ev);
4376		return ev.execv(&sync, options.expand_arg0, argv_buf.ptr[0].?, argv_buf.ptr, env_block, PATH);
4377		}
4378
4379		fn processReplacePath(
4380		userdata: ?*anyopaque,
4381		dir: Dir,
4382		options: process.ReplaceOptions,
4383		) process.ReplaceError {
4384		const ev: *Evented = @ptrCast(@alignCast(userdata));
4385		_ = ev;
4386		_ = dir;
4387		_ = options;
4388		@panic("TODO processReplacePath");
4389		}
4390
4391		fn processSpawn(userdata: ?*anyopaque, options: process.SpawnOptions) process.SpawnError!process.Child {
4392		const ev: *Evented = @ptrCast(@alignCast(userdata));
4393		const spawned = try ev.spawn(options);
4394		var cancel_region: CancelRegion = .initBlocked();
4395		defer cancel_region.deinit();
4396		defer ev.close(&cancel_region, spawned.err_fd);
4397
4398		// Wait for the child to report any errors in or before `execvpe`.
4399		var child_err: ForkBailError = undefined;
4400		ev.readAll(&cancel_region, spawned.err_fd, @ptrCast(&child_err)) catch \|read_err\| {
4401		switch (read_err) {
4402		error.Canceled => unreachable, // blocked
4403		error.EndOfStream => {
4404		// Write end closed by CLOEXEC at the time of the `execvpe` call,
4405		// indicating success.
4406		},
4407		else => {
4408		// Problem reading the error from the error reporting pipe. We
4409		// don't know if the child is alive or dead. Better to assume it is
4410		// alive so the resource does not risk being leaked.
4411		},
4412		}
4413		return .{
4414		.id = spawned.pid,
4415		.thread_handle = {},
4416		.stdin = spawned.stdin,
4417		.stdout = spawned.stdout,
4418		.stderr = spawned.stderr,
4419		.request_resource_usage_statistics = options.request_resource_usage_statistics,
4420		};
4421		};
4422		return child_err;
4423		}
4424
4425		fn processSpawnPath(
4426		userdata: ?*anyopaque,
4427		dir: Dir,
4428		options: process.SpawnOptions,
4429		) process.SpawnError!process.Child {
4430		const ev: *Evented = @ptrCast(@alignCast(userdata));
4431		_ = ev;
4432		_ = dir;
4433		_ = options;
4434		@panic("TODO processSpawnPath");
4435		}
4436
4437		const prog_fileno = 3;
4438
4439		const Spawned = struct {
4440		pid: pid_t,
4441		err_fd: fd_t,
4442		stdin: ?File,
4443		stdout: ?File,
4444		stderr: ?File,
4445		};
4446		fn spawn(ev: *Evented, options: process.SpawnOptions) process.SpawnError!Spawned {
4447		var cancel_region: CancelRegion = .init();
4448		defer cancel_region.deinit();
4449
4450		// The child process does need to access (one end of) these pipes. However,
4451		// we must initially set CLOEXEC to avoid a race condition. If another thread
4452		// is racing to spawn a different child process, we don't want it to inherit
4453		// these FDs in any scenario; that would mean that, for instance, calls to
4454		// `poll` from the parent would not report the child's stdout as closing when
4455		// expected, since the other child may retain a reference to the write end of
4456		// the pipe. So, we create the pipes with CLOEXEC initially. After fork, we
4457		// need to do something in the new child to make sure we preserve the reference
4458		// we want. We could use `fcntl` to remove CLOEXEC from the FD, but as it
4459		// turns out, we `dup2` everything anyway, so there's no need!
4460		const pipe_flags: linux.O = .{ .CLOEXEC = true };
4461
4462		const stdin_pipe = if (options.stdin == .pipe) try pipe2(pipe_flags) else undefined;
4463		errdefer if (options.stdin == .pipe) {
4464		ev.destroyPipe(&cancel_region, stdin_pipe);
4465		};
4466
4467		const stdout_pipe = if (options.stdout == .pipe) try pipe2(pipe_flags) else undefined;
4468		errdefer if (options.stdout == .pipe) {
4469		ev.destroyPipe(&cancel_region, stdout_pipe);
4470		};
4471
4472		const stderr_pipe = if (options.stderr == .pipe) try pipe2(pipe_flags) else undefined;
4473		errdefer if (options.stderr == .pipe) {
4474		ev.destroyPipe(&cancel_region, stderr_pipe);
4475		};
4476
4477		const any_ignore =
4478		options.stdin == .ignore or options.stdout == .ignore or options.stderr == .ignore;
4479		const dev_null_fd = if (any_ignore) try ev.null_fd.open(ev, &cancel_region, "/dev/null", .{
4480		.ACCMODE = .RDWR,
4481		}) else undefined;
4482
4483		const prog_pipe: [2]fd_t = if (options.progress_node.index != .none) pipe: {
4484		// We use CLOEXEC for the same reason as in `pipe_flags`.
4485		const pipe = try pipe2(.{ .NONBLOCK = true, .CLOEXEC = true });
4486		_ = linux.fcntl(pipe[0], linux.F.SETPIPE_SZ, @as(u32, std.Progress.max_packet_len * 2));
4487		break :pipe pipe;
4488		} else .{ -1, -1 };
4489		errdefer ev.destroyPipe(&cancel_region, prog_pipe);
4490
4491		var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
4492		defer arena_allocator.deinit();
4493		const arena = arena_allocator.allocator();
4494
4495		// The POSIX standard does not allow malloc() between fork() and execve(),
4496		// and this allocator may be a libc allocator.
4497		// I have personally observed the child process deadlocking when it tries
4498		// to call malloc() due to a heap allocation between fork() and execve(),
4499		// in musl v1.1.24.
4500		// Additionally, we want to reduce the number of possible ways things
4501		// can fail between fork() and execve().
4502		// Therefore, we do all the allocation for the execve() before the fork().
4503		// This means we must do the null-termination of argv and env vars here.
4504		const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
4505		for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
4506
4507		comptime assert(@max(linux.STDIN_FILENO, linux.STDOUT_FILENO, linux.STDERR_FILENO) + 1 == prog_fileno);
4508
4509		const env_block = env_block: {
4510		const prog_fd: i32 = if (prog_pipe[1] == -1) -1 else prog_fileno;
4511		if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
4512		.zig_progress_fd = prog_fd,
4513		});
4514		break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
4515		.zig_progress_fd = prog_fd,
4516		});
4517		};
4518
4519		// This pipe communicates to the parent errors in the child between `fork` and `execvpe`.
4520		// It is closed by the child (via CLOEXEC) without writing if `execvpe` succeeds.
4521		const err_pipe: [2]fd_t = try pipe2(.{ .CLOEXEC = true });
4522		errdefer ev.destroyPipe(&cancel_region, err_pipe);
4523
4524		try ev.scanEnviron(); // for PATH
4525		const PATH = ev.environ.string.PATH orelse default_PATH;
4526
4527		const pid_result: pid_t = fork: {
4528		const rc = linux.fork();
4529		switch (linux.errno(rc)) {
4530		.SUCCESS => break :fork @intCast(rc),
4531		.AGAIN => return error.SystemResources,
4532		.NOMEM => return error.SystemResources,
4533		.NOSYS => return error.OperationUnsupported,
4534		else => \|err\| return unexpectedErrno(err),
4535		}
4536		};
4537
4538		if (pid_result == 0) {
4539		defer comptime unreachable; // We are the child.
4540		var sync: CancelRegion.Sync = .{ .cancel_region = .initBlocked() };
4541		const err = ev.setUpChild(&sync, .{
4542		.stdin_pipe = stdin_pipe[0],
4543		.stdout_pipe = stdout_pipe[1],
4544		.stderr_pipe = stderr_pipe[1],
4545		.dev_null_fd = dev_null_fd,
4546		.prog_pipe = prog_pipe[1],
4547		.argv_buf = argv_buf,
4548		.env_block = env_block,
4549		.PATH = PATH,
4550		.spawn = options,
4551		});
4552		ev.writeAll(&sync.cancel_region, err_pipe[1], @ptrCast(&err)) catch {};
4553		const exit = if (builtin.single_threaded) linux.exit else linux.exit_group;
4554		exit(1);
4555		}
4556
4557		const pid: pid_t = @intCast(pid_result); // We are the parent.
4558		errdefer comptime unreachable; // The child is forked; we must not error from now on
4559
4560		ev.close(&cancel_region, err_pipe[1]); // make sure only the child holds the write end open
4561
4562		if (options.stdin == .pipe) ev.close(&cancel_region, stdin_pipe[0]);
4563		if (options.stdout == .pipe) ev.close(&cancel_region, stdout_pipe[1]);
4564		if (options.stderr == .pipe) ev.close(&cancel_region, stderr_pipe[1]);
4565
4566		if (prog_pipe[1] != -1) ev.close(&cancel_region, prog_pipe[1]);
4567
4568		options.progress_node.setIpcFile(ev, .{ .handle = prog_pipe[0], .flags = .{ .nonblocking = true } });
4569
4570		return .{
4571		.pid = pid,
4572		.err_fd = err_pipe[0],
4573		.stdin = switch (options.stdin) {
4574		.pipe => .{ .handle = stdin_pipe[1], .flags = .{ .nonblocking = false } },
4575		else => null,
4576		},
4577		.stdout = switch (options.stdout) {
4578		.pipe => .{ .handle = stdout_pipe[0], .flags = .{ .nonblocking = false } },
4579		else => null,
4580		},
4581		.stderr = switch (options.stderr) {
4582		.pipe => .{ .handle = stderr_pipe[0], .flags = .{ .nonblocking = false } },
4583		else => null,
4584		},
4585		};
4586		}
4587
4588		pub const PipeError = error{
4589		SystemFdQuotaExceeded,
4590		ProcessFdQuotaExceeded,
4591		} \|\| Io.UnexpectedError;
4592		pub fn pipe2(flags: linux.O) PipeError![2]fd_t {
4593		var fds: [2]fd_t = undefined;
4594		switch (linux.errno(linux.pipe2(&fds, flags))) {
4595		.SUCCESS => return fds,
4596		.INVAL => \|err\| return errnoBug(err), // Invalid flags
4597		.NFILE => return error.SystemFdQuotaExceeded,
4598		.MFILE => return error.ProcessFdQuotaExceeded,
4599		else => \|err\| return unexpectedErrno(err),
4600		}
4601		}
4602		fn destroyPipe(ev: Evented, cancel_region: CancelRegion, pipe: [2]fd_t) void {
4603		if (pipe[0] != -1) ev.close(cancel_region, pipe[0]);
4604		if (pipe[0] != pipe[1]) ev.close(cancel_region, pipe[1]);
4605		}
4606
4607		/// Errors that can occur between fork() and execv()
4608		const ForkBailError = process.SetCurrentDirError \|\| ChdirError \|\|
4609		process.SpawnError \|\| process.ReplaceError;
4610		fn setUpChild(
4611		ev: *Evented,
4612		sync: *CancelRegion.Sync,
4613		options: struct {
4614		stdin_pipe: fd_t,
4615		stdout_pipe: fd_t,
4616		stderr_pipe: fd_t,
4617		dev_null_fd: fd_t,
4618		prog_pipe: fd_t,
4619		argv_buf: [:null]?[*:0]const u8,
4620		env_block: process.Environ.Block,
4621		PATH: []const u8,
4622		spawn: process.SpawnOptions,
4623		},
4624		) ForkBailError {
4625		try ev.setUpChildIo(
4626		sync,
4627		options.spawn.stdin,
4628		options.stdin_pipe,
4629		linux.STDIN_FILENO,
4630		options.dev_null_fd,
4631		);
4632		try ev.setUpChildIo(
4633		sync,
4634		options.spawn.stdout,
4635		options.stdout_pipe,
4636		linux.STDOUT_FILENO,
4637		options.dev_null_fd,
4638		);
4639		try ev.setUpChildIo(
4640		sync,
4641		options.spawn.stderr,
4642		options.stderr_pipe,
4643		linux.STDERR_FILENO,
4644		options.dev_null_fd,
4645		);
4646
4647		switch (options.spawn.cwd) {
4648		.inherit => {},
4649		.dir => \|cwd_dir\| try ev.fchdir(sync, cwd_dir.handle),
4650		.path => \|cwd_path\| {
4651		var cwd_path_buffer: [PATH_MAX]u8 = undefined;
4652		const cwd_path_posix = try pathToPosix(cwd_path, &cwd_path_buffer);
4653		try ev.chdir(sync, cwd_path_posix);
4654		},
4655		}
4656
4657		// Must happen after fchdir above, the cwd file descriptor might be
4658		// equal to prog_fileno and be clobbered by this dup2 call.
4659		if (options.prog_pipe != -1) try ev.dup2(sync, options.prog_pipe, prog_fileno);
4660
4661		if (options.spawn.gid) \|gid\| {
4662		switch (linux.errno(linux.setregid(gid, gid))) {
4663		.SUCCESS => {},
4664		.AGAIN => return error.ResourceLimitReached,
4665		.INVAL => return error.InvalidUserId,
4666		.PERM => return error.PermissionDenied,
4667		else => return error.Unexpected,
4668		}
4669		}
4670
4671		if (options.spawn.uid) \|uid\| {
4672		switch (linux.errno(linux.setreuid(uid, uid))) {
4673		.SUCCESS => {},
4674		.AGAIN => return error.ResourceLimitReached,
4675		.INVAL => return error.InvalidUserId,
4676		.PERM => return error.PermissionDenied,
4677		else => return error.Unexpected,
4678		}
4679		}
4680
4681		if (options.spawn.pgid) \|pid\| {
4682		switch (linux.errno(linux.setpgid(0, pid))) {
4683		.SUCCESS => {},
4684		.ACCES => return error.ProcessAlreadyExec,
4685		.INVAL => return error.InvalidProcessGroupId,
4686		.PERM => return error.PermissionDenied,
4687		else => return error.Unexpected,
4688		}
4689		}
4690
4691		if (options.spawn.start_suspended) {
4692		switch (linux.errno(linux.kill(linux.getpid(), .STOP))) {
4693		.SUCCESS => {},
4694		.PERM => return error.PermissionDenied,
4695		else => return error.Unexpected,
4696		}
4697		}
4698
4699		return ev.execv(
4700		sync,
4701		options.spawn.expand_arg0,
4702		options.argv_buf.ptr[0].?,
4703		options.argv_buf.ptr,
4704		options.env_block,
4705		options.PATH,
4706		);
4707		}
4708
4709		fn setUpChildIo(
4710		ev: *Evented,
4711		sync: *CancelRegion.Sync,
4712		stdio: process.SpawnOptions.StdIo,
4713		pipe_fd: fd_t,
4714		std_fileno: i32,
4715		dev_null_fd: fd_t,
4716		) !void {
4717		switch (stdio) {
4718		.pipe => try ev.dup2(sync, pipe_fd, std_fileno),
4719		.close => ev.close(&sync.cancel_region, std_fileno),
4720		.inherit => {},
4721		.ignore => try ev.dup2(sync, dev_null_fd, std_fileno),
4722		.file => \|file\| try ev.dup2(sync, file.handle, std_fileno),
4723		}
4724		}
4725
4726		pub const DupError = error{
4727		ProcessFdQuotaExceeded,
4728		SystemResources,
4729		} \|\| Io.UnexpectedError \|\| Io.Cancelable;
4730		pub fn dup2(ev: Evented, sync: CancelRegion.Sync, old_fd: fd_t, new_fd: fd_t) DupError!void {
4731		_ = ev;
4732		while (true) {
4733		try sync.cancel_region.await(.nothing);
4734		switch (linux.errno(linux.dup2(old_fd, new_fd))) {
4735		.SUCCESS => {},
4736		.BUSY, .INTR => continue,
4737		.INVAL => \|err\| return errnoBug(err), // invalid parameters
4738		.BADF => \|err\| return errnoBug(err), // use after free
4739		.MFILE => return error.ProcessFdQuotaExceeded,
4740		.NOMEM => return error.SystemResources,
4741		else => \|err\| return unexpectedErrno(err),
4742		}
4743		}
4744		}
4745
4746		fn execv(
4747		ev: *Evented,
4748		sync: *CancelRegion.Sync,
4749		arg0_expand: process.ArgExpansion,
4750		file: [*:0]const u8,
4751		child_argv: [:null]?[:0]const u8,
4752		env_block: process.Environ.PosixBlock,
4753		PATH: []const u8,
4754		) process.ReplaceError {
4755		const file_slice = std.mem.sliceTo(file, 0);
4756		if (std.mem.findScalar(u8, file_slice, '/') != null) return ev.execvPath(sync, file, child_argv, env_block);
4757
4758		// Use of PATH_MAX here is valid as the path_buf will be passed
4759		// directly to the operating system in posixExecvPath.
4760		var path_buf: [PATH_MAX]u8 = undefined;
4761		var it = std.mem.tokenizeScalar(u8, PATH, ':');
4762		var seen_eacces = false;
4763		var err: process.ReplaceError = error.FileNotFound;
4764
4765		// In case of expanding arg0 we must put it back if we return with an error.
4766		const prev_arg0 = child_argv[0];
4767		defer switch (arg0_expand) {
4768		.expand => child_argv[0] = prev_arg0,
4769		.no_expand => {},
4770		};
4771
4772		while (it.next()) \|search_path\| {
4773		const path_len = search_path.len + file_slice.len + 1;
4774		if (path_buf.len < path_len + 1) return error.NameTooLong;
4775		@memcpy(path_buf[0..search_path.len], search_path);
4776		path_buf[search_path.len] = '/';
4777		@memcpy(path_buf[search_path.len + 1 ..][0..file_slice.len], file_slice);
4778		path_buf[path_len] = 0;
4779		const full_path = path_buf[0..path_len :0].ptr;
4780		switch (arg0_expand) {
4781		.expand => child_argv[0] = full_path,
4782		.no_expand => {},
4783		}
4784		err = ev.execvPath(sync, full_path, child_argv, env_block);
4785		switch (err) {
4786		error.AccessDenied => seen_eacces = true,
4787		error.FileNotFound, error.NotDir => {},
4788		else => \|e\| return e,
4789		}
4790		}
4791		if (seen_eacces) return error.AccessDenied;
4792		return err;
4793		}
4794		/// This function ignores PATH environment variable.
4795		pub fn execvPath(
4796		ev: *Evented,
4797		sync: *CancelRegion.Sync,
4798		path: [*:0]const u8,
4799		child_argv: [:null]const ?[:0]const u8,
4800		env_block: process.Environ.PosixBlock,
4801		) process.ReplaceError {
4802		_ = ev;
4803		try sync.cancel_region.await(.nothing);
4804		switch (linux.errno(linux.execve(path, child_argv, env_block.slice.ptr))) {
4805		.FAULT => \|err\| return errnoBug(err), // Bad pointer parameter.
4806		.@"2BIG" => return error.SystemResources,
4807		.MFILE => return error.ProcessFdQuotaExceeded,
4808		.NAMETOOLONG => return error.NameTooLong,
4809		.NFILE => return error.SystemFdQuotaExceeded,
4810		.NOMEM => return error.SystemResources,
4811		.ACCES => return error.AccessDenied,
4812		.PERM => return error.PermissionDenied,
4813		.INVAL => return error.InvalidExe,
4814		.NOEXEC => return error.InvalidExe,
4815		.IO => return error.FileSystem,
4816		.LOOP => return error.FileSystem,
4817		.ISDIR => return error.IsDir,
4818		.NOENT => return error.FileNotFound,
4819		.NOTDIR => return error.NotDir,
4820		.TXTBSY => return error.FileBusy,
4821		.LIBBAD => return error.InvalidExe,
4822		else => \|err\| return unexpectedErrno(err),
4823		}
4824		}
4825
4826		fn childWait(userdata: ?anyopaque, child: process.Child) process.Child.WaitError!process.Child.Term {
4827		const ev: *Evented = @ptrCast(@alignCast(userdata));
4828
4829		var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
4830		defer maybe_sync.deinit(ev);
4831		defer ev.childCleanup(maybe_sync.cancelRegion(), child);
4832
4833		const pid = child.id.?;
4834		var info: linux.siginfo_t = undefined;
4835		while (true) {
4836		const thread = try maybe_sync.cancel_region.awaitIoUring();
4837		thread.enqueue().* = .{
4838		.opcode = .WAITID,
4839		.flags = 0,
4840		.ioprio = 0,
4841		.fd = pid,
4842		.off = @intFromPtr(&info),
4843		.addr = 0,
4844		.len = @intFromEnum(linux.P.PID),
4845		.rw_flags = 0,
4846		.user_data = @intFromPtr(maybe_sync.cancel_region.fiber),
4847		.buf_index = 0,
4848		.personality = 0,
4849		.splice_fd_in = linux.W.EXITED \|
4850		@as(i32, if (child.request_resource_usage_statistics) linux.W.NOWAIT else 0),
4851		.addr3 = 0,
4852		.resv = 0,
4853		};
4854		ev.yield(null, .nothing);
4855		switch (maybe_sync.cancel_region.errno()) {
4856		.SUCCESS => {
4857		if (child.request_resource_usage_statistics) {
4858		const sync = try maybe_sync.enterSync(ev);
4859		while (true) {
4860		try sync.cancel_region.await(.nothing);
4861		var rusage: linux.rusage = undefined;
4862		switch (linux.errno(linux.waitid(
4863		.PID,
4864		pid,
4865		&info,
4866		linux.W.EXITED \| linux.W.NOHANG,
4867		&rusage,
4868		))) {
4869		.SUCCESS => {
4870		child.resource_usage_statistics.rusage = rusage;
4871		break;
4872		},
4873		.INTR, .CANCELED => continue,
4874		.CHILD => \|err\| return errnoBug(err), // Double-free.
4875		else => \|err\| return unexpectedErrno(err),
4876		}
4877		}
4878		}
4879		const status: u32 = @bitCast(info.fields.common.second.sigchld.status);
4880		const code: linux.CLD = @enumFromInt(info.code);
4881		return switch (code) {
4882		.EXITED => .{ .exited = @truncate(status) },
4883		.KILLED, .DUMPED => .{ .signal = @enumFromInt(status) },
4884		.TRAPPED, .STOPPED => .{ .stopped = status },
4885		_, .CONTINUED => .{ .unknown = status },
4886		};
4887		},
4888		.INTR, .CANCELED => continue,
4889		.CHILD => \|err\| return errnoBug(err), // Double-free.
4890		else => \|err\| return unexpectedErrno(err),
4891		}
4892		}
4893		}
4894
4895		fn childKill(userdata: ?anyopaque, child: process.Child) void {
4896		const ev: *Evented = @ptrCast(@alignCast(userdata));
4897
4898		var maybe_sync: CancelRegion.Sync.Maybe = .{ .sync = .initBlocked(ev) };
4899		defer maybe_sync.deinit(ev);
4900		defer ev.childCleanup(maybe_sync.cancelRegion(), child);
4901
4902		const pid = child.id.?;
4903		while (true) switch (linux.errno(linux.kill(pid, .TERM))) {
4904		.SUCCESS => break,
4905		.INTR => continue,
4906		.PERM => return,
4907		.INVAL => \|err\| return errnoBug(err) catch {},
4908		.SRCH => \|err\| return errnoBug(err) catch {},
4909		else => \|err\| return unexpectedErrno(err) catch {},
4910		};
4911		maybe_sync.leaveSync(ev);
4912
4913		var info: linux.siginfo_t = undefined;
4914		while (true) {
4915		const thread = maybe_sync.cancel_region.awaitIoUring() catch \|err\| switch (err) {
4916		error.Canceled => unreachable, // blocked
4917		};
4918		thread.enqueue().* = .{
4919		.opcode = .WAITID,
4920		.flags = 0,
4921		.ioprio = 0,
4922		.fd = pid,
4923		.off = @intFromPtr(&info),
4924		.addr = 0,
4925		.len = @intFromEnum(linux.P.PID),
4926		.rw_flags = 0,
4927		.user_data = @intFromPtr(maybe_sync.cancel_region.fiber),
4928		.buf_index = 0,
4929		.personality = 0,
4930		.splice_fd_in = linux.W.EXITED,
4931		.addr3 = 0,
4932		.resv = 0,
4933		};
4934		ev.yield(null, .nothing);
4935		switch (maybe_sync.cancel_region.errno()) {
4936		.SUCCESS => return,
4937		.INTR, .CANCELED => continue,
4938		.CHILD => \|err\| return errnoBug(err) catch {}, // Double-free.
4939		else => \|err\| return unexpectedErrno(err) catch {},
4940		}
4941		}
4942		}
4943
4944		fn childCleanup(ev: Evented, cancel_region: CancelRegion, child: *process.Child) void {
4945		if (child.stdin) \|*stdin\| {
4946		ev.close(cancel_region, stdin.handle);
4947		child.stdin = null;
4948		}
4949		if (child.stdout) \|*stdout\| {
4950		ev.close(cancel_region, stdout.handle);
4951		child.stdout = null;
4952		}
4953		if (child.stderr) \|*stderr\| {
4954		ev.close(cancel_region, stderr.handle);
4955		child.stderr = null;
4956		}
4957		child.id = null;
4958		}
4959
4960		fn progressParentFile(userdata: ?*anyopaque) std.Progress.ParentFileError!File {
4961		const ev: *Evented = @ptrCast(@alignCast(userdata));
4962		const cancel_protection = swapCancelProtection(ev, .blocked);
4963		defer assert(swapCancelProtection(ev, cancel_protection) == .blocked);
4964		ev.scanEnviron() catch \|err\| switch (err) {
4965		error.Canceled => unreachable, // blocked
4966		};
4967		return ev.environ.zig_progress_file;
4968		}
4969
4970		fn scanEnviron(ev: *Evented) Io.Cancelable!void {
4971		const ev_io = ev.io();
4972		try ev.environ_mutex.lock(ev_io);
4973		defer ev.environ_mutex.unlock(ev_io);
4974		ev.environ.scan(ev.allocator());
4975		}
4976
4977		fn clockResolution(userdata: ?*anyopaque, clock: Io.Clock) Io.Clock.ResolutionError!Io.Duration {
4978		const ev: *Evented = @ptrCast(@alignCast(userdata));
4979		_ = ev;
4980		const clock_id = clockToPosix(clock);
4981		var timespec: linux.timespec = undefined;
4982		return switch (linux.errno(linux.clock_getres(clock_id, &timespec))) {
4983		.SUCCESS => .fromNanoseconds(nanosecondsFromPosix(&timespec)),
4984		.INVAL => return error.ClockUnavailable,
4985		else => \|err\| return unexpectedErrno(err),
4986		};
4987		}
4988
4989		fn now(userdata: ?*anyopaque, clock: Io.Clock) Io.Timestamp {
4990		const ev: *Evented = @ptrCast(@alignCast(userdata));
4991		_ = ev;
4992		var tp: linux.timespec = undefined;
4993		switch (linux.errno(linux.clock_gettime(clockToPosix(clock), &tp))) {
4994		.SUCCESS => return timestampFromPosix(&tp),
4995		else => return .zero,
4996		}
4997		}
4998
4999		fn sleep(userdata: ?*anyopaque, timeout: Io.Timeout) Io.Cancelable!void {
5000		const ev: *Evented = @ptrCast(@alignCast(userdata));
5001
5002		const timespec: linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = timespec: switch (timeout) {
5003		.none => .{
5004		.{
5005		.sec = std.math.maxInt(i64),
5006		.nsec = std.time.ns_per_s - 1,
5007		},
5008		.awake,
5009		linux.IORING_TIMEOUT_ABS,
5010		},
5011		.duration => \|duration\| {
5012		const ns = duration.raw.toNanoseconds();
5013		break :timespec .{
5014		.{
5015		.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
5016		.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
5017		},
5018		duration.clock,
5019		0,
5020		};
5021		},
5022		.deadline => \|deadline\| {
5023		const ns = deadline.raw.toNanoseconds();
5024		break :timespec .{
5025		.{
5026		.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
5027		.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
5028		},
5029		deadline.clock,
5030		linux.IORING_TIMEOUT_ABS,
5031		};
5032		},
5033		};
5034		var cancel_region: CancelRegion = .init();
5035		defer cancel_region.deinit();
5036		const thread = try cancel_region.awaitIoUring();
5037		thread.enqueue().* = .{
5038		.opcode = .TIMEOUT,
5039		.flags = 0,
5040		.ioprio = 0,
5041		.fd = 0,
5042		.off = 0,
5043		.addr = @intFromPtr(&timespec),
5044		.len = 1,
5045		.rw_flags = timeout_flags \| @as(u32, switch (clock) {
5046		.real => linux.IORING_TIMEOUT_REALTIME,
5047		else => 0,
5048		.boot => linux.IORING_TIMEOUT_BOOTTIME,
5049		}),
5050		.user_data = @intFromPtr(cancel_region.fiber),
5051		.buf_index = 0,
5052		.personality = 0,
5053		.splice_fd_in = 0,
5054		.addr3 = 0,
5055		.resv = 0,
5056		};
5057		ev.yield(null, .nothing);
5058		switch (cancel_region.errno()) {
5059		// Handles SUCCESS as well as clock not available and unexpected
5060		// errors. The user had a chance to check clock resolution before
5061		// getting here, which would have reported 0, making this a legal
5062		// amount of time to sleep.
5063		else => return,
5064		.INTR, .CANCELED => return error.Canceled,
5065		}
5066		}
5067
5068		fn random(userdata: ?*anyopaque, buffer: []u8) void {
5069		const ev: *Evented = @ptrCast(@alignCast(userdata));
5070		var thread: *Thread = .current();
5071		if (!thread.csprng.isInitialized()) {
5072		@branchHint(.unlikely);
5073		var seed: [Csprng.seed_len]u8 = undefined;
5074		{
5075		const ev_io = ev.io();
5076		ev.csprng_mutex.lockUncancelable(ev_io);
5077		defer ev.csprng_mutex.unlock(ev_io);
5078		if (!ev.csprng.isInitialized()) {
5079		@branchHint(.unlikely);
5080		var cancel_region: CancelRegion = .initBlocked();
5081		defer cancel_region.deinit();
5082		ev.urandomReadAll(&cancel_region, &seed) catch \|err\| switch (err) {
5083		error.Canceled => unreachable, // blocked
5084		else => fallbackSeed(ev, &seed),
5085		};
5086		ev.csprng.rng = .init(seed);
5087		thread = .current();
5088		}
5089		ev.csprng.rng.fill(&seed);
5090		}
5091		if (!thread.csprng.isInitialized()) {
5092		@branchHint(.likely);
5093		thread.csprng.rng = .init(seed);
5094		} else thread.csprng.rng.addEntropy(&seed);
5095		}
5096		thread.csprng.rng.fill(buffer);
5097		}
5098
5099		fn randomSecure(userdata: ?*anyopaque, buffer: []u8) Io.RandomSecureError!void {
5100		const ev: *Evented = @ptrCast(@alignCast(userdata));
5101		if (buffer.len == 0) return;
5102		var cancel_region: CancelRegion = .init();
5103		defer cancel_region.deinit();
5104		ev.urandomReadAll(&cancel_region, buffer) catch \|err\| switch (err) {
5105		error.Canceled => return error.Canceled,
5106		else => return error.EntropyUnavailable,
5107		};
5108		}
5109
5110		fn netListenIpUnavailable(
5111		userdata: ?*anyopaque,
5112		address: net.IpAddress,
5113		options: net.IpAddress.ListenOptions,
5114		) net.IpAddress.ListenError!net.Server {
5115		const ev: *Evented = @ptrCast(@alignCast(userdata));
5116		_ = ev;
5117		_ = address;
5118		_ = options;
5119		return error.NetworkDown;
5120		}
5121
5122		fn netAcceptUnavailable(
5123		userdata: ?*anyopaque,
5124		listen_handle: net.Socket.Handle,
5125		) net.Server.AcceptError!net.Stream {
5126		const ev: *Evented = @ptrCast(@alignCast(userdata));
5127		_ = ev;
5128		_ = listen_handle;
5129		return error.NetworkDown;
5130		}
5131
5132		fn netBindIp(
5133		userdata: ?*anyopaque,
5134		address: *const net.IpAddress,
5135		options: net.IpAddress.BindOptions,
5136		) net.IpAddress.BindError!net.Socket {
5137		const ev: *Evented = @ptrCast(@alignCast(userdata));
5138		const family = posixAddressFamily(address);
5139		var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
5140		defer maybe_sync.deinit(ev);
5141		const socket_fd = try ev.socket(&maybe_sync.cancel_region, family, options);
5142		errdefer ev.close(maybe_sync.cancelRegion(), socket_fd);
5143		var storage: PosixAddress = undefined;
5144		var addr_len = addressToPosix(address, &storage);
5145		try ev.bind(&maybe_sync.cancel_region, socket_fd, &storage.any, addr_len);
5146		try ev.getsockname(try maybe_sync.enterSync(ev), socket_fd, &storage.any, &addr_len);
5147		return .{
5148		.handle = socket_fd,
5149		.address = addressFromPosix(&storage),
5150		};
5151		}
5152
5153		fn netBindIpUnavailable(
5154		userdata: ?*anyopaque,
5155		address: *const net.IpAddress,
5156		options: net.IpAddress.BindOptions,
5157		) net.IpAddress.BindError!net.Socket {
5158		const ev: *Evented = @ptrCast(@alignCast(userdata));
5159		_ = ev;
5160		_ = address;
5161		_ = options;
5162		return error.NetworkDown;
5163		}
5164
5165		fn netConnectIpUnavailable(
5166		userdata: ?*anyopaque,
5167		address: *const net.IpAddress,
5168		options: net.IpAddress.ConnectOptions,
5169		) net.IpAddress.ConnectError!net.Stream {
5170		const ev: *Evented = @ptrCast(@alignCast(userdata));
5171		_ = ev;
5172		_ = address;
5173		_ = options;
5174		return error.NetworkDown;
5175		}
5176
5177		fn netListenUnixUnavailable(
5178		userdata: ?*anyopaque,
5179		address: *const net.UnixAddress,
5180		options: net.UnixAddress.ListenOptions,
5181		) net.UnixAddress.ListenError!net.Socket.Handle {
5182		const ev: *Evented = @ptrCast(@alignCast(userdata));
5183		_ = ev;
5184		_ = address;
5185		_ = options;
5186		return error.AddressFamilyUnsupported;
5187		}
5188
5189		fn netConnectUnixUnavailable(
5190		userdata: ?*anyopaque,
5191		address: *const net.UnixAddress,
5192		) net.UnixAddress.ConnectError!net.Socket.Handle {
5193		const ev: *Evented = @ptrCast(@alignCast(userdata));
5194		_ = ev;
5195		_ = address;
5196		return error.AddressFamilyUnsupported;
5197		}
5198
5199		fn netSocketCreatePairUnavailable(
5200		userdata: ?*anyopaque,
5201		options: net.Socket.CreatePairOptions,
5202		) net.Socket.CreatePairError![2]net.Socket {
5203		_ = userdata;
5204		_ = options;
5205		return error.OperationUnsupported;
5206		}
5207
5208		fn netSendUnavailable(
5209		userdata: ?*anyopaque,
5210		handle: net.Socket.Handle,
5211		messages: []net.OutgoingMessage,
5212		flags: net.SendFlags,
5213		) struct { ?net.Socket.SendError, usize } {
5214		const ev: *Evented = @ptrCast(@alignCast(userdata));
5215		_ = ev;
5216		_ = handle;
5217		_ = messages;
5218		_ = flags;
5219		return .{ error.NetworkDown, 0 };
5220		}
5221
5222		fn netReceive(
5223		userdata: ?*anyopaque,
5224		handle: net.Socket.Handle,
5225		message_buffer: []net.IncomingMessage,
5226		data_buffer: []u8,
5227		flags: net.ReceiveFlags,
5228		timeout: Io.Timeout,
5229		) struct { ?net.Socket.ReceiveTimeoutError, usize } {
5230		const ev: *Evented = @ptrCast(@alignCast(userdata));
5231		const ev_io = ev.io();
5232
5233		var message_i: usize = 0;
5234		var data_i: usize = 0;
5235
5236		const deadline: ?struct {
5237		raw: Io.Timestamp,
5238		timespec: linux.kernel_timespec,
5239		clock: Io.Clock,
5240		} = if (timeout.toTimestamp(ev_io)) \|deadline\| deadline: {
5241		const ns = deadline.raw.toNanoseconds();
5242		break :deadline .{
5243		.raw = deadline.raw,
5244		.timespec = .{
5245		.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
5246		.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
5247		},
5248		.clock = deadline.clock,
5249		};
5250		} else null;
5251
5252		var cancel_region: CancelRegion = .init();
5253		defer cancel_region.deinit();
5254		while (true) {
5255		if (message_buffer.len - message_i == 0) return .{ null, message_i };
5256		const message = &message_buffer[message_i];
5257		const remaining_data_buffer = data_buffer[data_i..];
5258		var storage: PosixAddress = undefined;
5259		var iov: iovec = .{ .base = remaining_data_buffer.ptr, .len = remaining_data_buffer.len };
5260		var msg: linux.msghdr = .{
5261		.name = &storage.any,
5262		.namelen = @sizeOf(PosixAddress),
5263		.iov = (&iov)[0..1],
5264		.iovlen = 1,
5265		.control = message.control.ptr,
5266		.controllen = @intCast(message.control.len),
5267		.flags = undefined,
5268		};
5269
5270		const thread = cancel_region.awaitIoUring() catch \|err\| return .{ err, message_i };
5271		thread.enqueue().* = .{
5272		.opcode = .RECVMSG,
5273		.flags = if (deadline) \|_\| linux.IOSQE_IO_LINK else 0,
5274		.ioprio = 0,
5275		.fd = handle,
5276		.off = 0,
5277		.addr = @intFromPtr(&msg),
5278		.len = 0,
5279		.rw_flags = linux.MSG.NOSIGNAL \|
5280		@as(u32, if (flags.oob) linux.MSG.OOB else 0) \|
5281		@as(u32, if (flags.peek) linux.MSG.PEEK else 0) \|
5282		@as(u32, if (flags.trunc) linux.MSG.TRUNC else 0),
5283		.user_data = @intFromPtr(cancel_region.fiber),
5284		.buf_index = 0,
5285		.personality = 0,
5286		.splice_fd_in = 0,
5287		.addr3 = 0,
5288		.resv = 0,
5289		};
5290		if (deadline) \|deadline_ptr\| thread.enqueue(). = .{
5291		.opcode = .LINK_TIMEOUT,
5292		.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
5293		.ioprio = 0,
5294		.fd = 0,
5295		.off = 0,
5296		.addr = @intFromPtr(&deadline_ptr.timespec),
5297		.len = 1,
5298		.rw_flags = linux.IORING_TIMEOUT_ABS \| @as(u32, switch (deadline_ptr.clock) {
5299		.real => linux.IORING_TIMEOUT_REALTIME,
5300		else => 0,
5301		.boot => linux.IORING_TIMEOUT_BOOTTIME,
5302		}),
5303		.user_data = @intFromEnum(Completion.UserData.wakeup),
5304		.buf_index = 0,
5305		.personality = 0,
5306		.splice_fd_in = 0,
5307		.addr3 = 0,
5308		.resv = 0,
5309		};
5310		ev.yield(null, .nothing);
5311		const completion = cancel_region.completion();
5312		switch (completion.errno()) {
5313		.SUCCESS => {
5314		const data = remaining_data_buffer[0..@intCast(completion.result)];
5315		data_i += data.len;
5316		message.* = .{
5317		.from = addressFromPosix(&storage),
5318		.data = data,
5319		.control = if (msg.control) \|ptr\| @as([*]u8, @ptrCast(ptr))[0..msg.controllen] else message.control,
5320		.flags = .{
5321		.eor = (msg.flags & linux.MSG.EOR) != 0,
5322		.trunc = (msg.flags & linux.MSG.TRUNC) != 0,
5323		.ctrunc = (msg.flags & linux.MSG.CTRUNC) != 0,
5324		.oob = (msg.flags & linux.MSG.OOB) != 0,
5325		.errqueue = if (@hasDecl(linux.MSG, "ERRQUEUE")) (msg.flags & linux.MSG.ERRQUEUE) != 0 else false,
5326		},
5327		};
5328		message_i += 1;
5329		continue;
5330		},
5331		.AGAIN => unreachable,
5332		.INTR, .CANCELED => {
5333		if (deadline) \|d\| {
5334		if (now(ev, d.clock).nanoseconds >= d.raw.nanoseconds) return .{ error.Timeout, message_i };
5335		}
5336		continue;
5337		},
5338
5339		.BADF => \|err\| return .{ errnoBug(err), message_i },
5340		.NFILE => return .{ error.SystemFdQuotaExceeded, message_i },
5341		.MFILE => return .{ error.ProcessFdQuotaExceeded, message_i },
5342		.FAULT => \|err\| return .{ errnoBug(err), message_i },
5343		.INVAL => \|err\| return .{ errnoBug(err), message_i },
5344		.NOBUFS => return .{ error.SystemResources, message_i },
5345		.NOMEM => return .{ error.SystemResources, message_i },
5346		.NOTCONN => return .{ error.SocketUnconnected, message_i },
5347		.NOTSOCK => \|err\| return .{ errnoBug(err), message_i },
5348		.MSGSIZE => return .{ error.MessageOversize, message_i },
5349		.PIPE => return .{ error.SocketUnconnected, message_i },
5350		.OPNOTSUPP => \|err\| return .{ errnoBug(err), message_i },
5351		.CONNRESET => return .{ error.ConnectionResetByPeer, message_i },
5352		.NETDOWN => return .{ error.NetworkDown, message_i },
5353		else => \|err\| return .{ unexpectedErrno(err), message_i },
5354		}
5355		}
5356		}
5357
5358		fn netReceiveUnavailable(
5359		userdata: ?*anyopaque,
5360		handle: net.Socket.Handle,
5361		message_buffer: []net.IncomingMessage,
5362		data_buffer: []u8,
5363		flags: net.ReceiveFlags,
5364		timeout: Io.Timeout,
5365		) struct { ?net.Socket.ReceiveTimeoutError, usize } {
5366		const ev: *Evented = @ptrCast(@alignCast(userdata));
5367		_ = ev;
5368		_ = handle;
5369		_ = message_buffer;
5370		_ = data_buffer;
5371		_ = flags;
5372		_ = timeout;
5373		return .{ error.NetworkDown, 0 };
5374		}
5375
5376		fn netReadUnavailable(
5377		userdata: ?*anyopaque,
5378		fd: net.Socket.Handle,
5379		data: [][]u8,
5380		) net.Stream.Reader.Error!usize {
5381		const ev: *Evented = @ptrCast(@alignCast(userdata));
5382		_ = ev;
5383		_ = fd;
5384		_ = data;
5385		return error.NetworkDown;
5386		}
5387
5388		fn netWriteUnavailable(
5389		userdata: ?*anyopaque,
5390		handle: net.Socket.Handle,
5391		header: []const u8,
5392		data: []const []const u8,
5393		splat: usize,
5394		) net.Stream.Writer.Error!usize {
5395		const ev: *Evented = @ptrCast(@alignCast(userdata));
5396		_ = ev;
5397		_ = handle;
5398		_ = header;
5399		_ = data;
5400		_ = splat;
5401		return error.NetworkDown;
5402		}
5403
5404		fn netWriteFileUnavailable(
5405		userdata: ?*anyopaque,
5406		socket_handle: net.Socket.Handle,
5407		header: []const u8,
5408		file_reader: *File.Reader,
5409		limit: Io.Limit,
5410		) net.Stream.Writer.WriteFileError!usize {
5411		const ev: *Evented = @ptrCast(@alignCast(userdata));
5412		_ = ev;
5413		_ = socket_handle;
5414		_ = header;
5415		_ = file_reader;
5416		_ = limit;
5417		return error.NetworkDown;
5418		}
5419
5420		fn netClose(userdata: ?*anyopaque, handles: []const net.Socket.Handle) void {
5421		const ev: *Evented = @ptrCast(@alignCast(userdata));
5422		var cancel_region: CancelRegion = .init();
5423		defer cancel_region.deinit();
5424		for (handles) \|handle\| ev.close(&cancel_region, handle);
5425		}
5426
5427		fn netCloseUnavailable(userdata: ?*anyopaque, handles: []const net.Socket.Handle) void {
5428		const ev: *Evented = @ptrCast(@alignCast(userdata));
5429		_ = ev;
5430		_ = handles;
5431		unreachable; // How you gonna close something that was impossible to open?
5432		}
5433
5434		fn netShutdown(
5435		userdata: ?*anyopaque,
5436		handle: net.Socket.Handle,
5437		how: net.ShutdownHow,
5438		) net.ShutdownError!void {
5439		const ev: *Evented = @ptrCast(@alignCast(userdata));
5440		var cancel_region: CancelRegion = .init();
5441		defer cancel_region.deinit();
5442		while (true) {
5443		const thread = try cancel_region.awaitIoUring();
5444		thread.enqueue().* = .{
5445		.opcode = .SHUTDOWN,
5446		.flags = 0,
5447		.ioprio = 0,
5448		.fd = handle,
5449		.off = 0,
5450		.addr = 0,
5451		.len = switch (how) {
5452		.recv => linux.SHUT.RD,
5453		.send => linux.SHUT.WR,
5454		.both => linux.SHUT.RDWR,
5455		},
5456		.rw_flags = 0,
5457		.user_data = @intFromPtr(cancel_region.fiber),
5458		.buf_index = 0,
5459		.personality = 0,
5460		.splice_fd_in = 0,
5461		.addr3 = 0,
5462		.resv = 0,
5463		};
5464		ev.yield(null, .nothing);
5465		switch (cancel_region.errno()) {
5466		.SUCCESS => return,
5467		.INTR, .CANCELED => continue,
5468		.BADF, .NOTSOCK, .INVAL => \|err\| return errnoBug(err),
5469		.NOTCONN => return error.SocketUnconnected,
5470		.NOBUFS => return error.SystemResources,
5471		else => \|err\| return unexpectedErrno(err),
5472		}
5473		}
5474		}
5475
5476		fn netShutdownUnavailable(
5477		userdata: ?*anyopaque,
5478		handle: net.Socket.Handle,
5479		how: net.ShutdownHow,
5480		) net.ShutdownError!void {
5481		const ev: *Evented = @ptrCast(@alignCast(userdata));
5482		_ = ev;
5483		_ = handle;
5484		_ = how;
5485		unreachable; // How you gonna shutdown something that was impossible to open?
5486		}
5487
5488		fn netInterfaceNameResolveUnavailable(
5489		userdata: ?*anyopaque,
5490		name: *const net.Interface.Name,
5491		) net.Interface.Name.ResolveError!net.Interface {
5492		const ev: *Evented = @ptrCast(@alignCast(userdata));
5493		_ = ev;
5494		_ = name;
5495		return error.InterfaceNotFound;
5496		}
5497
5498		fn netInterfaceNameUnavailable(
5499		userdata: ?*anyopaque,
5500		interface: net.Interface,
5501		) net.Interface.NameError!net.Interface.Name {
5502		const ev: *Evented = @ptrCast(@alignCast(userdata));
5503		_ = ev;
5504		_ = interface;
5505		return error.Unexpected;
5506		}
5507
5508		fn netLookupUnavailable(
5509		userdata: ?*anyopaque,
5510		host_name: net.HostName,
5511		resolved: *Io.Queue(net.HostName.LookupResult),
5512		options: net.HostName.LookupOptions,
5513		) net.HostName.LookupError!void {
5514		const ev: *Evented = @ptrCast(@alignCast(userdata));
5515		_ = host_name;
5516		_ = options;
5517		resolved.close(ev.io());
5518		return error.NetworkDown;
5519		}
5520
5521		fn bind(
5522		ev: *Evented,
5523		cancel_region: *CancelRegion,
5524		socket_fd: fd_t,
5525		addr: *const linux.sockaddr,
5526		addr_len: linux.socklen_t,
5527		) !void {
5528		while (true) {
5529		const thread = try cancel_region.awaitIoUring();
5530		thread.enqueue().* = .{
5531		.opcode = .BIND,
5532		.flags = 0,
5533		.ioprio = 0,
5534		.fd = socket_fd,
5535		.off = addr_len,
5536		.addr = @intFromPtr(addr),
5537		.len = 0,
5538		.rw_flags = 0,
5539		.user_data = @intFromPtr(cancel_region.fiber),
5540		.buf_index = 0,
5541		.personality = 0,
5542		.splice_fd_in = 0,
5543		.addr3 = 0,
5544		.resv = 0,
5545		};
5546		ev.yield(null, .nothing);
5547		switch (cancel_region.errno()) {
5548		.SUCCESS => return,
5549		.INTR, .CANCELED => continue,
5550		.ADDRINUSE => return error.AddressInUse,
5551		.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
5552		.INVAL => \|err\| return errnoBug(err), // invalid parameters
5553		.NOTSOCK => \|err\| return errnoBug(err), // invalid `sockfd`
5554		.AFNOSUPPORT => return error.AddressFamilyUnsupported,
5555		.ADDRNOTAVAIL => return error.AddressUnavailable,
5556		.FAULT => \|err\| return errnoBug(err), // invalid `addr` pointer
5557		.NOMEM => return error.SystemResources,
5558		else => \|err\| return unexpectedErrno(err),
5559		}
5560		}
5561		}
5562
5563		fn chdir(ev: Evented, sync: CancelRegion.Sync, path: [*:0]const u8) ChdirError!void {
5564		_ = ev;
5565		while (true) {
5566		try sync.cancel_region.await(.nothing);
5567		switch (linux.errno(linux.chdir(path))) {
5568		.SUCCESS => return,
5569		.INTR => continue,
5570		.ACCES => return error.AccessDenied,
5571		.IO => return error.FileSystem,
5572		.LOOP => return error.SymLinkLoop,
5573		.NAMETOOLONG => return error.NameTooLong,
5574		.NOENT => return error.FileNotFound,
5575		.NOMEM => return error.SystemResources,
5576		.NOTDIR => return error.NotDir,
5577		.ILSEQ => return error.BadPathName,
5578		.FAULT => \|err\| return errnoBug(err),
5579		else => \|err\| return unexpectedErrno(err),
5580		}
5581		}
5582		}
5583
5584		fn close(ev: Evented, cancel_region: CancelRegion, fd: fd_t) void {
5585		while (true) {
5586		const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
5587		error.Canceled => unreachable, // blocked
5588		};
5589		thread.enqueue().* = .{
5590		.opcode = .CLOSE,
5591		.flags = 0,
5592		.ioprio = 0,
5593		.fd = fd,
5594		.off = 0,
5595		.addr = 0,
5596		.len = 0,
5597		.rw_flags = 0,
5598		.user_data = @intFromPtr(cancel_region.fiber),
5599		.buf_index = 0,
5600		.personality = 0,
5601		.splice_fd_in = 0,
5602		.addr3 = 0,
5603		.resv = 0,
5604		};
5605		ev.yield(null, .nothing);
5606		switch (cancel_region.errno()) {
5607		.SUCCESS => return,
5608		.INTR, .CANCELED => continue,
5609		.BADF => unreachable, // Always a race condition.
5610		else => break,
5611		}
5612		}
5613		}
5614
5615		fn fchdir(ev: Evented, sync: CancelRegion.Sync, dir: fd_t) process.SetCurrentDirError!void {
5616		_ = ev;
5617		if (dir == linux.AT.FDCWD) return;
5618		while (true) {
5619		try sync.cancel_region.await(.nothing);
5620		switch (linux.errno(linux.fchdir(dir))) {
5621		.SUCCESS => return,
5622		.INTR => continue,
5623		.ACCES => return error.AccessDenied,
5624		.NOTDIR => return error.NotDir,
5625		.IO => return error.FileSystem,
5626		.BADF => \|err\| return errnoBug(err),
5627		else => \|err\| return unexpectedErrno(err),
5628		}
5629		}
5630		}
5631
5632		fn fchmodat(
5633		ev: *Evented,
5634		sync: *CancelRegion.Sync,
5635		dir: fd_t,
5636		path: [*:0]const u8,
5637		mode: linux.mode_t,
5638		flags: u32,
5639		) Dir.SetFilePermissionsError!void {
5640		_ = ev;
5641		while (true) {
5642		try sync.cancel_region.await(.nothing);
5643		switch (linux.errno(linux.fchmodat2(dir, path, mode, flags))) {
5644		.SUCCESS => return,
5645		.INTR => continue,
5646		.BADF => \|err\| return errnoBug(err),
5647		.FAULT => \|err\| return errnoBug(err),
5648		.INVAL => \|err\| return errnoBug(err),
5649		.ACCES => return error.AccessDenied,
5650		.IO => return error.InputOutput,
5651		.LOOP => return error.SymLinkLoop,
5652		.NOENT => return error.FileNotFound,
5653		.NOMEM => return error.SystemResources,
5654		.NOTDIR => return error.FileNotFound,
5655		.OPNOTSUPP => return error.OperationUnsupported,
5656		.PERM => return error.PermissionDenied,
5657		.ROFS => return error.ReadOnlyFileSystem,
5658		else => \|err\| return unexpectedErrno(err),
5659		}
5660		}
5661		}
5662
5663		fn fchownat(
5664		ev: *Evented,
5665		sync: *CancelRegion.Sync,
5666		dir: fd_t,
5667		path: [*:0]const u8,
5668		owner: linux.uid_t,
5669		group: linux.gid_t,
5670		flags: u32,
5671		) File.SetOwnerError!void {
5672		_ = ev;
5673		while (true) {
5674		try sync.cancel_region.await(.nothing);
5675		switch (linux.errno(linux.fchownat(dir, path, owner, group, flags))) {
5676		.SUCCESS => return,
5677		.INTR => continue,
5678		.BADF => \|err\| return errnoBug(err), // likely fd refers to directory opened without `Dir.OpenOptions.iterate`
5679		.FAULT => \|err\| return errnoBug(err),
5680		.INVAL => \|err\| return errnoBug(err),
5681		.ACCES => return error.AccessDenied,
5682		.IO => return error.InputOutput,
5683		.LOOP => return error.SymLinkLoop,
5684		.NOENT => return error.FileNotFound,
5685		.NOMEM => return error.SystemResources,
5686		.NOTDIR => return error.FileNotFound,
5687		.PERM => return error.PermissionDenied,
5688		.ROFS => return error.ReadOnlyFileSystem,
5689		else => \|err\| return unexpectedErrno(err),
5690		}
5691		}
5692		}
5693
5694		fn flock(
5695		ev: *Evented,
5696		sync: *CancelRegion.Sync,
5697		fd: fd_t,
5698		op: File.Lock,
5699		blocking: enum { blocking, nonblocking },
5700		) (File.LockError \|\| error{WouldBlock})!void {
5701		while (true) {
5702		try sync.cancel_region.await(.nothing);
5703		switch (linux.errno(linux.flock(fd, LOCK.NB \| @as(i32, switch (op) {
5704		.none => LOCK.UN,
5705		.shared => LOCK.SH,
5706		.exclusive => LOCK.EX,
5707		})))) {
5708		.SUCCESS => return,
5709		.INTR => continue,
5710		.BADF => \|err\| return errnoBug(err),
5711		.INVAL => \|err\| return errnoBug(err), // invalid parameters
5712		.NOLCK => return error.SystemResources,
5713		.AGAIN => {
5714		const thread = try sync.cancel_region.awaitIoUring();
5715		thread.enqueue().* = .{
5716		.opcode = .NOP,
5717		.flags = 0,
5718		.ioprio = 0,
5719		.fd = 0,
5720		.off = 0,
5721		.addr = 0,
5722		.len = 0,
5723		.rw_flags = 0,
5724		.user_data = @intFromPtr(sync.cancel_region.fiber),
5725		.buf_index = 0,
5726		.personality = 0,
5727		.splice_fd_in = 0,
5728		.addr3 = 0,
5729		.resv = 0,
5730		};
5731		ev.yield(null, .nothing);
5732		switch (sync.cancel_region.errno()) {
5733		.SUCCESS, .INTR, .CANCELED => {},
5734		else => unreachable,
5735		}
5736		switch (blocking) {
5737		.blocking => continue,
5738		.nonblocking => return error.WouldBlock,
5739		}
5740		},
5741		.OPNOTSUPP => return error.FileLocksUnsupported,
5742		else => \|err\| return unexpectedErrno(err),
5743		}
5744		}
5745		}
5746
5747		fn getsockname(
5748		ev: *Evented,
5749		sync: *CancelRegion.Sync,
5750		socket_fd: fd_t,
5751		addr: *linux.sockaddr,
5752		addr_len: *linux.socklen_t,
5753		) !void {
5754		_ = ev;
5755		while (true) {
5756		try sync.cancel_region.await(.nothing);
5757		switch (linux.errno(linux.getsockname(socket_fd, addr, addr_len))) {
5758		.SUCCESS => return,
5759		.INTR => continue,
5760		.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
5761		.FAULT => \|err\| return errnoBug(err),
5762		.INVAL => \|err\| return errnoBug(err), // invalid parameters
5763		.NOTSOCK => \|err\| return errnoBug(err), // always a race condition
5764		.NOBUFS => return error.SystemResources,
5765		else => \|err\| return unexpectedErrno(err),
5766		}
5767		}
5768		}
5769
5770		fn linkat(
5771		ev: *Evented,
5772		cancel_region: *CancelRegion,
5773		old_dir: fd_t,
5774		old_path: [*:0]const u8,
5775		new_dir: fd_t,
5776		new_path: [*:0]const u8,
5777		flags: u32,
5778		) File.HardLinkError!void {
5779		while (true) {
5780		const thread = try cancel_region.awaitIoUring();
5781		thread.enqueue().* = .{
5782		.opcode = .LINKAT,
5783		.flags = 0,
5784		.ioprio = 0,
5785		.fd = old_dir,
5786		.off = @intFromPtr(new_path),
5787		.addr = @intFromPtr(old_path),
5788		.len = @bitCast(new_dir),
5789		.rw_flags = flags,
5790		.user_data = @intFromPtr(cancel_region.fiber),
5791		.buf_index = 0,
5792		.personality = 0,
5793		.splice_fd_in = 0,
5794		.addr3 = 0,
5795		.resv = 0,
5796		};
5797		ev.yield(null, .nothing);
5798		switch (cancel_region.errno()) {
5799		.SUCCESS => return,
5800		.INTR, .CANCELED => continue,
5801		.ACCES => return error.AccessDenied,
5802		.DQUOT => return error.DiskQuota,
5803		.EXIST => return error.PathAlreadyExists,
5804		.IO => return error.HardwareFailure,
5805		.LOOP => return error.SymLinkLoop,
5806		.MLINK => return error.LinkQuotaExceeded,
5807		.NAMETOOLONG => return error.NameTooLong,
5808		.NOENT => return error.FileNotFound,
5809		.NOMEM => return error.SystemResources,
5810		.NOSPC => return error.NoSpaceLeft,
5811		.NOTDIR => return error.NotDir,
5812		.PERM => return error.PermissionDenied,
5813		.ROFS => return error.ReadOnlyFileSystem,
5814		.XDEV => return error.CrossDevice,
5815		.ILSEQ => return error.BadPathName,
5816		.FAULT => \|err\| return errnoBug(err),
5817		.INVAL => \|err\| return errnoBug(err),
5818		else => \|err\| return unexpectedErrno(err),
5819		}
5820		}
5821		}
5822
5823		fn lseek(
5824		ev: *Evented,
5825		sync: *CancelRegion.Sync,
5826		fd: fd_t,
5827		offset: u64,
5828		whence: u32,
5829		) File.SeekError!void {
5830		_ = ev;
5831		while (true) {
5832		try sync.cancel_region.await(.nothing);
5833		var result: u64 = undefined;
5834		switch (linux.errno(switch (@sizeOf(usize)) {
5835		else => comptime unreachable,
5836		4 => linux.llseek(fd, offset, &result, whence),
5837		8 => linux.lseek(fd, @bitCast(offset), whence),
5838		})) {
5839		.SUCCESS => return,
5840		.INTR => continue,
5841		.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
5842		.INVAL => return error.Unseekable,
5843		.OVERFLOW => return error.Unseekable,
5844		.SPIPE => return error.Unseekable,
5845		.NXIO => return error.Unseekable,
5846		else => \|err\| return unexpectedErrno(err),
5847		}
5848		}
5849		}
5850
5851		fn openat(
5852		ev: *Evented,
5853		cancel_region: *CancelRegion,
5854		dir: fd_t,
5855		path: [*:0]const u8,
5856		flags: linux.O,
5857		mode: linux.mode_t,
5858		) File.OpenError!fd_t {
5859		var mut_flags = flags;
5860		if (@hasField(linux.O, "LARGEFILE")) mut_flags.LARGEFILE = true;
5861		while (true) {
5862		const thread = try cancel_region.awaitIoUring();
5863		thread.enqueue().* = .{
5864		.opcode = .OPENAT,
5865		.flags = 0,
5866		.ioprio = 0,
5867		.fd = dir,
5868		.off = 0,
5869		.addr = @intFromPtr(path),
5870		.len = mode,
5871		.rw_flags = @bitCast(mut_flags),
5872		.user_data = @intFromPtr(cancel_region.fiber),
5873		.buf_index = 0,
5874		.personality = 0,
5875		.splice_fd_in = 0,
5876		.addr3 = 0,
5877		.resv = 0,
5878		};
5879		ev.yield(null, .nothing);
5880		const completion = cancel_region.completion();
5881		switch (completion.errno()) {
5882		.SUCCESS => return completion.result,
5883		.INTR, .CANCELED => continue,
5884		.FAULT => \|err\| return errnoBug(err),
5885		.INVAL => return error.BadPathName,
5886		.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
5887		.ACCES => return error.AccessDenied,
5888		.FBIG => return error.FileTooBig,
5889		.OVERFLOW => return error.FileTooBig,
5890		.ISDIR => return error.IsDir,
5891		.LOOP => return error.SymLinkLoop,
5892		.MFILE => return error.ProcessFdQuotaExceeded,
5893		.NAMETOOLONG => return error.NameTooLong,
5894		.NFILE => return error.SystemFdQuotaExceeded,
5895		.NODEV => return error.NoDevice,
5896		.NOENT => return error.FileNotFound,
5897		.SRCH => return error.FileNotFound, // Linux when opening procfs files.
5898		.NOMEM => return error.SystemResources,
5899		.NOSPC => return error.NoSpaceLeft,
5900		.NOTDIR => return error.NotDir,
5901		.PERM => return error.PermissionDenied,
5902		.EXIST => return error.PathAlreadyExists,
5903		.BUSY => return error.DeviceBusy,
5904		.OPNOTSUPP => return error.FileLocksUnsupported,
5905		.AGAIN => return error.WouldBlock,
5906		.TXTBSY => return error.FileBusy,
5907		.NXIO => return error.NoDevice,
5908		.ILSEQ => return error.BadPathName,
5909		else => \|err\| return unexpectedErrno(err),
5910		}
5911		}
5912		}
5913
5914		fn preadv(
5915		ev: *Evented,
5916		cancel_region: *CancelRegion,
5917		fd: fd_t,
5918		iov: []const iovec,
5919		offset: ?u64,
5920		) File.Reader.Error!usize {
5921		if (iov.len == 0) return 0;
5922		const gather = iov.len > 1 or iov[0].len > 0xfffff000;
5923		while (true) {
5924		const thread = try cancel_region.awaitIoUring();
5925		thread.enqueue().* = .{
5926		.opcode = if (gather) .READV else .READ,
5927		.flags = 0,
5928		.ioprio = 0,
5929		.fd = fd,
5930		.off = offset orelse std.math.maxInt(u64),
5931		.addr = if (gather) @intFromPtr(iov.ptr) else @intFromPtr(iov[0].base),
5932		.len = @intCast(if (gather) iov.len else iov[0].len),
5933		.rw_flags = 0,
5934		.user_data = @intFromPtr(cancel_region.fiber),
5935		.buf_index = 0,
5936		.personality = 0,
5937		.splice_fd_in = 0,
5938		.addr3 = 0,
5939		.resv = 0,
5940		};
5941		ev.yield(null, .nothing);
5942		const completion = cancel_region.completion();
5943		switch (completion.errno()) {
5944		.SUCCESS => return @as(u32, @bitCast(completion.result)),
5945		.INTR, .CANCELED => continue,
5946		.INVAL => \|err\| return errnoBug(err),
5947		.FAULT => \|err\| return errnoBug(err),
5948		.AGAIN => return error.WouldBlock,
5949		.BADF => \|err\| return errnoBug(err), // File descriptor used after closed
5950		.IO => return error.InputOutput,
5951		.ISDIR => return error.IsDir,
5952		.NOBUFS => return error.SystemResources,
5953		.NOMEM => return error.SystemResources,
5954		.NOTCONN => return error.SocketUnconnected,
5955		.CONNRESET => return error.ConnectionResetByPeer,
5956		else => \|err\| return unexpectedErrno(err),
5957		}
5958		}
5959		}
5960
5961		fn pwritev(
5962		ev: *Evented,
5963		cancel_region: *CancelRegion,
5964		fd: fd_t,
5965		iov: []const iovec_const,
5966		offset: ?u64,
5967		) File.Writer.Error!usize {
5968		if (iov.len == 0) return 0;
5969		const scatter = iov.len > 1 or iov[0].len > 0xfffff000;
5970		while (true) {
5971		const thread = try cancel_region.awaitIoUring();
5972		thread.enqueue().* = .{
5973		.opcode = if (scatter) .WRITEV else .WRITE,
5974		.flags = 0,
5975		.ioprio = 0,
5976		.fd = fd,
5977		.off = offset orelse std.math.maxInt(u64),
5978		.addr = if (scatter) @intFromPtr(iov.ptr) else @intFromPtr(iov[0].base),
5979		.len = @intCast(if (scatter) iov.len else iov[0].len),
5980		.rw_flags = 0,
5981		.user_data = @intFromPtr(cancel_region.fiber),
5982		.buf_index = 0,
5983		.personality = 0,
5984		.splice_fd_in = 0,
5985		.addr3 = 0,
5986		.resv = 0,
5987		};
5988		ev.yield(null, .nothing);
5989		const completion = cancel_region.completion();
5990		switch (completion.errno()) {
5991		.SUCCESS => return @as(u32, @bitCast(completion.result)),
5992		.INTR, .CANCELED => continue,
5993		.INVAL => \|err\| return errnoBug(err),
5994		.FAULT => \|err\| return errnoBug(err),
5995		.AGAIN => return error.WouldBlock,
5996		.BADF => return error.NotOpenForWriting, // Can be a race condition.
5997		.DESTADDRREQ => \|err\| return errnoBug(err), // `connect` was never called.
5998		.DQUOT => return error.DiskQuota,
5999		.FBIG => return error.FileTooBig,
6000		.IO => return error.InputOutput,
6001		.NOSPC => return error.NoSpaceLeft,
6002		.PERM => return error.PermissionDenied,
6003		.PIPE => return error.BrokenPipe,
6004		.CONNRESET => \|err\| return errnoBug(err), // Not a socket handle.
6005		.BUSY => return error.DeviceBusy,
6006		else => \|err\| return unexpectedErrno(err),
6007		}
6008		}
6009		}
6010
6011		fn readAll(
6012		ev: *Evented,
6013		cancel_region: *CancelRegion,
6014		fd: fd_t,
6015		buffer: []u8,
6016		) (File.Reader.Error \|\| error{EndOfStream})!void {
6017		var index: usize = 0;
6018		while (buffer.len - index != 0) {
6019		const len = try ev.preadv(cancel_region, fd, &.{
6020		.{ .base = buffer[index..].ptr, .len = buffer.len - index },
6021		}, null);
6022		if (len == 0) return error.EndOfStream;
6023		index += len;
6024		}
6025		}
6026
6027		fn realPath(
6028		ev: *Evented,
6029		sync: *CancelRegion.Sync,
6030		fd: fd_t,
6031		out_buffer: []u8,
6032		) File.RealPathError!usize {
6033		_ = ev;
6034		var procfs_buf: [std.fmt.count("/proc/self/fd/{d}\x00", .{std.math.minInt(fd_t)})]u8 = undefined;
6035		const proc_path = std.fmt.bufPrintSentinel(&procfs_buf, "/proc/self/fd/{d}", .{fd}, 0) catch
6036		unreachable;
6037		while (true) {
6038		try sync.cancel_region.await(.nothing);
6039		const rc = linux.readlink(proc_path, out_buffer.ptr, out_buffer.len);
6040		switch (linux.errno(rc)) {
6041		.SUCCESS => return rc,
6042		.INTR => continue,
6043		.ACCES => return error.AccessDenied,
6044		.FAULT => \|err\| return errnoBug(err),
6045		.IO => return error.FileSystem,
6046		.LOOP => return error.SymLinkLoop,
6047		.NAMETOOLONG => return error.NameTooLong,
6048		.NOENT => return error.FileNotFound,
6049		.NOMEM => return error.SystemResources,
6050		.NOTDIR => return error.NotDir,
6051		.ILSEQ => \|err\| return errnoBug(err),
6052		else => \|err\| return unexpectedErrno(err),
6053		}
6054		}
6055		}
6056
6057		fn renameat(
6058		ev: *Evented,
6059		cancel_region: *CancelRegion,
6060		old_dir: fd_t,
6061		old_path: [*:0]const u8,
6062		new_dir: fd_t,
6063		new_path: [*:0]const u8,
6064		flags: linux.RENAME,
6065		) Dir.RenameError!void {
6066		while (true) {
6067		const thread = try cancel_region.awaitIoUring();
6068		thread.enqueue().* = .{
6069		.opcode = .RENAMEAT,
6070		.flags = 0,
6071		.ioprio = 0,
6072		.fd = old_dir,
6073		.off = @intFromPtr(new_path),
6074		.addr = @intFromPtr(old_path),
6075		.len = @bitCast(new_dir),
6076		.rw_flags = @bitCast(flags),
6077		.user_data = @intFromPtr(cancel_region.fiber),
6078		.buf_index = 0,
6079		.personality = 0,
6080		.splice_fd_in = 0,
6081		.addr3 = 0,
6082		.resv = 0,
6083		};
6084		ev.yield(null, .nothing);
6085		switch (cancel_region.errno()) {
6086		.SUCCESS => return,
6087		.INTR, .CANCELED => continue,
6088		.ACCES => return error.AccessDenied,
6089		.PERM => return error.PermissionDenied,
6090		.BUSY => return error.FileBusy,
6091		.DQUOT => return error.DiskQuota,
6092		.ISDIR => return error.IsDir,
6093		.IO => return error.HardwareFailure,
6094		.LOOP => return error.SymLinkLoop,
6095		.MLINK => return error.LinkQuotaExceeded,
6096		.NAMETOOLONG => return error.NameTooLong,
6097		.NOENT => return error.FileNotFound,
6098		.NOTDIR => return error.NotDir,
6099		.NOMEM => return error.SystemResources,
6100		.NOSPC => return error.NoSpaceLeft,
6101		.EXIST => return error.DirNotEmpty,
6102		.NOTEMPTY => return error.DirNotEmpty,
6103		.ROFS => return error.ReadOnlyFileSystem,
6104		.XDEV => return error.CrossDevice,
6105		.ILSEQ => return error.BadPathName,
6106		.FAULT => \|err\| return errnoBug(err),
6107		.INVAL => \|err\| return errnoBug(err),
6108		else => \|err\| return unexpectedErrno(err),
6109		}
6110		}
6111		}
6112
6113		fn setsockopt(
6114		ev: *Evented,
6115		cancel_region: *CancelRegion,
6116		fd: fd_t,
6117		level: i32,
6118		opt_name: u32,
6119		option: u32,
6120		) !void {
6121		const o: []const u8 = @ptrCast(&option);
6122		while (true) {
6123		const off: extern struct {
6124		cmd_op: linux.IO_URING_SOCKET_OP,
6125		pad: u32,
6126		} align(@alignOf(u64)) = .{
6127		.cmd_op = .SETSOCKOPT,
6128		.pad = 0,
6129		};
6130		const addr: extern struct { level: i32, opt_name: u32 } align(@alignOf(u64)) = .{
6131		.level = level,
6132		.opt_name = opt_name,
6133		};
6134		const thread = try cancel_region.awaitIoUring();
6135		thread.enqueue().* = .{
6136		.opcode = .URING_CMD,
6137		.flags = 0,
6138		.ioprio = 0,
6139		.fd = fd,
6140		.off = @as(const u64, @ptrCast(&off)).,
6141		.addr = @as(const u64, @ptrCast(&addr)).,
6142		.len = 0,
6143		.rw_flags = 0,
6144		.user_data = @intFromPtr(cancel_region.fiber),
6145		.buf_index = 0,
6146		.personality = 0,
6147		.splice_fd_in = @intCast(o.len),
6148		.addr3 = @intFromPtr(o.ptr),
6149		.resv = 0,
6150		};
6151		ev.yield(null, .nothing);
6152		switch (cancel_region.errno()) {
6153		.SUCCESS => return,
6154		.INTR, .CANCELED => continue,
6155		.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
6156		.NOTSOCK => \|err\| return errnoBug(err),
6157		.INVAL => \|err\| return errnoBug(err),
6158		.FAULT => \|err\| return errnoBug(err),
6159		else => \|err\| return unexpectedErrno(err),
6160		}
6161		}
6162		}
6163
6164		fn socket(
6165		ev: *Evented,
6166		cancel_region: *CancelRegion,
6167		family: linux.sa_family_t,
6168		options: net.IpAddress.BindOptions,
6169		) error{
6170		AddressFamilyUnsupported,
6171		ProtocolUnsupportedBySystem,
6172		ProcessFdQuotaExceeded,
6173		SystemFdQuotaExceeded,
6174		SystemResources,
6175		ProtocolUnsupportedByAddressFamily,
6176		SocketModeUnsupported,
6177		OptionUnsupported,
6178		Unexpected,
6179		Canceled,
6180		}!fd_t {
6181		const mode = posixSocketMode(options.mode);
6182		const protocol = posixProtocol(options.protocol);
6183		const socket_fd = while (true) {
6184		const thread = try cancel_region.awaitIoUring();
6185		thread.enqueue().* = .{
6186		.opcode = .SOCKET,
6187		.flags = 0,
6188		.ioprio = 0,
6189		.fd = family,
6190		.off = mode \| linux.SOCK.CLOEXEC,
6191		.addr = 0,
6192		.len = protocol,
6193		.rw_flags = 0,
6194		.user_data = @intFromPtr(cancel_region.fiber),
6195		.buf_index = 0,
6196		.personality = 0,
6197		.splice_fd_in = 0,
6198		.addr3 = 0,
6199		.resv = 0,
6200		};
6201		ev.yield(null, .nothing);
6202		const completion = cancel_region.completion();
6203		switch (completion.errno()) {
6204		.SUCCESS => break completion.result,
6205		.INTR, .CANCELED => continue,
6206		.AFNOSUPPORT => return error.AddressFamilyUnsupported,
6207		.INVAL => return error.ProtocolUnsupportedBySystem,
6208		.MFILE => return error.ProcessFdQuotaExceeded,
6209		.NFILE => return error.SystemFdQuotaExceeded,
6210		.NOBUFS => return error.SystemResources,
6211		.NOMEM => return error.SystemResources,
6212		.PROTONOSUPPORT => return error.ProtocolUnsupportedByAddressFamily,
6213		.PROTOTYPE => return error.SocketModeUnsupported,
6214		else => \|err\| return unexpectedErrno(err),
6215		}
6216		};
6217		errdefer ev.close(cancel_region, socket_fd);
6218
6219		if (options.ip6_only) {
6220		if (linux.IPV6 == void) return error.OptionUnsupported;
6221		try ev.setsockopt(cancel_region, socket_fd, linux.IPPROTO.IPV6, linux.IPV6.V6ONLY, 0);
6222		}
6223
6224		return socket_fd;
6225		}
6226
6227		fn stat(ev: Evented, cancel_region: CancelRegion, fd: fd_t) Dir.StatError!Dir.Stat {
6228		return ev.statx(cancel_region, fd, "", linux.AT.EMPTY_PATH) catch \|err\| switch (err) {
6229		error.BadPathName, error.NameTooLong => unreachable, // path is empty
6230		error.AccessDenied => return errnoBug(.ACCES),
6231		error.SymLinkLoop => return errnoBug(.LOOP),
6232		error.FileNotFound => return errnoBug(.NOENT),
6233		error.NotDir => return errnoBug(.NOTDIR),
6234		else => \|e\| return e,
6235		};
6236		}
6237
6238		fn statx(
6239		ev: *Evented,
6240		cancel_region: *CancelRegion,
6241		dir: fd_t,
6242		path: [*:0]const u8,
6243		flags: u32,
6244		) (Dir.StatError \|\| Dir.PathNameError \|\| error{ FileNotFound, NotDir, SymLinkLoop })!Dir.Stat {
6245		while (true) {
6246		var statx_buf = std.mem.zeroes(linux.Statx);
6247		const thread = try cancel_region.awaitIoUring();
6248		thread.enqueue().* = .{
6249		.opcode = .STATX,
6250		.flags = 0,
6251		.ioprio = 0,
6252		.fd = dir,
6253		.off = @intFromPtr(&statx_buf),
6254		.addr = @intFromPtr(path),
6255		.len = @bitCast(linux_statx_request),
6256		.rw_flags = flags,
6257		.user_data = @intFromPtr(cancel_region.fiber),
6258		.buf_index = 0,
6259		.personality = 0,
6260		.splice_fd_in = 0,
6261		.addr3 = 0,
6262		.resv = 0,
6263		};
6264		ev.yield(null, .nothing);
6265		switch (cancel_region.errno()) {
6266		.SUCCESS => return statFromLinux(&statx_buf),
6267		.INTR, .CANCELED => continue,
6268		.ACCES => return error.AccessDenied,
6269		.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
6270		.FAULT => \|err\| return errnoBug(err),
6271		.INVAL => \|err\| return errnoBug(err),
6272		.LOOP => return error.SymLinkLoop,
6273		.NAMETOOLONG => \|err\| return errnoBug(err),
6274		.NOENT => return error.FileNotFound,
6275		.NOTDIR => return error.NotDir,
6276		.NOMEM => return error.SystemResources,
6277		else => \|err\| return unexpectedErrno(err),
6278		}
6279		}
6280		}
6281
6282		fn urandomReadAll(
6283		ev: *Evented,
6284		cancel_region: *CancelRegion,
6285		buffer: []u8,
6286		) (File.OpenError \|\| File.Reader.Error \|\| error{EndOfStream})!void {
6287		return ev.readAll(cancel_region, try ev.random_fd.open(ev, cancel_region, "/dev/urandom", .{
6288		.ACCMODE = .RDONLY,
6289		.CLOEXEC = true,
6290		}), buffer);
6291		}
6292
6293		fn utimensat(
6294		ev: *Evented,
6295		sync: *CancelRegion.Sync,
6296		dir: fd_t,
6297		path: [*:0]const u8,
6298		times: ?*const [2]linux.timespec,
6299		flags: u32,
6300		) File.SetTimestampsError!void {
6301		_ = ev;
6302		while (true) {
6303		try sync.cancel_region.await(.nothing);
6304		switch (linux.errno(linux.utimensat(dir, path, times, flags))) {
6305		.SUCCESS => return,
6306		.INTR => continue,
6307		.BADF => \|err\| return errnoBug(err), // always a race condition
6308		.FAULT => \|err\| return errnoBug(err),
6309		.INVAL => \|err\| return errnoBug(err),
6310		.ACCES => return error.AccessDenied,
6311		.PERM => return error.PermissionDenied,
6312		.ROFS => return error.ReadOnlyFileSystem,
6313		else => \|err\| return unexpectedErrno(err),
6314		}
6315		}
6316		}
6317
6318		fn writeAll(
6319		ev: *Evented,
6320		cancel_region: *CancelRegion,
6321		fd: fd_t,
6322		buffer: []const u8,
6323		) (File.Writer.Error \|\| error{EndOfStream})!void {
6324		var index: usize = 0;
6325		while (buffer.len - index != 0) {
6326		const len = try ev.pwritev(cancel_region, fd, &.{
6327		.{ .base = buffer[index..].ptr, .len = buffer.len - index },
6328		}, null);
6329		if (len == 0) return error.EndOfStream;
6330		index += len;
6331		}
6332		}
6333
6334		test {
6335		_ = Fiber.CancelProtection;
6336		}

lib/std/Io/Kqueue.zig+14-204

...	...	@@ -31,8 +31,8 @@ const changes_buffer_len = 64;
31	31
32	32	const Thread = struct {
33	33	thread: std.Thread,
34		idle_context: Context,
35		current_context: *Context,
	34	idle_context: Io.fiber.Context,
	35	current_context: *Io.fiber.Context,
36	36	ready_queue: ?*Fiber,
37	37	kq_fd: posix.fd_t,
38	38	idle_search_index: u32,
...	...	@@ -74,7 +74,7 @@ const Thread = struct {
74	74
75	75	const Fiber = struct {
76	76	required_align: void align(4),
77		context: Context,
	77	context: Io.fiber.Context,
78	78	awaiter: ?*Fiber,
79	79	queue_next: ?*Fiber,
80	80	cancel_thread: ?*Thread,
...	...	@@ -291,12 +291,12 @@ fn yield(k: Kqueue, maybe_ready_fiber: ?Fiber, pending_task: SwitchMessage.Pen
291	291	&thread.idle_context;
292	292	const message: SwitchMessage = .{
293	293	.contexts = .{
294		.prev = thread.current_context,
295		.ready = ready_context,
	294	.old = thread.current_context,
	295	.new = ready_context,
296	296	},
297	297	.pending_task = pending_task,
298	298	};
299		std.log.debug("switching from {} to {}", .{ message.contexts.prev, message.contexts.ready });
	299	std.log.debug("switching from {} to {}", .{ message.contexts.old, message.contexts.new });
300	300	contextSwitch(&message).handle(k);
301	301	}
302	302
...	...	@@ -393,7 +393,7 @@ fn schedule(k: Kqueue, thread: Thread, ready_queue: Fiber.Queue) void {
393	393	)) \|old_head\| ready_queue.tail.queue_next = old_head;
394	394	}
395	395
396		fn mainIdle(k: Kqueue, message: const SwitchMessage) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Context)))) noreturn {
	396	fn mainIdle(k: Kqueue, message: const SwitchMessage) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Io.fiber.Context)))) noreturn {
397	397	message.handle(k);
398	398	k.idle(&k.threads.allocated[0]);
399	399	k.yield(@ptrCast(&k.main_fiber_buffer), .nothing);
...	...	@@ -483,10 +483,7 @@ fn idle(k: Kqueue, thread: Thread) void {
483	483	}
484	484
485	485	const SwitchMessage = struct {
486		contexts: extern struct {
487		prev: *Context,
488		ready: *Context,
489		},
	486	contexts: Io.fiber.Switch,
490	487	pending_task: PendingTask,
491	488
492	489	const PendingTask = union(enum) {
...	...	@@ -500,11 +497,11 @@ const SwitchMessage = struct {
500	497
501	498	fn handle(message: const SwitchMessage, k: Kqueue) void {
502	499	const thread: *Thread = .current();
503		thread.current_context = message.contexts.ready;
	500	thread.current_context = message.contexts.new;
504	501	switch (message.pending_task) {
505	502	.nothing => {},
506		.reschedule => if (message.contexts.prev != &thread.idle_context) {
507		const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
	503	.reschedule => if (message.contexts.old != &thread.idle_context) {
	504	const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
508	505	assert(prev_fiber.queue_next == null);
509	506	k.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });
510	507	},
...	...	@@ -512,13 +509,13 @@ const SwitchMessage = struct {
512	509	k.recycle(fiber);
513	510	},
514	511	.register_awaiter => \|awaiter\| {
515		const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
	512	const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
516	513	assert(prev_fiber.queue_next == null);
517	514	if (@atomicRmw(?*Fiber, awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished)
518	515	k.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });
519	516	},
520	517	.register_select => \|futures\| {
521		const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
	518	const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
522	519	assert(prev_fiber.queue_next == null);
523	520	for (futures) \|any_future\| {
524	521	const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));
...	...	@@ -550,195 +547,8 @@ const SwitchMessage = struct {
550	547	}
551	548	};
552	549
553		const Context = switch (builtin.cpu.arch) {
554		.aarch64 => extern struct {
555		sp: u64,
556		fp: u64,
557		pc: u64,
558		},
559		.x86_64 => extern struct {
560		rsp: u64,
561		rbp: u64,
562		rip: u64,
563		},
564		else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
565		};
566
567	550	inline fn contextSwitch(message: const SwitchMessage) const SwitchMessage {
568		return @fieldParentPtr("contexts", switch (builtin.cpu.arch) {
569		.aarch64 => asm volatile (
570		\\ ldp x0, x2, [x1]
571		\\ ldr x3, [x2, #16]
572		\\ mov x4, sp
573		\\ stp x4, fp, [x0]
574		\\ adr x5, 0f
575		\\ ldp x4, fp, [x2]
576		\\ str x5, [x0, #16]
577		\\ mov sp, x4
578		\\ br x3
579		\\0:
580		: [received_message] "={x1}" (-> *const @FieldType(SwitchMessage, "contexts")),
581		: [message_to_send] "{x1}" (&message.contexts),
582		: .{
583		.x0 = true,
584		.x1 = true,
585		.x2 = true,
586		.x3 = true,
587		.x4 = true,
588		.x5 = true,
589		.x6 = true,
590		.x7 = true,
591		.x8 = true,
592		.x9 = true,
593		.x10 = true,
594		.x11 = true,
595		.x12 = true,
596		.x13 = true,
597		.x14 = true,
598		.x15 = true,
599		.x16 = true,
600		.x17 = true,
601		.x19 = true,
602		.x20 = true,
603		.x21 = true,
604		.x22 = true,
605		.x23 = true,
606		.x24 = true,
607		.x25 = true,
608		.x26 = true,
609		.x27 = true,
610		.x28 = true,
611		.x30 = true,
612		.z0 = true,
613		.z1 = true,
614		.z2 = true,
615		.z3 = true,
616		.z4 = true,
617		.z5 = true,
618		.z6 = true,
619		.z7 = true,
620		.z8 = true,
621		.z9 = true,
622		.z10 = true,
623		.z11 = true,
624		.z12 = true,
625		.z13 = true,
626		.z14 = true,
627		.z15 = true,
628		.z16 = true,
629		.z17 = true,
630		.z18 = true,
631		.z19 = true,
632		.z20 = true,
633		.z21 = true,
634		.z22 = true,
635		.z23 = true,
636		.z24 = true,
637		.z25 = true,
638		.z26 = true,
639		.z27 = true,
640		.z28 = true,
641		.z29 = true,
642		.z30 = true,
643		.z31 = true,
644		.p0 = true,
645		.p1 = true,
646		.p2 = true,
647		.p3 = true,
648		.p4 = true,
649		.p5 = true,
650		.p6 = true,
651		.p7 = true,
652		.p8 = true,
653		.p9 = true,
654		.p10 = true,
655		.p11 = true,
656		.p12 = true,
657		.p13 = true,
658		.p14 = true,
659		.p15 = true,
660		.fpcr = true,
661		.fpsr = true,
662		.ffr = true,
663		.memory = true,
664		}),
665		.x86_64 => asm volatile (
666		\\ movq 0(%%rsi), %%rax
667		\\ movq 8(%%rsi), %%rcx
668		\\ leaq 0f(%%rip), %%rdx
669		\\ movq %%rsp, 0(%%rax)
670		\\ movq %%rbp, 8(%%rax)
671		\\ movq %%rdx, 16(%%rax)
672		\\ movq 0(%%rcx), %%rsp
673		\\ movq 8(%%rcx), %%rbp
674		\\ jmpq *16(%%rcx)
675		\\0:
676		: [received_message] "={rsi}" (-> *const @FieldType(SwitchMessage, "contexts")),
677		: [message_to_send] "{rsi}" (&message.contexts),
678		: .{
679		.rax = true,
680		.rcx = true,
681		.rdx = true,
682		.rbx = true,
683		.rsi = true,
684		.rdi = true,
685		.r8 = true,
686		.r9 = true,
687		.r10 = true,
688		.r11 = true,
689		.r12 = true,
690		.r13 = true,
691		.r14 = true,
692		.r15 = true,
693		.mm0 = true,
694		.mm1 = true,
695		.mm2 = true,
696		.mm3 = true,
697		.mm4 = true,
698		.mm5 = true,
699		.mm6 = true,
700		.mm7 = true,
701		.zmm0 = true,
702		.zmm1 = true,
703		.zmm2 = true,
704		.zmm3 = true,
705		.zmm4 = true,
706		.zmm5 = true,
707		.zmm6 = true,
708		.zmm7 = true,
709		.zmm8 = true,
710		.zmm9 = true,
711		.zmm10 = true,
712		.zmm11 = true,
713		.zmm12 = true,
714		.zmm13 = true,
715		.zmm14 = true,
716		.zmm15 = true,
717		.zmm16 = true,
718		.zmm17 = true,
719		.zmm18 = true,
720		.zmm19 = true,
721		.zmm20 = true,
722		.zmm21 = true,
723		.zmm22 = true,
724		.zmm23 = true,
725		.zmm24 = true,
726		.zmm25 = true,
727		.zmm26 = true,
728		.zmm27 = true,
729		.zmm28 = true,
730		.zmm29 = true,
731		.zmm30 = true,
732		.zmm31 = true,
733		.fpsr = true,
734		.fpcr = true,
735		.mxcsr = true,
736		.rflags = true,
737		.dirflag = true,
738		.memory = true,
739		}),
740		else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
741		});
	551	return @fieldParentPtr("contexts", Io.fiber.contextSwitch(&message.contexts));
742	552	}
743	553
744	554	fn mainIdleEntry() callconv(.naked) void {

lib/std/Io/Reader.zig+1-1

...	...	@@ -375,7 +375,7 @@ pub fn appendRemainingAligned(
375	375	defer list.* = a.toArrayListAligned(alignment);
376	376
377	377	var remaining = limit;
378		while (remaining.nonzero()) {
	378	while (remaining != .nothing) {
379	379	const n = stream(r, &a.writer, remaining) catch \|err\| switch (err) {
380	380	error.EndOfStream => return,
381	381	error.WriteFailed => return error.OutOfMemory,

lib/std/Io/Threaded.zig+154-93

...	...	@@ -72,6 +72,9 @@ stderr_mutex_locker: std.Thread.Id = Thread.invalid_id,
72	72	stderr_mutex_lock_count: usize = 0,
73	73
74	74	argv0: Argv0,
	75	/// Protected by `mutex`. Determines whether `environ` has been
	76	/// memoized based on `process_environ`.
	77	environ_initialized: bool,
75	78	environ: Environ,
76	79
77	80	null_file: NullFile = .{},
...	...	@@ -125,9 +128,6 @@ pub const Argv0 = switch (native_os) {
125	128	pub const Environ = struct {
126	129	/// Unmodified data directly from the OS.
127	130	process_environ: process.Environ,
128		/// Protected by `mutex`. Determines whether the other fields have been
129		/// memoized based on `process_environ`.
130		initialized: bool = false,
131	131	/// Protected by `mutex`. Memoized based on `process_environ`. Tracks whether the
132	132	/// environment variables are present, ignoring their value.
133	133	exist: Exist = .{},
...	...	@@ -161,9 +161,6 @@ pub const Environ = struct {
161	161	};
162	162
163	163	pub fn scan(environ: *Environ, allocator: std.mem.Allocator) void {
164		if (environ.initialized) return;
165		environ.initialized = true;
166
167	164	if (is_windows) {
168	165	// This value expires with any call that modifies the environment,
169	166	// which is outside of this Io implementation's control, so references
...	...	@@ -1589,6 +1586,7 @@ pub fn init(
1589	1586	.old_sig_pipe = undefined,
1590	1587	.have_signal_handler = init_single_threaded.have_signal_handler,
1591	1588	.argv0 = options.argv0,
	1589	.environ_initialized = options.environ.block.isEmpty(),
1592	1590	.environ = .{ .process_environ = options.environ },
1593	1591	.worker_threads = init_single_threaded.worker_threads,
1594	1592	.disable_memory_mapping = options.disable_memory_mapping,
...	...	@@ -1606,6 +1604,7 @@ pub fn init(
1606	1604	.old_sig_pipe = undefined,
1607	1605	.have_signal_handler = false,
1608	1606	.argv0 = options.argv0,
	1607	.environ_initialized = options.environ.block.isEmpty(),
1609	1608	.environ = .{ .process_environ = options.environ },
1610	1609	.worker_threads = .init(null),
1611	1610	.disable_memory_mapping = options.disable_memory_mapping,
...	...	@@ -1643,9 +1642,8 @@ pub const init_single_threaded: Threaded = .{
1643	1642	.old_sig_pipe = undefined,
1644	1643	.have_signal_handler = false,
1645	1644	.argv0 = .empty,
1646		.environ = .{ .process_environ = .{
1647		.block = if (process.Environ.Block == process.Environ.GlobalBlock) .global else .empty,
1648		} },
	1645	.environ_initialized = true,
	1646	.environ = .empty,
1649	1647	.worker_threads = .init(null),
1650	1648	.disable_memory_mapping = false,
1651	1649	};
...	...	@@ -1768,6 +1766,8 @@ pub fn io(t: *Threaded) Io {
1768	1766	return .{
1769	1767	.userdata = t,
1770	1768	.vtable = &.{
	1769	.crashHandler = crashHandler,
	1770
1771	1771	.async = async,
1772	1772	.concurrent = concurrent,
1773	1773	.await = await,
...	...	@@ -1932,6 +1932,8 @@ pub fn ioBasic(t: *Threaded) Io {
1932	1932	return .{
1933	1933	.userdata = t,
1934	1934	.vtable = &.{
	1935	.crashHandler = crashHandler,
	1936
1935	1937	.async = async,
1936	1938	.concurrent = concurrent,
1937	1939	.await = await,
...	...	@@ -2157,6 +2159,14 @@ const use_libc_getrandom = std.c.versionCheck(if (builtin.abi.isAndroid()) .{
2157	2159
2158	2160	const use_dev_urandom = @TypeOf(posix.system.getrandom) == void and native_os == .linux;
2159	2161
	2162	fn crashHandler(userdata: ?*anyopaque) void {
	2163	const t: *Threaded = @ptrCast(@alignCast(userdata));
	2164	_ = t;
	2165	const thread = Thread.current orelse return;
	2166	thread.status.store(.{ .cancelation = .canceled, .awaitable = .null }, .monotonic);
	2167	thread.cancel_protection = .blocked;
	2168	}
	2169
2160	2170	fn async(
2161	2171	userdata: ?*anyopaque,
2162	2172	result: []u8,
...	...	@@ -2838,19 +2848,19 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2838	2848	var poll_buffer: [poll_buffer_len]posix.pollfd = undefined;
2839	2849	var poll_storage: struct {
2840	2850	gpa: std.mem.Allocator,
2841		b: *Io.Batch,
	2851	batch: *Io.Batch,
2842	2852	slice: []posix.pollfd,
2843	2853	len: u32,
2844	2854
2845	2855	fn add(storage: *@This(), file: Io.File, events: @FieldType(posix.pollfd, "events")) Io.ConcurrentError!void {
2846	2856	const len = storage.len;
2847	2857	if (len == poll_buffer_len) {
2848		const slice: []posix.pollfd = if (storage.b.context) \|context\|
2849		@as([*]posix.pollfd, @ptrCast(@alignCast(context)))[0..storage.b.storage.len]
	2858	const slice: []posix.pollfd = if (storage.batch.userdata) \|batch_userdata\|
	2859	@as([*]posix.pollfd, @ptrCast(@alignCast(batch_userdata)))[0..storage.batch.storage.len]
2850	2860	else allocation: {
2851		const allocation = storage.gpa.alloc(posix.pollfd, storage.b.storage.len) catch
	2861	const allocation = storage.gpa.alloc(posix.pollfd, storage.batch.storage.len) catch
2852	2862	return error.ConcurrencyUnavailable;
2853		storage.b.context = allocation.ptr;
	2863	storage.batch.userdata = allocation.ptr;
2854	2864	break :allocation allocation;
2855	2865	};
2856	2866	@memcpy(slice[0..poll_buffer_len], storage.slice);
...	...	@@ -2863,7 +2873,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2863	2873	};
2864	2874	storage.len = len + 1;
2865	2875	}
2866		} = .{ .gpa = t.allocator, .b = b, .slice = &poll_buffer, .len = 0 };
	2876	} = .{ .gpa = t.allocator, .batch = b, .slice = &poll_buffer, .len = 0 };
2867	2877	{
2868	2878	var index = b.submitted.head;
2869	2879	while (index != .none) {
...	...	@@ -2962,21 +2972,21 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2962	2972	}
2963	2973	}
2964	2974
2965		const WindowsBatchPendingOperationContext = extern struct {
	2975	const WindowsBatchOperationUserdata = extern struct {
2966	2976	file: windows.HANDLE,
2967	2977	iosb: windows.IO_STATUS_BLOCK,
2968	2978
2969		const Erased = Io.Operation.Storage.Pending.Context;
	2979	const Erased = Io.Operation.Storage.Pending.Userdata;
2970	2980
2971	2981	comptime {
2972		assert(@sizeOf(Erased) <= @sizeOf(WindowsBatchPendingOperationContext));
	2982	assert(@sizeOf(WindowsBatchOperationUserdata) <= @sizeOf(Erased));
2973	2983	}
2974	2984
2975		fn toErased(context: WindowsBatchPendingOperationContext) Erased {
2976		return @ptrCast(context);
	2985	fn toErased(userdata: WindowsBatchOperationUserdata) Erased {
	2986	return @ptrCast(userdata);
2977	2987	}
2978	2988
2979		fn fromErased(erased: Erased) WindowsBatchPendingOperationContext {
	2989	fn fromErased(erased: Erased) WindowsBatchOperationUserdata {
2980	2990	return @ptrCast(erased);
2981	2991	}
2982	2992	};
...	...	@@ -2989,15 +2999,16 @@ fn batchCancel(userdata: ?anyopaque, b: Io.Batch) void {
2989	2999	var index = b.pending.head;
2990	3000	while (index != .none) {
2991	3001	const pending = &b.storage[index.toIndex()].pending;
2992		const context: *WindowsBatchPendingOperationContext = .fromErased(&pending.context);
	3002	const operation_userdata: *WindowsBatchOperationUserdata = .fromErased(&pending.userdata);
2993	3003	var cancel_iosb: windows.IO_STATUS_BLOCK = undefined;
2994		_ = windows.ntdll.NtCancelIoFileEx(context.file, &context.iosb, &cancel_iosb);
	3004	_ = windows.ntdll.NtCancelIoFileEx(operation_userdata.file, &operation_userdata.iosb, &cancel_iosb);
2995	3005	index = pending.node.next;
2996	3006	}
2997	3007	while (b.pending.head != .none) waitForApcOrAlert();
2998		} else if (b.context) \|context\| {
2999		t.allocator.free(@as([*]posix.pollfd, @ptrCast(@alignCast(context)))[0..b.storage.len]);
3000		b.context = null;
	3008	} else if (b.userdata) \|batch_userdata\| {
	3009	const poll_storage: [*]posix.pollfd = @ptrCast(@alignCast(batch_userdata));
	3010	t.allocator.free(poll_storage[0..b.storage.len]);
	3011	b.userdata = null;
3001	3012	}
3002	3013	}
3003	3014
...	...	@@ -3007,9 +3018,9 @@ fn batchApc(
3007	3018	_: windows.ULONG,
3008	3019	) callconv(.winapi) void {
3009	3020	const b: *Io.Batch = @ptrCast(@alignCast(apc_context));
3010		const context: *WindowsBatchPendingOperationContext = @fieldParentPtr("iosb", iosb);
3011		const erased_context = context.toErased();
3012		const pending: *Io.Operation.Storage.Pending = @fieldParentPtr("context", erased_context);
	3021	const operation_userdata: *WindowsBatchOperationUserdata = @fieldParentPtr("iosb", iosb);
	3022	const erased_userdata = operation_userdata.toErased();
	3023	const pending: *Io.Operation.Storage.Pending = @fieldParentPtr("userdata", erased_userdata);
3013	3024	switch (pending.node.prev) {
3014	3025	.none => b.pending.head = pending.node.next,
3015	3026	else => \|prev_index\| b.storage[prev_index.toIndex()].pending.node.next = pending.node.next,
...	...	@@ -3019,24 +3030,23 @@ fn batchApc(
3019	3030	else => \|next_index\| b.storage[next_index.toIndex()].pending.node.prev = pending.node.prev,
3020	3031	}
3021	3032	const storage: *Io.Operation.Storage = @fieldParentPtr("pending", pending);
3022		const index = storage - b.storage.ptr;
	3033	const index: Io.Operation.OptionalIndex = .fromIndex(storage - b.storage.ptr);
3023	3034	switch (iosb.u.Status) {
3024	3035	.CANCELLED => {
3025	3036	const tail_index = b.unused.tail;
3026	3037	switch (tail_index) {
3027		.none => b.unused.head = .fromIndex(index),
3028		else => b.storage[tail_index.toIndex()].unused.next = .fromIndex(index),
	3038	.none => b.unused.head = index,
	3039	else => b.storage[tail_index.toIndex()].unused.next = index,
3029	3040	}
3030	3041	storage.* = .{ .unused = .{ .prev = tail_index, .next = .none } };
3031		b.unused.tail = .fromIndex(index);
	3042	b.unused.tail = index;
3032	3043	},
3033	3044	else => {
3034	3045	switch (b.completed.tail) {
3035		.none => b.completed.head = .fromIndex(index),
3036		else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next =
3037		.fromIndex(index),
	3046	.none => b.completed.head = index,
	3047	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next = index,
3038	3048	}
3039		b.completed.tail = .fromIndex(index);
	3049	b.completed.tail = index;
3040	3050	const result: Io.Operation.Result = switch (pending.tag) {
3041	3051	.file_read_streaming => .{ .file_read_streaming = ntReadFileResult(iosb) },
3042	3052	.file_write_streaming => .{ .file_write_streaming = ntWriteFileResult(iosb) },
...	...	@@ -3057,38 +3067,38 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3057	3067	storage.* = .{ .pending = .{
3058	3068	.node = .{ .prev = b.pending.tail, .next = .none },
3059	3069	.tag = submission.operation,
3060		.context = undefined,
	3070	.userdata = undefined,
3061	3071	} };
3062	3072	switch (b.pending.tail) {
3063	3073	.none => b.pending.head = index,
3064	3074	else => \|tail_index\| b.storage[tail_index.toIndex()].pending.node.next = index,
3065	3075	}
3066	3076	b.pending.tail = index;
3067		const context: *WindowsBatchPendingOperationContext = .fromErased(&storage.pending.context);
	3077	const operation_userdata: *WindowsBatchOperationUserdata = .fromErased(&storage.pending.userdata);
3068	3078	errdefer {
3069		context.iosb = .{ .u = .{ .Status = .CANCELLED }, .Information = undefined };
3070		batchApc(b, &context.iosb, 0);
	3079	operation_userdata.iosb = .{ .u = .{ .Status = .CANCELLED }, .Information = undefined };
	3080	batchApc(b, &operation_userdata.iosb, 0);
3071	3081	}
3072	3082	switch (submission.operation) {
3073	3083	.file_read_streaming => \|o\| o: {
3074	3084	var data_index: usize = 0;
3075	3085	while (o.data.len - data_index != 0 and o.data[data_index].len == 0) data_index += 1;
3076	3086	if (o.data.len - data_index == 0) {
3077		context.iosb = .{ .u = .{ .Status = .SUCCESS }, .Information = 0 };
3078		batchApc(b, &context.iosb, 0);
	3087	operation_userdata.iosb = .{ .u = .{ .Status = .SUCCESS }, .Information = 0 };
	3088	batchApc(b, &operation_userdata.iosb, 0);
3079	3089	break :o;
3080	3090	}
3081	3091	const buffer = o.data[data_index];
3082	3092	const short_buffer_len = std.math.lossyCast(u32, buffer.len);
3083	3093
3084	3094	if (o.file.flags.nonblocking) {
3085		context.file = o.file.handle;
	3095	operation_userdata.file = o.file.handle;
3086	3096	switch (windows.ntdll.NtReadFile(
3087	3097	o.file.handle,
3088	3098	null, // event
3089	3099	&batchApc,
3090	3100	b,
3091		&context.iosb,
	3101	&operation_userdata.iosb,
3092	3102	buffer.ptr,
3093	3103	short_buffer_len,
3094	3104	null, // byte offset
...	...	@@ -3097,8 +3107,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3097	3107	.PENDING, .SUCCESS => {},
3098	3108	.CANCELLED => unreachable,
3099	3109	else => \|status\| {
3100		context.iosb.u.Status = status;
3101		batchApc(b, &context.iosb, 0);
	3110	operation_userdata.iosb.u.Status = status;
	3111	batchApc(b, &operation_userdata.iosb, 0);
3102	3112	},
3103	3113	}
3104	3114	} else {
...	...	@@ -3110,7 +3120,7 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3110	3120	null, // event
3111	3121	null, // APC routine
3112	3122	null, // APC context
3113		&context.iosb,
	3123	&operation_userdata.iosb,
3114	3124	buffer.ptr,
3115	3125	short_buffer_len,
3116	3126	null, // byte offset
...	...	@@ -3123,9 +3133,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3123	3133	},
3124	3134	else => \|status\| {
3125	3135	syscall.finish();
3126
3127		context.iosb.u.Status = status;
3128		batchApc(b, &context.iosb, 0);
	3136	operation_userdata.iosb.u.Status = status;
	3137	batchApc(b, &operation_userdata.iosb, 0);
3129	3138	break;
3130	3139	},
3131	3140	};
...	...	@@ -3134,18 +3143,18 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3134	3143	.file_write_streaming => \|o\| o: {
3135	3144	const buffer = windowsWriteBuffer(o.header, o.data, o.splat);
3136	3145	if (buffer.len == 0) {
3137		context.iosb = .{ .u = .{ .Status = .SUCCESS }, .Information = 0 };
3138		batchApc(b, &context.iosb, 0);
	3146	operation_userdata.iosb = .{ .u = .{ .Status = .SUCCESS }, .Information = 0 };
	3147	batchApc(b, &operation_userdata.iosb, 0);
3139	3148	break :o;
3140	3149	}
3141	3150	if (o.file.flags.nonblocking) {
3142		context.file = o.file.handle;
	3151	operation_userdata.file = o.file.handle;
3143	3152	switch (windows.ntdll.NtWriteFile(
3144	3153	o.file.handle,
3145	3154	null, // event
3146	3155	&batchApc,
3147	3156	b,
3148		&context.iosb,
	3157	&operation_userdata.iosb,
3149	3158	buffer.ptr,
3150	3159	@intCast(buffer.len),
3151	3160	null, // byte offset
...	...	@@ -3154,8 +3163,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3154	3163	.PENDING, .SUCCESS => {},
3155	3164	.CANCELLED => unreachable,
3156	3165	else => \|status\| {
3157		context.iosb.u.Status = status;
3158		batchApc(b, &context.iosb, 0);
	3166	operation_userdata.iosb.u.Status = status;
	3167	batchApc(b, &operation_userdata.iosb, 0);
3159	3168	},
3160	3169	}
3161	3170	} else {
...	...	@@ -3167,7 +3176,7 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3167	3176	null, // event
3168	3177	null, // APC routine
3169	3178	null, // APC context
3170		&context.iosb,
	3179	&operation_userdata.iosb,
3171	3180	buffer.ptr,
3172	3181	@intCast(buffer.len),
3173	3182	null, // byte offset
...	...	@@ -3180,9 +3189,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3180	3189	},
3181	3190	else => \|status\| {
3182	3191	syscall.finish();
3183
3184		context.iosb.u.Status = status;
3185		batchApc(b, &context.iosb, 0);
	3192	operation_userdata.iosb.u.Status = status;
	3193	batchApc(b, &operation_userdata.iosb, 0);
3186	3194	break;
3187	3195	},
3188	3196	};
...	...	@@ -3194,13 +3202,13 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3194	3202	else => &windows.ntdll.NtDeviceIoControlFile,
3195	3203	};
3196	3204	if (o.file.flags.nonblocking) {
3197		context.file = o.file.handle;
	3205	operation_userdata.file = o.file.handle;
3198	3206	switch (NtControlFile(
3199	3207	o.file.handle,
3200	3208	null, // event
3201	3209	&batchApc,
3202	3210	b,
3203		&context.iosb,
	3211	&operation_userdata.iosb,
3204	3212	o.code,
3205	3213	if (o.in.len > 0) o.in.ptr else null,
3206	3214	@intCast(o.in.len),
...	...	@@ -3210,8 +3218,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3210	3218	.PENDING, .SUCCESS => {},
3211	3219	.CANCELLED => unreachable,
3212	3220	else => \|status\| {
3213		context.iosb.u.Status = status;
3214		batchApc(b, &context.iosb, 0);
	3221	operation_userdata.iosb.u.Status = status;
	3222	batchApc(b, &operation_userdata.iosb, 0);
3215	3223	},
3216	3224	}
3217	3225	} else {
...	...	@@ -3223,7 +3231,7 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3223	3231	null, // event
3224	3232	null, // APC routine
3225	3233	null, // APC context
3226		&context.iosb,
	3234	&operation_userdata.iosb,
3227	3235	o.code,
3228	3236	if (o.in.len > 0) o.in.ptr else null,
3229	3237	@intCast(o.in.len),
...	...	@@ -3237,9 +3245,8 @@ fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentErr
3237	3245	},
3238	3246	else => \|status\| {
3239	3247	syscall.finish();
3240
3241		context.iosb.u.Status = status;
3242		batchApc(b, &context.iosb, 0);
	3248	operation_userdata.iosb.u.Status = status;
	3249	batchApc(b, &operation_userdata.iosb, 0);
3243	3250	break;
3244	3251	},
3245	3252	};
...	...	@@ -3816,7 +3823,13 @@ fn filePathKind(t: *Threaded, dir: Dir, sub_path: []const u8) !File.Kind {
3816	3823	const syscall: Syscall = try .start();
3817	3824	while (true) {
3818	3825	var statx = std.mem.zeroes(linux.Statx);
3819		switch (linux.errno(linux.statx(dir.handle, sub_path_posix, 0, .{ .TYPE = true }, &statx))) {
	3826	switch (linux.errno(linux.statx(
	3827	dir.handle,
	3828	sub_path_posix,
	3829	linux.AT.NO_AUTOMOUNT \| linux.AT.SYMLINK_NOFOLLOW,
	3830	.{ .TYPE = true },
	3831	&statx,
	3832	))) {
3820	3833	.SUCCESS => {
3821	3834	syscall.finish();
3822	3835	if (!statx.mask.TYPE) return error.Unexpected;
...	...	@@ -3832,7 +3845,7 @@ fn filePathKind(t: *Threaded, dir: Dir, sub_path: []const u8) !File.Kind {
3832	3845	}
3833	3846	}
3834	3847
3835		const stat = try dirStatFile(t, dir, sub_path, .{});
	3848	const stat = try dirStatFile(t, dir, sub_path, .{ .follow_symlinks = false });
3836	3849	return stat.kind;
3837	3850	}
3838	3851
...	...	@@ -13573,13 +13586,13 @@ fn netWriteWindows(
13573	13586	addWsaBuf(&iovecs, &len, header);
13574	13587	for (data[0 .. data.len - 1]) \|bytes\| addWsaBuf(&iovecs, &len, bytes);
13575	13588	const pattern = data[data.len - 1];
	13589	var backup_buffer: [64]u8 = undefined;
13576	13590	if (iovecs.len - len != 0) switch (splat) {
13577	13591	0 => {},
13578	13592	1 => addWsaBuf(&iovecs, &len, pattern),
13579	13593	else => switch (pattern.len) {
13580	13594	0 => {},
13581	13595	1 => {
13582		var backup_buffer: [64]u8 = undefined;
13583	13596	const splat_buffer = &backup_buffer;
13584	13597	const memset_len = @min(splat_buffer.len, splat);
13585	13598	const buf = splat_buffer[0..memset_len];
...	...	@@ -14519,7 +14532,7 @@ pub fn statFromLinux(stx: *const std.os.linux.Statx) Io.UnexpectedError!File.Sta
14519	14532	};
14520	14533	}
14521	14534
14522		fn statxKind(stx_mode: u16) File.Kind {
	14535	pub fn statxKind(stx_mode: u16) File.Kind {
14523	14536	return switch (stx_mode & std.os.linux.S.IFMT) {
14524	14537	std.os.linux.S.IFDIR => .directory,
14525	14538	std.os.linux.S.IFCHR => .character_device,
...	...	@@ -14532,7 +14545,7 @@ fn statxKind(stx_mode: u16) File.Kind {
14532	14545	};
14533	14546	}
14534	14547
14535		fn statFromPosix(st: *const posix.Stat) File.Stat {
	14548	pub fn statFromPosix(st: *const posix.Stat) File.Stat {
14536	14549	const atime = st.atime();
14537	14550	const mtime = st.mtime();
14538	14551	const ctime = st.ctime();
...	...	@@ -15116,7 +15129,9 @@ const WindowsEnvironStrings = struct {
15116	15129	fn scanEnviron(t: *Threaded) void {
15117	15130	mutexLock(&t.mutex);
15118	15131	defer mutexUnlock(&t.mutex);
	15132	if (t.environ_initialized) return;
15119	15133	t.environ.scan(t.allocator);
	15134	t.environ_initialized = true;
15120	15135	}
15121	15136
15122	15137	fn processReplace(userdata: ?*anyopaque, options: process.ReplaceOptions) process.ReplaceError {
...	...	@@ -15256,7 +15271,7 @@ fn spawnPosix(t: *Threaded, options: process.SpawnOptions) process.SpawnError!Sp
15256	15271
15257	15272	// This pipe communicates to the parent errors in the child between `fork` and `execvpe`.
15258	15273	// It is closed by the child (via CLOEXEC) without writing if `execvpe` succeeds.
15259		const err_pipe: [2]posix.fd_t = try pipe2(.{ .CLOEXEC = true });
	15274	const err_pipe = try pipe2(.{ .CLOEXEC = true });
15260	15275	errdefer destroyPipe(err_pipe);
15261	15276
15262	15277	t.scanEnviron(); // for PATH
...	...	@@ -15327,7 +15342,7 @@ fn spawnPosix(t: *Threaded, options: process.SpawnOptions) process.SpawnError!Sp
15327	15342	}
15328	15343
15329	15344	if (options.start_suspended) {
15330		switch (posix.errno(posix.system.kill(posix.system.getpid(), .STOP))) {
	15345	switch (posix.errno(posix.system.kill(0, .STOP))) {
15331	15346	.SUCCESS => {},
15332	15347	.PERM => forkBail(ep1, error.PermissionDenied),
15333	15348	else => forkBail(ep1, error.Unexpected),
...	...	@@ -15539,15 +15554,15 @@ fn childCleanupWindows(child: *process.Child) void {
15539	15554	windows.CloseHandle(child.thread_handle);
15540	15555	child.thread_handle = undefined;
15541	15556
15542		if (child.stdin) \|*stdin\| {
	15557	if (child.stdin) \|stdin\| {
15543	15558	windows.CloseHandle(stdin.handle);
15544	15559	child.stdin = null;
15545	15560	}
15546		if (child.stdout) \|*stdout\| {
	15561	if (child.stdout) \|stdout\| {
15547	15562	windows.CloseHandle(stdout.handle);
15548	15563	child.stdout = null;
15549	15564	}
15550		if (child.stderr) \|*stderr\| {
	15565	if (child.stderr) \|stderr\| {
15551	15566	windows.CloseHandle(stderr.handle);
15552	15567	child.stderr = null;
15553	15568	}
...	...	@@ -15621,7 +15636,7 @@ fn childWaitPosix(child: *process.Child) process.Child.WaitError!process.Child.T
15621	15636	};
15622	15637	}
15623	15638
15624		fn statusToTerm(status: u32) process.Child.Term {
	15639	pub fn statusToTerm(status: u32) process.Child.Term {
15625	15640	return if (posix.W.IFEXITED(status))
15626	15641	.{ .exited = posix.W.EXITSTATUS(status) }
15627	15642	else if (posix.W.IFSIGNALED(status))
...	...	@@ -15677,15 +15692,15 @@ fn childKillPosix(child: *process.Child) !void {
15677	15692	}
15678	15693
15679	15694	fn childCleanupPosix(child: *process.Child) void {
15680		if (child.stdin) \|*stdin\| {
	15695	if (child.stdin) \|stdin\| {
15681	15696	closeFd(stdin.handle);
15682	15697	child.stdin = null;
15683	15698	}
15684		if (child.stdout) \|*stdout\| {
	15699	if (child.stdout) \|stdout\| {
15685	15700	closeFd(stdout.handle);
15686	15701	child.stdout = null;
15687	15702	}
15688		if (child.stderr) \|*stderr\| {
	15703	if (child.stderr) \|stderr\| {
15689	15704	closeFd(stderr.handle);
15690	15705	child.stderr = null;
15691	15706	}
...	...	@@ -15818,21 +15833,57 @@ fn processSpawnWindows(userdata: ?*anyopaque, options: process.SpawnOptions) pro
15818	15833	.dwFlags = windows.STARTF_USESTDHANDLES,
15819	15834	.hStdInput = switch (options.stdin) {
15820	15835	.inherit => peb.ProcessParameters.hStdInput,
15821		.file => \|file\| file.handle,
	15836	.file => \|file\| try OpenFile(&.{}, .{
	15837	.access_mask = .{
	15838	.STANDARD = .{ .SYNCHRONIZE = true },
	15839	.GENERIC = .{ .READ = true },
	15840	},
	15841	.dir = file.handle,
	15842	.sa = &.{
	15843	.nLength = @sizeOf(windows.SECURITY_ATTRIBUTES),
	15844	.lpSecurityDescriptor = null,
	15845	.bInheritHandle = windows.TRUE,
	15846	},
	15847	.creation = .OPEN,
	15848	}),
15822	15849	.ignore => nul_handle,
15823	15850	.pipe => stdin_pipe[1],
15824	15851	.close => null,
15825	15852	},
15826	15853	.hStdOutput = switch (options.stdout) {
15827	15854	.inherit => peb.ProcessParameters.hStdOutput,
15828		.file => \|file\| file.handle,
	15855	.file => \|file\| try OpenFile(&.{}, .{
	15856	.access_mask = .{
	15857	.STANDARD = .{ .SYNCHRONIZE = true },
	15858	.GENERIC = .{ .WRITE = true },
	15859	},
	15860	.dir = file.handle,
	15861	.sa = &.{
	15862	.nLength = @sizeOf(windows.SECURITY_ATTRIBUTES),
	15863	.lpSecurityDescriptor = null,
	15864	.bInheritHandle = windows.TRUE,
	15865	},
	15866	.creation = .OPEN,
	15867	}),
15829	15868	.ignore => nul_handle,
15830	15869	.pipe => stdout_pipe[1],
15831	15870	.close => null,
15832	15871	},
15833	15872	.hStdError = switch (options.stderr) {
15834	15873	.inherit => peb.ProcessParameters.hStdError,
15835		.file => \|file\| file.handle,
	15874	.file => \|file\| try OpenFile(&.{}, .{
	15875	.access_mask = .{
	15876	.STANDARD = .{ .SYNCHRONIZE = true },
	15877	.GENERIC = .{ .WRITE = true },
	15878	},
	15879	.dir = file.handle,
	15880	.sa = &.{
	15881	.nLength = @sizeOf(windows.SECURITY_ATTRIBUTES),
	15882	.lpSecurityDescriptor = null,
	15883	.bInheritHandle = windows.TRUE,
	15884	},
	15885	.creation = .OPEN,
	15886	}),
15836	15887	.ignore => nul_handle,
15837	15888	.pipe => stderr_pipe[1],
15838	15889	.close => null,
...	...	@@ -16030,6 +16081,10 @@ fn processSpawnWindows(userdata: ?*anyopaque, options: process.SpawnOptions) pro
16030	16081	.id = piProcInfo.hProcess,
16031	16082	.thread_handle = piProcInfo.hThread,
16032	16083	.stdin = stdin: switch (options.stdin) {
	16084	.file => {
	16085	windows.CloseHandle(siStartInfo.hStdInput.?);
	16086	break :stdin null;
	16087	},
16033	16088	.pipe => {
16034	16089	windows.CloseHandle(stdin_pipe[1]);
16035	16090	break :stdin .{ .handle = stdin_pipe[0], .flags = .{ .nonblocking = false } };
...	...	@@ -16037,6 +16092,10 @@ fn processSpawnWindows(userdata: ?*anyopaque, options: process.SpawnOptions) pro
16037	16092	else => null,
16038	16093	},
16039	16094	.stdout = stdout: switch (options.stdout) {
	16095	.file => {
	16096	windows.CloseHandle(siStartInfo.hStdOutput.?);
	16097	break :stdout null;
	16098	},
16040	16099	.pipe => {
16041	16100	windows.CloseHandle(stdout_pipe[1]);
16042	16101	break :stdout .{ .handle = stdout_pipe[0], .flags = .{ .nonblocking = true } };
...	...	@@ -16044,6 +16103,10 @@ fn processSpawnWindows(userdata: ?*anyopaque, options: process.SpawnOptions) pro
16044	16103	else => null,
16045	16104	},
16046	16105	.stderr = stderr: switch (options.stderr) {
	16106	.file => {
	16107	windows.CloseHandle(siStartInfo.hStdError.?);
	16108	break :stderr null;
	16109	},
16047	16110	.pipe => {
16048	16111	windows.CloseHandle(stderr_pipe[1]);
16049	16112	break :stderr .{ .handle = stderr_pipe[0], .flags = .{ .nonblocking = true } };
...	...	@@ -16054,6 +16117,8 @@ fn processSpawnWindows(userdata: ?*anyopaque, options: process.SpawnOptions) pro
16054	16117	};
16055	16118	}
16056	16119
	16120	fn inheritFile() windows.HANDLE {}
	16121
16057	16122	fn getCngDevice(t: *Threaded) Io.RandomSecureError!windows.HANDLE {
16058	16123	{
16059	16124	mutexLock(&t.mutex);
...	...	@@ -17758,11 +17823,7 @@ const parking_futex = struct {
17758	17823	waiter.node.next = waking_head;
17759	17824	waking_head = &waiter.node;
17760	17825	num_removed += 1;
17761		// Signal to `waiter` that they're about to be unparked, in case we're racing with their
17762		// timeout. See corresponding logic in `wake`.
17763		waiter.address = 0;
17764	17826	}
17765
17766	17827	_ = bucket.num_waiters.fetchSub(num_removed, .monotonic);
17767	17828	}
17768	17829
...	...	@@ -19056,7 +19117,7 @@ const OpenError = error{
19056	19117	const OpenFileOptions = struct {
19057	19118	access_mask: windows.ACCESS_MASK,
19058	19119	dir: ?windows.HANDLE = null,
19059		sa: ?*windows.SECURITY_ATTRIBUTES = null,
	19120	sa: ?*const windows.SECURITY_ATTRIBUTES = null,
19060	19121	share_access: windows.FILE.SHARE = .VALID_FLAGS,
19061	19122	creation: windows.FILE.CREATE_DISPOSITION,
19062	19123	filter: Filter = .non_directory_only,
...	...	@@ -19076,10 +19137,10 @@ const OpenFileOptions = struct {
19076	19137
19077	19138	/// TODO: inline this logic everywhere and delete this function
19078	19139	fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!windows.HANDLE {
19079		if (std.mem.eql(u16, sub_path_w, &[_]u16{'.'}) and options.filter == .non_directory_only) {
	19140	if (std.mem.eql(u16, sub_path_w, &.{'.'}) and options.filter == .non_directory_only) {
19080	19141	return error.IsDir;
19081	19142	}
19082		if (std.mem.eql(u16, sub_path_w, &[_]u16{ '.', '.' }) and options.filter == .non_directory_only) {
	19143	if (std.mem.eql(u16, sub_path_w, &.{ '.', '.' }) and options.filter == .non_directory_only) {
19083	19144	return error.IsDir;
19084	19145	}
19085	19146

lib/std/Io/Uring.zig created+6173

...	...	@@ -0,0 +1,6173 @@
	1	const addressFromPosix = Io.Threaded.addressFromPosix;
	2	const addressToPosix = Io.Threaded.addressToPosix;
	3	const Alignment = std.mem.Alignment;
	4	const Allocator = std.mem.Allocator;
	5	const Argv0 = Io.Threaded.Argv0;
	6	const assert = std.debug.assert;
	7	const builtin = @import("builtin");
	8	const ChdirError = Io.Threaded.ChdirError;
	9	const clockToPosix = Io.Threaded.clockToPosix;
	10	const Csprng = Io.Threaded.Csprng;
	11	const default_PATH = Io.Threaded.default_PATH;
	12	const Dir = Io.Dir;
	13	const Environ = Io.Threaded.Environ;
	14	const errnoBug = Io.Threaded.errnoBug;
	15	const Evented = @This();
	16	const fallbackSeed = Io.Threaded.fallbackSeed;
	17	const fd_t = linux.fd_t;
	18	const File = Io.File;
	19	const Io = std.Io;
	20	const IoUring = linux.IoUring;
	21	const iovec = std.posix.iovec;
	22	const iovec_const = std.posix.iovec_const;
	23	const linux = std.os.linux;
	24	const linux_statx_request = Io.Threaded.linux_statx_request;
	25	const LOCK = std.posix.LOCK;
	26	const log = std.log.scoped(.@"io-uring");
	27	const max_iovecs_len = Io.Threaded.max_iovecs_len;
	28	const nanosecondsFromPosix = Io.Threaded.nanosecondsFromPosix;
	29	const net = Io.net;
	30	const PATH_MAX = linux.PATH_MAX;
	31	const pathToPosix = Io.Threaded.pathToPosix;
	32	const pid_t = linux.pid_t;
	33	const PosixAddress = Io.Threaded.PosixAddress;
	34	const posixAddressFamily = Io.Threaded.posixAddressFamily;
	35	const posixProtocol = Io.Threaded.posixProtocol;
	36	const posixSocketMode = Io.Threaded.posixSocketMode;
	37	const process = std.process;
	38	const recoverableOsBugDetected = Io.Threaded.recoverableOsBugDetected;
	39	const setTimestampToPosix = Io.Threaded.setTimestampToPosix;
	40	const splat_buffer_size = Io.Threaded.splat_buffer_size;
	41	const statFromLinux = Io.Threaded.statFromLinux;
	42	const statxKind = Io.Threaded.statxKind;
	43	const std = @import("../std.zig");
	44	const timestampFromPosix = Io.Threaded.timestampFromPosix;
	45	const unexpectedErrno = std.posix.unexpectedErrno;
	46	const winsize = std.posix.winsize;
	47
	48	const tracy = if (@hasDecl(@import("root"), "tracy")) @import("root").tracy else struct {
	49	const enable = false;
	50	inline fn fiberEnter(fiber: [*:0]const u8) void {
	51	_ = fiber;
	52	}
	53	inline fn fiberLeave() void {}
	54	};
	55
	56	/// Empirically saw >128KB being used by the self-hosted backend to panic.
	57	/// Empirically saw glibc complain about 256KB.
	58	const idle_stack_size = 512 * 1024;
	59
	60	const max_idle_search = 1;
	61	const max_steal_ready_search = 2;
	62	const max_steal_free_search = 4;
	63
	64	backing_allocator_needs_mutex: bool,
	65	backing_allocator_mutex: Io.Mutex,
	66	/// Does not need to be thread-safe if not used elsewhere.
	67	backing_allocator: Allocator,
	68	main_fiber_buffer: [
	69	std.mem.alignForward(usize, @sizeOf(Fiber), @alignOf(Completion)) + @sizeOf(Completion)
	70	]u8 align(@max(@alignOf(Fiber), @alignOf(Completion))),
	71	log2_ring_entries: u4,
	72	threads: Thread.List,
	73	sync_limit: ?Io.Semaphore,
	74
	75	stderr_writer_initialized: bool = false,
	76	stderr_mutex: Io.Mutex,
	77	stderr_writer: File.Writer = .{
	78	.io = undefined,
	79	.interface = Io.File.Writer.initInterface(&.{}),
	80	.file = .stderr(),
	81	.mode = .streaming,
	82	},
	83	stderr_mode: Io.Terminal.Mode = .no_color,
	84
	85	environ_mutex: Io.Mutex,
	86	environ_initialized: bool,
	87	environ: Environ,
	88
	89	null_fd: CachedFd,
	90	random_fd: CachedFd,
	91
	92	csprng_mutex: Io.Mutex,
	93	csprng: Csprng,
	94
	95	const Thread = struct {
	96	required_align: void align(4),
	97	thread: std.Thread,
	98	idle_context: Io.fiber.Context,
	99	current_context: *Io.fiber.Context,
	100	ready_queue: ?*Fiber,
	101	free_queue: ?*Fiber,
	102	io_uring: IoUring,
	103	idle_search_index: u32,
	104	steal_ready_search_index: u32,
	105	steal_free_search_index: u32,
	106	name_arena: if (tracy.enable) std.heap.ArenaAllocator.State else struct {},
	107	csprng: Csprng,
	108
	109	threadlocal var self: ?*Thread = null;
	110
	111	noinline fn current() *Thread {
	112	return self.?;
	113	}
	114
	115	fn deinit(thread: *Thread, gpa: Allocator) void {
	116	var next_fiber = thread.free_queue;
	117	while (next_fiber) \|free_fiber\| {
	118	next_fiber = free_fiber.status.free_next;
	119	gpa.free(free_fiber.allocatedSlice());
	120	}
	121	thread.io_uring.deinit();
	122	}
	123
	124	fn currentFiber(thread: Thread) Fiber {
	125	assert(thread.current_context != &thread.idle_context);
	126	return @fieldParentPtr("context", thread.current_context);
	127	}
	128
	129	fn enqueue(thread: Thread) linux.io_uring_sqe {
	130	while (true) return thread.io_uring.get_sqe() catch {
	131	thread.submit();
	132	continue;
	133	};
	134	}
	135
	136	fn submit(thread: *Thread) void {
	137	_ = thread.io_uring.submit() catch \|err\| switch (err) {
	138	error.SignalInterrupt => {},
	139	else => \|e\| @panic(@errorName(e)),
	140	};
	141	}
	142
	143	const List = struct {
	144	allocated: []Thread,
	145	reserved: u32,
	146	active: u32,
	147	};
	148	};
	149
	150	const Fiber = struct {
	151	required_align: void align(4),
	152	context: Io.fiber.Context,
	153	await_count: i32,
	154	link: union {
	155	awaiter: ?*Fiber,
	156	group: struct { prev: ?Fiber, next: ?Fiber },
	157	},
	158	status: union(enum) {
	159	queue_next: ?*Fiber,
	160	awaiting_group: Group,
	161	free_next: ?*Fiber,
	162	},
	163	cancel_status: CancelStatus,
	164	cancel_protection: CancelProtection,
	165	name: if (tracy.enable) [*:0]const u8 else void,
	166
	167	var next_name: u64 = 0;
	168
	169	const CancelStatus = packed struct(u32) {
	170	requested: bool,
	171	awaiting: Awaiting,
	172
	173	const unrequested: CancelStatus = .{ .requested = false, .awaiting = .nothing };
	174
	175	const Awaiting = enum(u31) {
	176	nothing = std.math.maxInt(u31),
	177	group = std.math.maxInt(u31) - 1,
	178	select = std.math.maxInt(u31) - 2,
	179	/// An io_uring fd.
	180	_,
	181
	182	fn subWrap(lhs: Awaiting, rhs: Awaiting) Awaiting {
	183	return @enumFromInt(@intFromEnum(lhs) -% @intFromEnum(rhs));
	184	}
	185
	186	fn fromIoUringFd(fd: fd_t) Awaiting {
	187	const awaiting: Awaiting = @enumFromInt(fd);
	188	switch (awaiting) {
	189	.nothing, .group, .select => unreachable,
	190	_ => return awaiting,
	191	}
	192	}
	193
	194	fn toIoUringFd(awaiting: Awaiting) fd_t {
	195	switch (awaiting) {
	196	.nothing, .group, .select => unreachable,
	197	_ => return @intFromEnum(awaiting),
	198	}
	199	}
	200	};
	201
	202	fn changeAwaiting(
	203	cancel_status: *CancelStatus,
	204	old_awaiting: Awaiting,
	205	new_awaiting: Awaiting,
	206	) bool {
	207	const old_cancel_status = @atomicRmw(CancelStatus, cancel_status, .Add, .{
	208	.requested = false,
	209	.awaiting = new_awaiting.subWrap(old_awaiting),
	210	}, .monotonic);
	211	assert(old_cancel_status.awaiting == old_awaiting);
	212	return old_cancel_status.requested;
	213	}
	214	};
	215
	216	const CancelProtection = packed struct {
	217	user: Io.CancelProtection,
	218	acknowledged: bool,
	219
	220	const unblocked: CancelProtection = .{ .user = .unblocked, .acknowledged = false };
	221
	222	fn check(cancel_protection: CancelProtection) Io.CancelProtection {
	223	return @enumFromInt(@intFromBool(cancel_protection != unblocked));
	224	}
	225
	226	fn acknowledge(cancel_protection: *CancelProtection) void {
	227	assert(!cancel_protection.acknowledged);
	228	cancel_protection.acknowledged = true;
	229	}
	230
	231	fn recancel(cancel_protection: *CancelProtection) void {
	232	assert(cancel_protection.acknowledged);
	233	cancel_protection.acknowledged = false;
	234	}
	235
	236	test check {
	237	try std.testing.expectEqual(Io.CancelProtection.unblocked, check(.unblocked));
	238	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
	239	.user = .unblocked,
	240	.acknowledged = true,
	241	}));
	242	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
	243	.user = .blocked,
	244	.acknowledged = false,
	245	}));
	246	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
	247	.user = .blocked,
	248	.acknowledged = true,
	249	}));
	250	}
	251	};
	252
	253	const finished: ?*Fiber = @ptrFromInt(@alignOf(Fiber));
	254
	255	const max_result_align: Alignment = .@"16";
	256	const max_result_size = max_result_align.forward(512);
	257	/// This includes any stack realignments that need to happen, and also the
	258	/// initial frame return address slot and argument frame, depending on target.
	259	const min_stack_size = 60 * 1024 * 1024;
	260	const max_context_align: Alignment = .@"16";
	261	const max_context_size = max_context_align.forward(1024);
	262	const max_closure_size: usize = @sizeOf(AsyncClosure);
	263	const max_closure_align: Alignment = .of(AsyncClosure);
	264	const allocation_size = std.mem.alignForward(
	265	usize,
	266	max_closure_align.max(max_context_align).forward(
	267	max_result_align.forward(@sizeOf(Fiber)) + max_result_size + min_stack_size,
	268	) + max_closure_size + max_context_size,
	269	std.heap.page_size_max,
	270	);
	271	comptime {
	272	assert(max_result_align.compare(.gte, .of(Completion)));
	273	assert(max_result_size >= @sizeOf(Completion));
	274	}
	275
	276	fn create(ev: Evented) error{OutOfMemory}!Fiber {
	277	const thread: *Thread = .current();
	278	if (@atomicRmw(?*Fiber, &thread.free_queue, .Xchg, finished, .acquire)) \|free_fiber\| {
	279	assert(free_fiber != finished);
	280	@atomicStore(?*Fiber, &thread.free_queue, free_fiber.status.free_next, .release);
	281	return free_fiber;
	282	}
	283	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
	284	for (0..@min(max_steal_free_search, active_threads)) \|_\| {
	285	defer thread.steal_free_search_index += 1;
	286	if (thread.steal_free_search_index == active_threads) thread.steal_free_search_index = 0;
	287	const steal_free_search_thread =
	288	&ev.threads.allocated[0..active_threads][thread.steal_free_search_index];
	289	if (steal_free_search_thread == thread) continue;
	290	const free_fiber =
	291	@atomicLoad(?*Fiber, &steal_free_search_thread.free_queue, .monotonic) orelse continue;
	292	if (free_fiber == finished) continue;
	293	if (@cmpxchgWeak(
	294	?*Fiber,
	295	&steal_free_search_thread.free_queue,
	296	free_fiber,
	297	null,
	298	.acquire,
	299	.monotonic,
	300	)) \|_\| continue;
	301	@atomicStore(?*Fiber, &thread.free_queue, free_fiber.status.free_next, .release);
	302	return free_fiber;
	303	}
	304	@atomicStore(?*Fiber, &thread.free_queue, null, .monotonic);
	305	return @ptrCast(try ev.allocator().alignedAlloc(u8, .of(Fiber), allocation_size));
	306	}
	307
	308	fn destroy(fiber: *Fiber) void {
	309	const thread: *Thread = .current();
	310	assert(fiber.status.queue_next == null);
	311	fiber.status = .{ .free_next = @atomicLoad(?*Fiber, &thread.free_queue, .acquire) };
	312	while (true) fiber.status.free_next = @cmpxchgWeak(
	313	?*Fiber,
	314	&thread.free_queue,
	315	fiber.status.free_next,
	316	fiber,
	317	.acq_rel,
	318	.acquire,
	319	) orelse break;
	320	}
	321
	322	fn allocatedSlice(f: *Fiber) []align(@alignOf(Fiber)) u8 {
	323	return @as([*]align(@alignOf(Fiber)) u8, @ptrCast(f))[0..allocation_size];
	324	}
	325
	326	fn allocatedEnd(f: Fiber) []u8 {
	327	const allocated_slice = f.allocatedSlice();
	328	return allocated_slice[allocated_slice.len..].ptr;
	329	}
	330
	331	fn resultPointer(f: Fiber, comptime Result: type) Result {
	332	return @ptrCast(@alignCast(f.resultBytes(.of(Result))));
	333	}
	334
	335	fn resultBytes(f: Fiber, alignment: Alignment) []u8 {
	336	return @ptrFromInt(alignment.forward(@intFromPtr(f) + @sizeOf(Fiber)));
	337	}
	338
	339	const Queue = struct { head: Fiber, tail: Fiber };
	340
	341	/// Like a `*Fiber`, but 2 bits smaller than a pointer (because the LSBs are always 0 due to
	342	/// alignment) so that those two bits can be used in a `packed struct`.
	343	const PackedPtr = enum(@Int(.unsigned, @bitSizeOf(usize) - 2)) {
	344	null = 0,
	345	all_ones = std.math.maxInt(@Int(.unsigned, @bitSizeOf(usize) - 2)),
	346	_,
	347
	348	const Split = packed struct(usize) { low: u2, high: PackedPtr };
	349	fn pack(ptr: ?*Fiber) PackedPtr {
	350	const split: Split = @bitCast(@intFromPtr(ptr));
	351	assert(split.low == 0);
	352	return split.high;
	353	}
	354	fn unpack(ptr: PackedPtr) ?*Fiber {
	355	const split: Split = .{ .low = 0, .high = ptr };
	356	return @ptrFromInt(@as(usize, @bitCast(split)));
	357	}
	358	};
	359
	360	fn requestCancel(fiber: Fiber, ev: Evented) void {
	361	const cancel_status = @atomicRmw(
	362	Fiber.CancelStatus,
	363	&fiber.cancel_status,
	364	.Or,
	365	.{ .requested = true, .awaiting = @enumFromInt(0) },
	366	.acquire,
	367	);
	368	assert(!cancel_status.requested);
	369	switch (cancel_status.awaiting) {
	370	.nothing => {},
	371	.group => {
	372	// The awaiter received a cancelation request while awaiting a group,
	373	// so propagate the cancelation to the group.
	374	if (fiber.status.awaiting_group.cancel(ev, null)) {
	375	fiber.status = .{ .queue_next = null };
	376	_ = ev.schedule(.current(), .{ .head = fiber, .tail = fiber });
	377	}
	378	},
	379	.select => if (@atomicRmw(i32, &fiber.await_count, .Add, 1, .monotonic) == -1) {
	380	_ = ev.schedule(.current(), .{ .head = fiber, .tail = fiber });
	381	},
	382	_ => \|awaiting\| {
	383	const awaiting_io_uring_fd = awaiting.toIoUringFd();
	384	const thread: *Thread = .current();
	385	thread.enqueue().* = if (thread.io_uring.fd == awaiting_io_uring_fd) .{
	386	.opcode = .ASYNC_CANCEL,
	387	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	388	.ioprio = 0,
	389	.fd = 0,
	390	.off = 0,
	391	.addr = @intFromPtr(fiber),
	392	.len = 0,
	393	.rw_flags = 0,
	394	.user_data = @intFromEnum(Completion.Userdata.wakeup),
	395	.buf_index = 0,
	396	.personality = 0,
	397	.splice_fd_in = 0,
	398	.addr3 = 0,
	399	.resv = 0,
	400	} else .{
	401	.opcode = .MSG_RING,
	402	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	403	.ioprio = 0,
	404	.fd = awaiting_io_uring_fd,
	405	.off = @intFromPtr(fiber) \| 0b01,
	406	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
	407	.len = 0,
	408	.rw_flags = 0,
	409	.user_data = @intFromEnum(Completion.Userdata.cleanup),
	410	.buf_index = 0,
	411	.personality = 0,
	412	.splice_fd_in = 0,
	413	.addr3 = 0,
	414	.resv = 0,
	415	};
	416	},
	417	}
	418	}
	419	};
	420
	421	const CancelRegion = struct {
	422	fiber: *Fiber,
	423	status: Fiber.CancelStatus,
	424	fn init() CancelRegion {
	425	const fiber = Thread.current().currentFiber();
	426	return .{
	427	.fiber = fiber,
	428	.status = .{
	429	.requested = fiber.cancel_protection.check() == .unblocked,
	430	.awaiting = .nothing,
	431	},
	432	};
	433	}
	434	fn initBlocked() CancelRegion {
	435	return .{
	436	.fiber = Thread.current().currentFiber(),
	437	.status = .{ .requested = false, .awaiting = .nothing },
	438	};
	439	}
	440	fn deinit(cancel_region: *CancelRegion) void {
	441	if (cancel_region.status.requested) {
	442	@branchHint(.likely);
	443	_ = cancel_region.fiber.cancel_status.changeAwaiting(
	444	cancel_region.status.awaiting,
	445	.nothing,
	446	);
	447	}
	448	cancel_region.* = undefined;
	449	}
	450	fn await(cancel_region: *CancelRegion, awaiting: Fiber.CancelStatus.Awaiting) Io.Cancelable!void {
	451	if (!cancel_region.status.requested) {
	452	@branchHint(.unlikely);
	453	return;
	454	}
	455	const status: Fiber.CancelStatus = .{ .requested = true, .awaiting = awaiting };
	456	if (cancel_region.fiber.cancel_status.changeAwaiting(
	457	cancel_region.status.awaiting,
	458	status.awaiting,
	459	)) {
	460	@branchHint(.unlikely);
	461	cancel_region.fiber.cancel_protection.acknowledge();
	462	cancel_region.status = .unrequested;
	463	return error.Canceled;
	464	}
	465	cancel_region.status = status;
	466	}
	467	fn awaitIoUring(cancel_region: CancelRegion) Io.Cancelable!Thread {
	468	const thread: *Thread = .current();
	469	try cancel_region.await(.fromIoUringFd(thread.io_uring.fd));
	470	return thread;
	471	}
	472	fn completion(cancel_region: *const CancelRegion) Completion {
	473	return cancel_region.fiber.resultPointer(Completion).*;
	474	}
	475	fn errno(cancel_region: *const CancelRegion) linux.E {
	476	return cancel_region.completion().errno();
	477	}
	478
	479	const Sync = struct {
	480	cancel_region: CancelRegion,
	481	fn init(ev: *Evented) Io.Cancelable!Sync {
	482	if (ev.sync_limit) \|*sync_limit\| try sync_limit.wait(ev.io());
	483	return .{ .cancel_region = .init() };
	484	}
	485	fn initBlocked(ev: *Evented) Sync {
	486	if (ev.sync_limit) \|*sync_limit\| sync_limit.waitUncancelable(ev.io());
	487	return .{ .cancel_region = .initBlocked() };
	488	}
	489	fn deinit(sync: Sync, ev: Evented) void {
	490	sync.cancel_region.deinit();
	491	if (ev.sync_limit) \|*sync_limit\| sync_limit.post(ev.io());
	492	}
	493
	494	const Maybe = union(enum) {
	495	cancel_region: CancelRegion,
	496	sync: Sync,
	497
	498	fn deinit(maybe: Maybe, ev: Evented) void {
	499	switch (maybe.*) {
	500	.cancel_region => \|*cancel_region\| cancel_region.deinit(),
	501	.sync => \|*sync\| sync.deinit(ev),
	502	}
	503	}
	504
	505	fn enterSync(maybe: Maybe, ev: Evented) Io.Cancelable!*Sync {
	506	switch (maybe.*) {
	507	.cancel_region => \|cancel_region\| {
	508	if (ev.sync_limit) \|*sync_limit\| try sync_limit.wait(ev.io());
	509	maybe.* = .{ .sync = .{ .cancel_region = cancel_region } };
	510	},
	511	.sync => {},
	512	}
	513	return &maybe.sync;
	514	}
	515
	516	fn leaveSync(maybe: Maybe, ev: Evented) void {
	517	switch (maybe.*) {
	518	.cancel_region => {},
	519	.sync => \|sync\| {
	520	if (ev.sync_limit) \|*sync_limit\| sync_limit.post(ev.io());
	521	maybe.* = .{ .cancel_region = sync.cancel_region };
	522	},
	523	}
	524	}
	525
	526	fn cancelRegion(maybe: Maybe) CancelRegion {
	527	return switch (maybe.*) {
	528	.cancel_region => \|*cancel_region\| cancel_region,
	529	.sync => \|*sync\| &sync.cancel_region,
	530	};
	531	}
	532	};
	533	};
	534	};
	535
	536	const CachedFd = struct {
	537	once: Once,
	538
	539	const Once = enum(fd_t) {
	540	uninitialized = -1,
	541	initializing = -2,
	542	/// fd
	543	_,
	544
	545	fn fromFd(fd: fd_t) Once {
	546	return @enumFromInt(@as(u31, @intCast(fd)));
	547	}
	548
	549	fn toFd(once: Once) fd_t {
	550	return @as(u31, @intCast(@intFromEnum(once)));
	551	}
	552	};
	553
	554	const init: CachedFd = .{ .once = .uninitialized };
	555
	556	fn close(cached_fd: *CachedFd) void {
	557	switch (cached_fd.once) {
	558	.uninitialized => {},
	559	.initializing => unreachable,
	560	_ => \|fd\| {
	561	assert(@intFromEnum(fd) >= 0);
	562	_ = linux.close(@intFromEnum(fd));
	563	cached_fd.* = .init;
	564	},
	565	}
	566	}
	567
	568	fn open(
	569	cached_fd: *CachedFd,
	570	ev: *Evented,
	571	cancel_region: *CancelRegion,
	572	path: [*:0]const u8,
	573	flags: linux.O,
	574	) File.OpenError!fd_t {
	575	var once = @atomicLoad(Once, &cached_fd.once, .monotonic);
	576	while (true) {
	577	switch (once) {
	578	.uninitialized => {},
	579	.initializing => try futexWait(
	580	ev,
	581	@ptrCast(&cached_fd.once),
	582	@bitCast(@intFromEnum(once)),
	583	.none,
	584	),
	585	_ => \|fd\| {
	586	@branchHint(.likely);
	587	return fd.toFd();
	588	},
	589	}
	590	once = @cmpxchgWeak(
	591	Once,
	592	&cached_fd.once,
	593	.uninitialized,
	594	.initializing,
	595	.monotonic,
	596	.monotonic,
	597	) orelse {
	598	errdefer {
	599	@atomicStore(Once, &cached_fd.once, .uninitialized, .monotonic);
	600	futexWake(ev, @ptrCast(&cached_fd.once), 1);
	601	}
	602	const fd = try ev.openat(cancel_region, linux.AT.FDCWD, path, flags, 0);
	603	@atomicStore(Once, &cached_fd.once, .fromFd(fd), .monotonic);
	604	futexWake(ev, @ptrCast(&cached_fd.once), std.math.maxInt(u32));
	605	return fd;
	606	};
	607	}
	608	}
	609	};
	610
	611	pub fn allocator(ev: *Evented) std.mem.Allocator {
	612	return if (ev.backing_allocator_needs_mutex) .{
	613	.ptr = ev,
	614	.vtable = &.{
	615	.alloc = alloc,
	616	.resize = resize,
	617	.remap = remap,
	618	.free = free,
	619	},
	620	} else ev.backing_allocator;
	621	}
	622
	623	fn alloc(userdata: anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[]u8 {
	624	const ev: *Evented = @ptrCast(@alignCast(userdata));
	625	const ev_io = ev.io();
	626	ev.backing_allocator_mutex.lockUncancelable(ev_io);
	627	defer ev.backing_allocator_mutex.unlock(ev_io);
	628	return ev.backing_allocator.rawAlloc(len, alignment, ret_addr);
	629	}
	630
	631	fn resize(
	632	userdata: *anyopaque,
	633	memory: []u8,
	634	alignment: std.mem.Alignment,
	635	new_len: usize,
	636	ret_addr: usize,
	637	) bool {
	638	const ev: *Evented = @ptrCast(@alignCast(userdata));
	639	const ev_io = ev.io();
	640	ev.backing_allocator_mutex.lockUncancelable(ev_io);
	641	defer ev.backing_allocator_mutex.unlock(ev_io);
	642	return ev.backing_allocator.rawResize(memory, alignment, new_len, ret_addr);
	643	}
	644
	645	fn remap(
	646	userdata: *anyopaque,
	647	memory: []u8,
	648	alignment: Alignment,
	649	new_len: usize,
	650	ret_addr: usize,
	651	) ?[*]u8 {
	652	const ev: *Evented = @ptrCast(@alignCast(userdata));
	653	const ev_io = ev.io();
	654	ev.backing_allocator_mutex.lockUncancelable(ev_io);
	655	defer ev.backing_allocator_mutex.unlock(ev_io);
	656	return ev.backing_allocator.rawRemap(memory, alignment, new_len, ret_addr);
	657	}
	658
	659	fn free(userdata: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void {
	660	const ev: *Evented = @ptrCast(@alignCast(userdata));
	661	const ev_io = ev.io();
	662	ev.backing_allocator_mutex.lockUncancelable(ev_io);
	663	defer ev.backing_allocator_mutex.unlock(ev_io);
	664	return ev.backing_allocator.rawFree(memory, alignment, ret_addr);
	665	}
	666
	667	pub fn io(ev: *Evented) Io {
	668	return .{
	669	.userdata = ev,
	670	.vtable = &.{
	671	.crashHandler = crashHandler,
	672
	673	.async = async,
	674	.concurrent = concurrent,
	675	.await = await,
	676	.cancel = cancel,
	677
	678	.groupAsync = groupAsync,
	679	.groupConcurrent = groupConcurrent,
	680	.groupAwait = groupAwait,
	681	.groupCancel = groupCancel,
	682
	683	.recancel = recancel,
	684	.swapCancelProtection = swapCancelProtection,
	685	.checkCancel = checkCancel,
	686
	687	.select = select,
	688
	689	.futexWait = futexWait,
	690	.futexWaitUncancelable = futexWaitUncancelable,
	691	.futexWake = futexWake,
	692
	693	.operate = operate,
	694	.batchAwaitAsync = batchAwaitAsync,
	695	.batchAwaitConcurrent = batchAwaitConcurrent,
	696	.batchCancel = batchCancel,
	697
	698	.dirCreateDir = dirCreateDir,
	699	.dirCreateDirPath = dirCreateDirPath,
	700	.dirCreateDirPathOpen = dirCreateDirPathOpen,
	701	.dirOpenDir = dirOpenDir,
	702	.dirStat = dirStat,
	703	.dirStatFile = dirStatFile,
	704	.dirAccess = dirAccess,
	705	.dirCreateFile = dirCreateFile,
	706	.dirCreateFileAtomic = dirCreateFileAtomic,
	707	.dirOpenFile = dirOpenFile,
	708	.dirClose = dirClose,
	709	.dirRead = dirRead,
	710	.dirRealPath = dirRealPath,
	711	.dirRealPathFile = dirRealPathFile,
	712	.dirDeleteFile = dirDeleteFile,
	713	.dirDeleteDir = dirDeleteDir,
	714	.dirRename = dirRename,
	715	.dirRenamePreserve = dirRenamePreserve,
	716	.dirSymLink = dirSymLink,
	717	.dirReadLink = dirReadLink,
	718	.dirSetOwner = dirSetOwner,
	719	.dirSetFileOwner = dirSetFileOwner,
	720	.dirSetPermissions = dirSetPermissions,
	721	.dirSetFilePermissions = dirSetFilePermissions,
	722	.dirSetTimestamps = dirSetTimestamps,
	723	.dirHardLink = dirHardLink,
	724
	725	.fileStat = fileStat,
	726	.fileLength = fileLength,
	727	.fileClose = fileClose,
	728	.fileWritePositional = fileWritePositional,
	729	.fileWriteFileStreaming = fileWriteFileStreaming,
	730	.fileWriteFilePositional = fileWriteFilePositional,
	731	.fileReadPositional = fileReadPositional,
	732	.fileSeekBy = fileSeekBy,
	733	.fileSeekTo = fileSeekTo,
	734	.fileSync = fileSync,
	735	.fileIsTty = fileIsTty,
	736	.fileEnableAnsiEscapeCodes = fileEnableAnsiEscapeCodes,
	737	.fileSupportsAnsiEscapeCodes = fileIsTty,
	738	.fileSetLength = fileSetLength,
	739	.fileSetOwner = fileSetOwner,
	740	.fileSetPermissions = fileSetPermissions,
	741	.fileSetTimestamps = fileSetTimestamps,
	742	.fileLock = fileLock,
	743	.fileTryLock = fileTryLock,
	744	.fileUnlock = fileUnlock,
	745	.fileDowngradeLock = fileDowngradeLock,
	746	.fileRealPath = fileRealPath,
	747	.fileHardLink = fileHardLink,
	748
	749	.fileMemoryMapCreate = fileMemoryMapCreate,
	750	.fileMemoryMapDestroy = fileMemoryMapDestroy,
	751	.fileMemoryMapSetLength = fileMemoryMapSetLength,
	752	.fileMemoryMapRead = fileMemoryMapRead,
	753	.fileMemoryMapWrite = fileMemoryMapWrite,
	754
	755	.processExecutableOpen = processExecutableOpen,
	756	.processExecutablePath = processExecutablePath,
	757	.lockStderr = lockStderr,
	758	.tryLockStderr = tryLockStderr,
	759	.unlockStderr = unlockStderr,
	760	.processCurrentPath = processCurrentPath,
	761	.processSetCurrentDir = processSetCurrentDir,
	762	.processReplace = processReplace,
	763	.processReplacePath = processReplacePath,
	764	.processSpawn = processSpawn,
	765	.processSpawnPath = processSpawnPath,
	766	.childWait = childWait,
	767	.childKill = childKill,
	768
	769	.progressParentFile = progressParentFile,
	770
	771	.now = now,
	772	.clockResolution = clockResolution,
	773	.sleep = sleep,
	774
	775	.random = random,
	776	.randomSecure = randomSecure,
	777
	778	.netListenIp = netListenIpUnavailable,
	779	.netAccept = netAcceptUnavailable,
	780	.netBindIp = netBindIp,
	781	.netConnectIp = netConnectIpUnavailable,
	782	.netListenUnix = netListenUnixUnavailable,
	783	.netConnectUnix = netConnectUnixUnavailable,
	784	.netSocketCreatePair = netSocketCreatePairUnavailable,
	785	.netSend = netSendUnavailable,
	786	.netReceive = netReceive,
	787	.netRead = netReadUnavailable,
	788	.netWrite = netWriteUnavailable,
	789	.netWriteFile = netWriteFileUnavailable,
	790	.netClose = netClose,
	791	.netShutdown = netShutdown,
	792	.netInterfaceNameResolve = netInterfaceNameResolveUnavailable,
	793	.netInterfaceName = netInterfaceNameUnavailable,
	794	.netLookup = netLookupUnavailable,
	795	},
	796	};
	797	}
	798
	799	pub const InitOptions = struct {
	800	backing_allocator_needs_mutex: bool = true,
	801
	802	/// Maximum thread pool size (excluding the main thread).
	803	/// Defaults to one less than the number of logical CPU cores.
	804	thread_limit: ?usize = null,
	805	/// Maximum number of threads that may perform synchronous syscalls.
	806	sync_limit: Io.Limit = .unlimited,
	807
	808	log2_ring_entries: u4 = 3,
	809
	810	/// Affects the following operations:
	811	/// * `processExecutablePath` on OpenBSD and Haiku.
	812	argv0: Argv0 = .empty,
	813	/// Affects the following operations:
	814	/// * `fileIsTty`
	815	/// * `processSpawn`, `processSpawnPath`, `processReplace`, `processReplacePath`
	816	environ: process.Environ = .empty,
	817	};
	818
	819	pub fn init(ev: *Evented, backing_allocator: Allocator, options: InitOptions) !void {
	820	const threads_size = @sizeOf(Thread) * if (options.thread_limit) \|thread_limit\|
	821	1 + thread_limit
	822	else
	823	@max(std.Thread.getCpuCount() catch 1, 1);
	824	const idle_stack_end_offset =
	825	std.mem.alignForward(usize, threads_size + idle_stack_size, std.heap.pageSize());
	826	const allocated_slice = try backing_allocator.alignedAlloc(u8, .of(Thread), idle_stack_end_offset);
	827	errdefer backing_allocator.free(allocated_slice);
	828	ev.* = .{
	829	.backing_allocator_needs_mutex = options.backing_allocator_needs_mutex,
	830	.backing_allocator_mutex = .init,
	831	.backing_allocator = backing_allocator,
	832	.main_fiber_buffer = undefined,
	833	.log2_ring_entries = options.log2_ring_entries,
	834	.threads = .{
	835	.allocated = @ptrCast(allocated_slice[0..threads_size]),
	836	.reserved = 1,
	837	.active = 1,
	838	},
	839	.sync_limit = if (options.sync_limit.toInt()) \|sync_limit\| .{ .permits = sync_limit } else null,
	840
	841	.stderr_writer_initialized = false,
	842	.stderr_mutex = .init,
	843	.stderr_writer = .{
	844	.io = ev.io(),
	845	.interface = Io.File.Writer.initInterface(&.{}),
	846	.file = .stderr(),
	847	.mode = .streaming,
	848	},
	849	.stderr_mode = .no_color,
	850
	851	.environ_mutex = .init,
	852	.environ_initialized = options.environ.block.isEmpty(),
	853	.environ = .{ .process_environ = options.environ },
	854
	855	.null_fd = .init,
	856	.random_fd = .init,
	857
	858	.csprng_mutex = .init,
	859	.csprng = .uninitialized,
	860	};
	861	const main_fiber: *Fiber = @ptrCast(&ev.main_fiber_buffer);
	862	main_fiber.* = .{
	863	.required_align = {},
	864	.context = undefined,
	865	.await_count = 0,
	866	.link = .{ .awaiter = null },
	867	.status = .{ .queue_next = null },
	868	.cancel_status = .unrequested,
	869	.cancel_protection = .unblocked,
	870	.name = if (tracy.enable) "main task",
	871	};
	872	const main_thread = &ev.threads.allocated[0];
	873	Thread.self = main_thread;
	874	main_thread.* = .{
	875	.required_align = {},
	876	.thread = undefined,
	877	.idle_context = switch (builtin.cpu.arch) {
	878	.aarch64 => .{
	879	.sp = @intFromPtr(allocated_slice[idle_stack_end_offset..].ptr),
	880	.fp = @intFromPtr(ev),
	881	.pc = @intFromPtr(&mainIdleEntry),
	882	},
	883	.x86_64 => .{
	884	.rsp = @intFromPtr(allocated_slice[idle_stack_end_offset..].ptr),
	885	.rbp = @intFromPtr(ev),
	886	.rip = @intFromPtr(&mainIdleEntry),
	887	},
	888	else => @compileError("unimplemented architecture"),
	889	},
	890	.current_context = &main_fiber.context,
	891	.ready_queue = null,
	892	.free_queue = null,
	893	.io_uring = try .init(
	894	@as(u16, 1) << ev.log2_ring_entries,
	895	linux.IORING_SETUP_COOP_TASKRUN \| linux.IORING_SETUP_SINGLE_ISSUER,
	896	),
	897	.idle_search_index = 1,
	898	.steal_ready_search_index = 1,
	899	.steal_free_search_index = 1,
	900	.name_arena = .{},
	901	.csprng = .uninitialized,
	902	};
	903	errdefer main_thread.io_uring.deinit();
	904	if (tracy.enable) tracy.fiberEnter(main_fiber.name);
	905	}
	906
	907	pub fn deinit(ev: *Evented) void {
	908	const main_fiber: *Fiber = @ptrCast(&ev.main_fiber_buffer);
	909	assert(Thread.current().currentFiber() == main_fiber);
	910	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
	911	for (ev.threads.allocated[0..active_threads]) \|*thread\| {
	912	const ready_fiber = @atomicLoad(?*Fiber, &thread.ready_queue, .monotonic);
	913	assert(ready_fiber == null or ready_fiber == Fiber.finished); // pending async
	914	}
	915	ev.yield(null, .exit);
	916	ev.null_fd.close();
	917	ev.random_fd.close();
	918	const allocated_ptr: [*]align(@alignOf(Thread)) u8 = @ptrCast(@alignCast(ev.threads.allocated.ptr));
	919	const idle_stack_end_offset = std.mem.alignForward(
	920	usize,
	921	ev.threads.allocated.len * @sizeOf(Thread) + idle_stack_size,
	922	std.heap.page_size_max,
	923	);
	924	for (ev.threads.allocated[1..active_threads]) \|*thread\| thread.thread.join();
	925	for (ev.threads.allocated[0..active_threads]) \|*thread\| thread.deinit(ev.backing_allocator);
	926	assert(active_threads == ev.threads.active); // spawned threads while there was no pending async?
	927	ev.backing_allocator.free(allocated_ptr[0..idle_stack_end_offset]);
	928	ev.* = undefined;
	929	}
	930
	931	fn findReadyFiber(ev: Evented, thread: Thread) ?*Fiber {
	932	if (@atomicRmw(?*Fiber, &thread.ready_queue, .Xchg, Fiber.finished, .acquire)) \|ready_fiber\| {
	933	assert(ready_fiber != Fiber.finished);
	934	@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.status.queue_next, .release);
	935	ready_fiber.status.queue_next = null;
	936	return ready_fiber;
	937	}
	938	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
	939	for (0..@min(max_steal_ready_search, active_threads)) \|_\| {
	940	defer thread.steal_ready_search_index += 1;
	941	if (thread.steal_ready_search_index == active_threads) thread.steal_ready_search_index = 0;
	942	const steal_ready_search_thread =
	943	&ev.threads.allocated[0..active_threads][thread.steal_ready_search_index];
	944	if (steal_ready_search_thread == thread) continue;
	945	const ready_fiber =
	946	@atomicLoad(?*Fiber, &steal_ready_search_thread.ready_queue, .monotonic) orelse continue;
	947	if (ready_fiber == Fiber.finished) continue;
	948	if (@cmpxchgWeak(
	949	?*Fiber,
	950	&steal_ready_search_thread.ready_queue,
	951	ready_fiber,
	952	null,
	953	.acquire,
	954	.monotonic,
	955	)) \|_\| continue;
	956	@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.status.queue_next, .release);
	957	ready_fiber.status.queue_next = null;
	958	return ready_fiber;
	959	}
	960	// couldn't find anything to do, so we are now open for business
	961	@atomicStore(?*Fiber, &thread.ready_queue, null, .monotonic);
	962	return null;
	963	}
	964
	965	fn yield(ev: Evented, maybe_ready_fiber: ?Fiber, pending_task: SwitchMessage.PendingTask) void {
	966	const thread: *Thread = .current();
	967	const ready_context = if (maybe_ready_fiber orelse ev.findReadyFiber(thread)) \|ready_fiber\|
	968	&ready_fiber.context
	969	else
	970	&thread.idle_context;
	971	const message: SwitchMessage = .{
	972	.contexts = .{
	973	.old = thread.current_context,
	974	.new = ready_context,
	975	},
	976	.pending_task = pending_task,
	977	};
	978	contextSwitch(&message).handle(ev);
	979	}
	980
	981	fn schedule(ev: Evented, thread: Thread, ready_queue: Fiber.Queue) bool {
	982	// shared fields of previous `Thread` must be initialized before later ones are marked as active
	983	const new_thread_index = @atomicLoad(u32, &ev.threads.active, .acquire);
	984	for (0..@min(max_idle_search, new_thread_index)) \|_\| {
	985	defer thread.idle_search_index += 1;
	986	if (thread.idle_search_index == new_thread_index) thread.idle_search_index = 0;
	987	const idle_search_thread = &ev.threads.allocated[0..new_thread_index][thread.idle_search_index];
	988	if (idle_search_thread == thread) continue;
	989	if (@cmpxchgWeak(
	990	?*Fiber,
	991	&idle_search_thread.ready_queue,
	992	null,
	993	ready_queue.head,
	994	.release,
	995	.monotonic,
	996	)) \|_\| continue;
	997	thread.enqueue().* = .{
	998	.opcode = .MSG_RING,
	999	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	1000	.ioprio = 0,
	1001	.fd = idle_search_thread.io_uring.fd,
	1002	.off = @intFromEnum(Completion.Userdata.wakeup),
	1003	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
	1004	.len = 0,
	1005	.rw_flags = 0,
	1006	.user_data = @intFromEnum(Completion.Userdata.wakeup),
	1007	.buf_index = 0,
	1008	.personality = 0,
	1009	.splice_fd_in = 0,
	1010	.addr3 = 0,
	1011	.resv = 0,
	1012	};
	1013	return true;
	1014	}
	1015	spawn_thread: {
	1016	// previous failed reservations must have completed before retrying
	1017	if (new_thread_index == ev.threads.allocated.len or @cmpxchgWeak(
	1018	u32,
	1019	&ev.threads.reserved,
	1020	new_thread_index,
	1021	new_thread_index + 1,
	1022	.acquire,
	1023	.monotonic,
	1024	) != null) break :spawn_thread;
	1025	const new_thread = &ev.threads.allocated[new_thread_index];
	1026	const next_thread_index = new_thread_index + 1;
	1027	var params = std.mem.zeroInit(linux.io_uring_params, .{
	1028	.flags = linux.IORING_SETUP_ATTACH_WQ \|
	1029	linux.IORING_SETUP_R_DISABLED \|
	1030	linux.IORING_SETUP_COOP_TASKRUN \|
	1031	linux.IORING_SETUP_SINGLE_ISSUER,
	1032	.wq_fd = @as(u32, @intCast(ev.threads.allocated[0].io_uring.fd)),
	1033	});
	1034	new_thread.* = .{
	1035	.required_align = {},
	1036	.thread = undefined,
	1037	.idle_context = undefined,
	1038	.current_context = &new_thread.idle_context,
	1039	.ready_queue = ready_queue.head,
	1040	.free_queue = null,
	1041	.io_uring = IoUring.init_params(@as(u16, 1) << ev.log2_ring_entries, &params) catch \|err\| {
	1042	@atomicStore(u32, &ev.threads.reserved, new_thread_index, .release);
	1043	// no more access to `thread` after giving up reservation
	1044	log.warn("unable to create worker thread due to io_uring init failure: {s}", .{
	1045	@errorName(err),
	1046	});
	1047	break :spawn_thread;
	1048	},
	1049	.idle_search_index = 0,
	1050	.steal_ready_search_index = 0,
	1051	.steal_free_search_index = 0,
	1052	.name_arena = .{},
	1053	.csprng = .uninitialized,
	1054	};
	1055	new_thread.thread = std.Thread.spawn(.{
	1056	.stack_size = idle_stack_size,
	1057	.allocator = ev.allocator(),
	1058	}, threadEntry, .{ ev, new_thread_index }) catch \|err\| {
	1059	new_thread.io_uring.deinit();
	1060	@atomicStore(u32, &ev.threads.reserved, new_thread_index, .release);
	1061	// no more access to `thread` after giving up reservation
	1062	log.warn("unable to create worker thread due spawn failure: {s}", .{@errorName(err)});
	1063	break :spawn_thread;
	1064	};
	1065	// shared fields of `Thread` must be initialized before being marked active
	1066	@atomicStore(u32, &ev.threads.active, next_thread_index, .release);
	1067	return false;
	1068	}
	1069	// nobody wanted it, so just queue it on ourselves
	1070	while (true) ready_queue.tail.status.queue_next = @cmpxchgWeak(
	1071	?*Fiber,
	1072	&thread.ready_queue,
	1073	ready_queue.tail.status.queue_next,
	1074	ready_queue.head,
	1075	.acq_rel,
	1076	.acquire,
	1077	) orelse break;
	1078	return false;
	1079	}
	1080
	1081	fn threadEntry(ev: *Evented, index: u32) void {
	1082	const thread: *Thread = &ev.threads.allocated[index];
	1083	Thread.self = thread;
	1084	switch (linux.errno(linux.io_uring_register(thread.io_uring.fd, .REGISTER_ENABLE_RINGS, null, 0))) {
	1085	.SUCCESS => ev.idle(thread),
	1086	else => \|err\| @panic(@tagName(err)),
	1087	}
	1088	}
	1089
	1090	const Completion = struct {
	1091	result: i32,
	1092	flags: u32,
	1093
	1094	const Userdata = enum(usize) {
	1095	unused,
	1096	wakeup,
	1097	futex_wake,
	1098	close,
	1099	cleanup,
	1100	exit,
	1101	/// If bit 0 is 1, a pointer to the `context` field of `Io.Batch.Storage.Pending`.
	1102	/// If bits 0 and 1 are 0, a `*Fiber`.
	1103	_,
	1104	};
	1105
	1106	fn errno(completion: Completion) linux.E {
	1107	return linux.errno(@bitCast(@as(isize, completion.result)));
	1108	}
	1109	};
	1110
	1111	fn mainIdleEntry() callconv(.naked) void {
	1112	switch (builtin.cpu.arch) {
	1113	.aarch64 => asm volatile (
	1114	\\ mov x0, fp
	1115	\\ mov fp, #0
	1116	\\ b %[mainIdle]
	1117	:
	1118	: [mainIdle] "X" (&mainIdle),
	1119	),
	1120	.x86_64 => asm volatile (
	1121	\\ movq %%rbp, %%rdi
	1122	\\ xor %%ebp, %%ebp
	1123	\\ jmp %[mainIdle:P]
	1124	:
	1125	: [mainIdle] "X" (&mainIdle),
	1126	),
	1127	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
	1128	}
	1129	}
	1130
	1131	fn mainIdle(
	1132	ev: *Evented,
	1133	message: *const SwitchMessage,
	1134	) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Io.fiber.Context)))) noreturn {
	1135	message.handle(ev);
	1136	ev.idle(&ev.threads.allocated[0]);
	1137	ev.yield(@ptrCast(&ev.main_fiber_buffer), .nothing);
	1138	unreachable; // switched to dead fiber
	1139	}
	1140
	1141	fn idle(ev: Evented, thread: Thread) void {
	1142	var maybe_ready_fiber: ?*Fiber = null;
	1143	while (true) {
	1144	while (maybe_ready_fiber orelse ev.findReadyFiber(thread)) \|ready_fiber\| {
	1145	ev.yield(ready_fiber, .nothing);
	1146	maybe_ready_fiber = null;
	1147	}
	1148	_ = thread.io_uring.submit_and_wait(1) catch \|err\| switch (err) {
	1149	error.SignalInterrupt => {},
	1150	else => \|e\| @panic(@errorName(e)),
	1151	};
	1152	var maybe_ready_queue: ?Fiber.Queue = null;
	1153	while (true) {
	1154	var cqes_buffer: [1 << 8]linux.io_uring_cqe = undefined;
	1155	const cqes = cqes_buffer[0 .. thread.io_uring.copy_cqes(&cqes_buffer, 0) catch \|err\| switch (err) {
	1156	error.SignalInterrupt => 0,
	1157	else => \|e\| @panic(@errorName(e)),
	1158	}];
	1159	if (cqes.len == 0) break;
	1160	for (cqes) \|cqe\| if (cqe.flags & linux.IORING_CQE_F_SKIP == 0) switch (@as(
	1161	Completion.Userdata,
	1162	@enumFromInt(cqe.user_data),
	1163	)) {
	1164	.unused => unreachable, // bad submission queued?
	1165	.wakeup => {},
	1166	.futex_wake => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
	1167	.SUCCESS => recoverableOsBugDetected(), // success is skipped
	1168	.INVAL => {}, // invalid futex_wait() on ptr done elsewhere
	1169	.INTR, .CANCELED => recoverableOsBugDetected(), // `Completion.Userdata.futex_wake` is not cancelable
	1170	.FAULT => {}, // pointer became invalid while doing the wake
	1171	else => recoverableOsBugDetected(), // deadlock due to operating system bug
	1172	},
	1173	.close => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
	1174	.BADF => recoverableOsBugDetected(), // Always a race condition.
	1175	.INTR => {}, // This is still a success. See https://github.com/ziglang/zig/issues/2425
	1176	else => {},
	1177	},
	1178	.cleanup => @panic("failed to notify other threads that we are exiting"),
	1179	.exit => {
	1180	assert(maybe_ready_fiber == null and maybe_ready_queue == null); // pending async
	1181	return;
	1182	},
	1183	_ => if (@as(?*Fiber, ready_fiber: switch (@as(u2, @truncate(cqe.user_data))) {
	1184	0b00 => {
	1185	const ready_fiber: *Fiber = @ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
	1186	ready_fiber.resultPointer(Completion).* = .{
	1187	.result = cqe.res,
	1188	.flags = cqe.flags,
	1189	};
	1190	break :ready_fiber ready_fiber;
	1191	},
	1192	0b01 => {
	1193	thread.enqueue().* = .{
	1194	.opcode = .ASYNC_CANCEL,
	1195	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	1196	.ioprio = 0,
	1197	.fd = 0,
	1198	.off = 0,
	1199	.addr = cqe.user_data & ~@as(usize, 0b11),
	1200	.len = 0,
	1201	.rw_flags = 0,
	1202	.user_data = @intFromEnum(Completion.Userdata.wakeup),
	1203	.buf_index = 0,
	1204	.personality = 0,
	1205	.splice_fd_in = 0,
	1206	.addr3 = 0,
	1207	.resv = 0,
	1208	};
	1209	break :ready_fiber null;
	1210	},
	1211	0b10 => {
	1212	const batch_userdata: *Io.Operation.Storage.Pending.Userdata =
	1213	@ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
	1214	const batch: *Io.Batch = @ptrFromInt(batch_userdata[0]);
	1215	var next: usize = 0b00;
	1216	batch_userdata[0..3].* = .{ next, @as(u32, @bitCast(cqe.res)), cqe.flags };
	1217	while (true) {
	1218	next = @cmpxchgWeak(
	1219	usize,
	1220	@as(*usize, @ptrCast(&batch.userdata)),
	1221	next,
	1222	cqe.user_data,
	1223	.release,
	1224	.acquire,
	1225	) orelse break;
	1226	batch_userdata[0] = next;
	1227	}
	1228	break :ready_fiber switch (@as(u2, @truncate(next))) {
	1229	0b00, 0b01 => @ptrFromInt(next & ~@as(usize, 0b11)),
	1230	0b10, 0b11 => null,
	1231	};
	1232	},
	1233	0b11 => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
	1234	.SUCCESS => unreachable, // no event count specified
	1235	.TIME => {
	1236	const context: *usize = @ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
	1237	const fiber = @atomicRmw(usize, context, .Add, 0b01, .acquire);
	1238	break :ready_fiber switch (@as(u2, @truncate(fiber))) {
	1239	else => unreachable, // timeout completed multiple times
	1240	0b00 => @ptrFromInt(fiber & ~@as(usize, 0b11)),
	1241	0b10 => null,
	1242	};
	1243	},
	1244	.CANCELED => null, // user data may have been invalidated
	1245	else => \|err\| unexpectedErrno(err) catch null,
	1246	},
	1247	})) \|ready_fiber\| {
	1248	assert(ready_fiber.status.queue_next == null);
	1249	if (maybe_ready_fiber == null) {
	1250	maybe_ready_fiber = ready_fiber;
	1251	} else if (maybe_ready_queue) \|*ready_queue\| {
	1252	ready_queue.tail.status.queue_next = ready_fiber;
	1253	ready_queue.tail = ready_fiber;
	1254	} else maybe_ready_queue = .{ .head = ready_fiber, .tail = ready_fiber };
	1255	},
	1256	};
	1257	}
	1258	if (maybe_ready_queue) \|ready_queue\| _ = ev.schedule(thread, ready_queue);
	1259	}
	1260	}
	1261
	1262	const SwitchMessage = struct {
	1263	contexts: Io.fiber.Switch,
	1264	pending_task: PendingTask,
	1265
	1266	const PendingTask = union(enum) {
	1267	nothing,
	1268	reschedule,
	1269	await: u31,
	1270	group_await: Group,
	1271	group_cancel: Group,
	1272	batch_await: *Io.Batch,
	1273	destroy,
	1274	exit,
	1275	};
	1276
	1277	fn handle(message: const SwitchMessage, ev: Evented) void {
	1278	const thread: *Thread = .current();
	1279	thread.current_context = message.contexts.new;
	1280	if (tracy.enable) {
	1281	if (message.contexts.new != &thread.idle_context) {
	1282	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.new));
	1283	tracy.fiberEnter(fiber.name);
	1284	} else tracy.fiberLeave();
	1285	}
	1286	switch (message.pending_task) {
	1287	.nothing => {},
	1288	.reschedule => if (message.contexts.old != &thread.idle_context) {
	1289	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
	1290	assert(fiber.status.queue_next == null);
	1291	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
	1292	},
	1293	.await => \|count\| {
	1294	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
	1295	if (@atomicRmw(i32, &fiber.await_count, .Sub, count, .monotonic) > 0)
	1296	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
	1297	},
	1298	.group_await => \|group\| {
	1299	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
	1300	if (group.await(ev, fiber))
	1301	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
	1302	},
	1303	.group_cancel => \|group\| {
	1304	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
	1305	if (group.cancel(ev, fiber))
	1306	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
	1307	},
	1308	.batch_await => \|batch\| {
	1309	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
	1310	if (@cmpxchgStrong(
	1311	?*anyopaque,
	1312	&batch.userdata,
	1313	null,
	1314	fiber,
	1315	.release,
	1316	.monotonic,
	1317	)) \|head\| {
	1318	assert(@as(u2, @truncate(@intFromPtr(head))) != 0b00);
	1319	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
	1320	}
	1321	},
	1322	.destroy => {
	1323	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
	1324	fiber.destroy();
	1325	},
	1326	.exit => for (
	1327	ev.threads.allocated[0..@atomicLoad(u32, &ev.threads.active, .acquire)],
	1328	) \|*each_thread\| {
	1329	thread.enqueue().* = .{
	1330	.opcode = .MSG_RING,
	1331	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	1332	.ioprio = 0,
	1333	.fd = each_thread.io_uring.fd,
	1334	.off = @intFromEnum(Completion.Userdata.exit),
	1335	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
	1336	.len = 0,
	1337	.rw_flags = 0,
	1338	.user_data = @intFromEnum(Completion.Userdata.cleanup),
	1339	.buf_index = 0,
	1340	.personality = 0,
	1341	.splice_fd_in = 0,
	1342	.addr3 = 0,
	1343	.resv = 0,
	1344	};
	1345	},
	1346	}
	1347	}
	1348	};
	1349
	1350	inline fn contextSwitch(message: const SwitchMessage) const SwitchMessage {
	1351	return @fieldParentPtr("contexts", Io.fiber.contextSwitch(&message.contexts));
	1352	}
	1353
	1354	fn crashHandler(userdata: ?*anyopaque) void {
	1355	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1356	_ = ev;
	1357	const thread = Thread.self orelse std.process.abort();
	1358	if (thread.current_context == &thread.idle_context) std.process.abort();
	1359	const fiber = thread.currentFiber();
	1360	@atomicStore(
	1361	Fiber.CancelStatus,
	1362	&fiber.cancel_status,
	1363	.{ .requested = true, .awaiting = .nothing },
	1364	.monotonic,
	1365	);
	1366	fiber.cancel_protection = .{ .user = .blocked, .acknowledged = true };
	1367	}
	1368
	1369	const AsyncClosure = struct {
	1370	ev: *Evented,
	1371	fiber: *Fiber,
	1372	start: const fn (context: const anyopaque, result: *anyopaque) void,
	1373	result_align: Alignment,
	1374
	1375	fn fromFiber(fiber: Fiber) AsyncClosure {
	1376	return @ptrFromInt(Fiber.max_context_align.max(.of(AsyncClosure)).backward(
	1377	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
	1378	) - @sizeOf(AsyncClosure));
	1379	}
	1380
	1381	fn contextPointer(closure: AsyncClosure) []align(Fiber.max_context_align.toByteUnits()) u8 {
	1382	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(AsyncClosure));
	1383	}
	1384
	1385	fn entry() callconv(.naked) void {
	1386	switch (builtin.cpu.arch) {
	1387	.aarch64 => asm volatile (
	1388	\\ mov x0, sp
	1389	\\ b %[call]
	1390	:
	1391	: [call] "X" (&call),
	1392	),
	1393	.x86_64 => asm volatile (
	1394	\\ leaq 8(%%rsp), %%rdi
	1395	\\ jmp %[call:P]
	1396	:
	1397	: [call] "X" (&call),
	1398	),
	1399	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
	1400	}
	1401	}
	1402
	1403	fn call(
	1404	closure: *AsyncClosure,
	1405	message: *const SwitchMessage,
	1406	) callconv(.withStackAlign(.c, @alignOf(AsyncClosure))) noreturn {
	1407	message.handle(closure.ev);
	1408	const fiber = closure.fiber;
	1409	closure.start(closure.contextPointer(), fiber.resultBytes(closure.result_align));
	1410	closure.ev.yield(
	1411	if (@atomicRmw(?*Fiber, &fiber.link.awaiter, .Xchg, Fiber.finished, .acq_rel)) \|awaiter\|
	1412	if (@atomicRmw(i32, &awaiter.await_count, .Add, 1, .monotonic) == -1) awaiter else null
	1413	else
	1414	null,
	1415	.nothing,
	1416	);
	1417	unreachable; // switched to dead fiber
	1418	}
	1419	};
	1420
	1421	fn async(
	1422	userdata: ?*anyopaque,
	1423	result: []u8,
	1424	result_alignment: Alignment,
	1425	context: []const u8,
	1426	context_alignment: Alignment,
	1427	start: const fn (context: const anyopaque, result: *anyopaque) void,
	1428	) ?*std.Io.AnyFuture {
	1429	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1430	return concurrent(ev, result.len, result_alignment, context, context_alignment, start) catch {
	1431	start(context.ptr, result.ptr);
	1432	return null;
	1433	};
	1434	}
	1435
	1436	fn concurrent(
	1437	userdata: ?*anyopaque,
	1438	result_len: usize,
	1439	result_alignment: Alignment,
	1440	context: []const u8,
	1441	context_alignment: Alignment,
	1442	start: const fn (context: const anyopaque, result: *anyopaque) void,
	1443	) Io.ConcurrentError!*std.Io.AnyFuture {
	1444	assert(result_alignment.compare(.lte, Fiber.max_result_align)); // TODO
	1445	assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
	1446	assert(result_len <= Fiber.max_result_size); // TODO
	1447	assert(context.len <= Fiber.max_context_size); // TODO
	1448
	1449	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1450	const fiber = Fiber.create(ev) catch \|err\| switch (err) {
	1451	error.OutOfMemory => return error.ConcurrencyUnavailable,
	1452	};
	1453
	1454	const closure: *AsyncClosure = .fromFiber(fiber);
	1455	fiber.* = .{
	1456	.required_align = {},
	1457	.context = switch (builtin.cpu.arch) {
	1458	.aarch64 => .{
	1459	.sp = @intFromPtr(closure),
	1460	.fp = 0,
	1461	.pc = @intFromPtr(&AsyncClosure.entry),
	1462	},
	1463	.x86_64 => .{
	1464	.rsp = @intFromPtr(closure) - 8,
	1465	.rbp = 0,
	1466	.rip = @intFromPtr(&AsyncClosure.entry),
	1467	},
	1468	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
	1469	},
	1470	.await_count = 0,
	1471	.link = .{ .awaiter = null },
	1472	.status = .{ .queue_next = null },
	1473	.cancel_status = .unrequested,
	1474	.cancel_protection = .unblocked,
	1475	.name = if (tracy.enable) name: {
	1476	const thread: *Thread = .current();
	1477	var name_arena = thread.name_arena.promote(std.heap.page_allocator);
	1478	defer thread.name_arena = name_arena.state;
	1479	break :name std.fmt.allocPrintSentinel(
	1480	name_arena.allocator(),
	1481	"task {d}",
	1482	.{@atomicRmw(u64, &Fiber.next_name, .Add, 1, .monotonic)},
	1483	0,
	1484	) catch return error.ConcurrencyUnavailable;
	1485	},
	1486	};
	1487	closure.* = .{
	1488	.ev = ev,
	1489	.fiber = fiber,
	1490	.start = start,
	1491	.result_align = result_alignment,
	1492	};
	1493	@memcpy(closure.contextPointer(), context);
	1494
	1495	const thread: *Thread = .current();
	1496	if (ev.schedule(thread, .{ .head = fiber, .tail = fiber })) thread.submit();
	1497	return @ptrCast(fiber);
	1498	}
	1499
	1500	fn await(
	1501	userdata: ?*anyopaque,
	1502	future: *std.Io.AnyFuture,
	1503	result: []u8,
	1504	result_alignment: Alignment,
	1505	) void {
	1506	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1507	const fiber = Thread.current().currentFiber();
	1508	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
	1509	if (@atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, fiber, .acq_rel)) \|awaiter\| {
	1510	assert(awaiter == Fiber.finished);
	1511	} else while (true) {
	1512	ev.yield(null, .{ .await = 1 });
	1513	const awaiter = @atomicLoad(?*Fiber, &future_fiber.link.awaiter, .acquire);
	1514	if (awaiter == Fiber.finished) break;
	1515	assert(awaiter == fiber); // spurious wakeup
	1516	}
	1517	@memcpy(result, future_fiber.resultBytes(result_alignment));
	1518	future_fiber.destroy();
	1519	}
	1520
	1521	fn cancel(
	1522	userdata: ?*anyopaque,
	1523	future: *std.Io.AnyFuture,
	1524	result: []u8,
	1525	result_alignment: Alignment,
	1526	) void {
	1527	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1528	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
	1529	future_fiber.requestCancel(ev);
	1530	await(ev, future, result, result_alignment);
	1531	}
	1532
	1533	const Group = struct {
	1534	ptr: *Io.Group,
	1535
	1536	const List = packed struct(usize) {
	1537	cancel_requested: bool,
	1538	awaiter_delayed: bool,
	1539	fibers: Fiber.PackedPtr,
	1540	};
	1541	fn listPtr(group: Group) *List {
	1542	return @ptrCast(&group.ptr.token);
	1543	}
	1544
	1545	const Mutex = packed struct(u32) {
	1546	locked: bool,
	1547	contended: bool,
	1548	shared2: u30,
	1549	};
	1550	fn mutexPtr(group: Group) *Mutex {
	1551	return switch (comptime builtin.cpu.arch.endian()) {
	1552	.little => @ptrCast(&group.ptr.state),
	1553	.big => @ptrCast(@alignCast(
	1554	@as([*]u8, @ptrCast(&group.ptr.state)) + @sizeOf(usize) - @sizeOf(u32),
	1555	)),
	1556	};
	1557	}
	1558
	1559	const Awaiter = packed struct(usize) {
	1560	locked: bool,
	1561	contended: bool,
	1562	awaiter: Fiber.PackedPtr,
	1563	};
	1564	fn awaiterPtr(group: Group) *Awaiter {
	1565	return @ptrCast(&group.ptr.state);
	1566	}
	1567
	1568	fn lock(group: Group, ev: *Evented) void {
	1569	const mutex = group.mutexPtr();
	1570	{
	1571	const old_state = @atomicRmw(
	1572	Mutex,
	1573	mutex,
	1574	.Or,
	1575	.{ .locked = true, .contended = false, .shared2 = 0 },
	1576	.acquire,
	1577	);
	1578	if (!old_state.locked) {
	1579	@branchHint(.likely);
	1580	return;
	1581	}
	1582	if (old_state.contended) {
	1583	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
	1584	}
	1585	}
	1586	while (true) {
	1587	var old_state = @atomicRmw(
	1588	Mutex,
	1589	mutex,
	1590	.Or,
	1591	.{ .locked = true, .contended = true, .shared2 = 0 },
	1592	.acquire,
	1593	);
	1594	if (!old_state.locked) {
	1595	@branchHint(.likely);
	1596	return;
	1597	}
	1598	old_state.contended = true;
	1599	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
	1600	}
	1601	}
	1602
	1603	fn unlock(group: Group, ev: *Evented) void {
	1604	const mutex = group.mutexPtr();
	1605	const old_state = @atomicRmw(
	1606	Mutex,
	1607	mutex,
	1608	.And,
	1609	.{ .locked = false, .contended = false, .shared2 = std.math.maxInt(u30) },
	1610	.release,
	1611	);
	1612	assert(old_state.locked);
	1613	if (old_state.contended) futexWake(ev, @ptrCast(mutex), 1);
	1614	}
	1615
	1616	fn addFiber(group: Group, ev: Evented, fiber: Fiber) void {
	1617	group.lock(ev);
	1618	defer group.unlock(ev);
	1619	const list_ptr = group.listPtr();
	1620	const list = @atomicLoad(List, list_ptr, .monotonic);
	1621	if (list.cancel_requested) fiber.cancel_status = .{ .requested = true, .awaiting = .nothing };
	1622	const old_head = list.fibers.unpack();
	1623	if (old_head) \|head\| head.link.group.prev = fiber;
	1624	fiber.link.group.next = old_head;
	1625	@atomicStore(List, list_ptr, .{
	1626	.cancel_requested = list.cancel_requested,
	1627	.awaiter_delayed = list.awaiter_delayed,
	1628	.fibers = .pack(fiber),
	1629	}, .monotonic);
	1630	}
	1631
	1632	fn removeFiber(group: Group, ev: Evented, fiber: Fiber) ?*Fiber {
	1633	group.lock(ev);
	1634	defer group.unlock(ev);
	1635	const list_ptr = group.listPtr();
	1636	const list = @atomicLoad(List, list_ptr, .monotonic);
	1637	if (fiber.link.group.next) \|next\| next.link.group.prev = fiber.link.group.prev;
	1638	if (fiber.link.group.prev) \|prev\| {
	1639	prev.link.group.next = fiber.link.group.next;
	1640	} else if (fiber.link.group.next) \|new_head\| {
	1641	@atomicStore(List, list_ptr, .{
	1642	.cancel_requested = list.cancel_requested,
	1643	.awaiter_delayed = list.awaiter_delayed,
	1644	.fibers = .pack(new_head),
	1645	}, .monotonic);
	1646	} else if (@atomicLoad(Awaiter, group.awaiterPtr(), .monotonic).awaiter.unpack()) \|awaiter\| {
	1647	if (!awaiter.cancel_status.changeAwaiting(.group, .nothing) or list.cancel_requested) {
	1648	@atomicStore(List, list_ptr, .{
	1649	.cancel_requested = false,
	1650	.awaiter_delayed = false,
	1651	.fibers = .null,
	1652	}, .release);
	1653	assert(awaiter.status.awaiting_group.ptr == group.ptr);
	1654	awaiter.status = .{ .queue_next = null };
	1655	return awaiter;
	1656	}
	1657	// Race with `Fiber.requestCancel`
	1658	@atomicStore(List, list_ptr, .{
	1659	.cancel_requested = false,
	1660	.awaiter_delayed = true,
	1661	.fibers = .null,
	1662	}, .monotonic);
	1663	} else @atomicStore(List, list_ptr, .{
	1664	.cancel_requested = false,
	1665	.awaiter_delayed = false,
	1666	.fibers = .null,
	1667	}, .release);
	1668	return null;
	1669	}
	1670
	1671	fn await(group: Group, ev: Evented, awaiter: Fiber) bool {
	1672	group.lock(ev);
	1673	defer group.unlock(ev);
	1674	if (@atomicLoad(List, group.listPtr(), .monotonic).fibers.unpack()) \|_\| {
	1675	if (group.registerAwaiter(awaiter) and awaiter.cancel_protection.check() == .unblocked) {
	1676	// The awaiter already had an unacknowledged cancelation request before
	1677	// attempting to await a group, so propagate the cancelation to the group.
	1678	assert(!group.cancelLocked(ev, null));
	1679	}
	1680	return false;
	1681	}
	1682	return true;
	1683	}
	1684
	1685	fn cancel(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
	1686	group.lock(ev);
	1687	defer group.unlock(ev);
	1688	return group.cancelLocked(ev, maybe_awaiter);
	1689	}
	1690
	1691	/// Assumes the mutex is held.
	1692	fn cancelLocked(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
	1693	const list_ptr = group.listPtr();
	1694	const list = @atomicRmw(
	1695	List,
	1696	list_ptr,
	1697	.Add,
	1698	.{ .cancel_requested = true, .awaiter_delayed = false, .fibers = .null },
	1699	.monotonic,
	1700	);
	1701	assert(!list.cancel_requested);
	1702	if (list.fibers.unpack()) \|head\| {
	1703	var maybe_fiber: ?*Fiber = head;
	1704	while (maybe_fiber) \|fiber\| {
	1705	fiber.requestCancel(ev);
	1706	maybe_fiber = fiber.link.group.next;
	1707	}
	1708	if (maybe_awaiter) \|awaiter\| _ = group.registerAwaiter(awaiter);
	1709	return false;
	1710	}
	1711	@atomicStore(
	1712	List,
	1713	list_ptr,
	1714	.{ .cancel_requested = false, .awaiter_delayed = false, .fibers = .null },
	1715	.release,
	1716	);
	1717	return if (maybe_awaiter) \|_\| true else list.awaiter_delayed;
	1718	}
	1719
	1720	/// Assumes the mutex is held.
	1721	fn registerAwaiter(group: Group, awaiter: *Fiber) bool {
	1722	assert(awaiter.status.queue_next == null);
	1723	awaiter.status = .{ .awaiting_group = group };
	1724	assert(@atomicRmw(
	1725	Awaiter,
	1726	group.awaiterPtr(),
	1727	.Add,
	1728	.{ .locked = false, .contended = false, .awaiter = .pack(awaiter) },
	1729	.monotonic,
	1730	).awaiter == .null);
	1731	return awaiter.cancel_status.changeAwaiting(.nothing, .group);
	1732	}
	1733
	1734	const AsyncClosure = struct {
	1735	ev: *Evented,
	1736	group: Group,
	1737	fiber: *Fiber,
	1738	start: const fn (context: const anyopaque) Io.Cancelable!void,
	1739
	1740	fn fromFiber(fiber: Fiber) Group.AsyncClosure {
	1741	return @ptrFromInt(Fiber.max_context_align.max(.of(Group.AsyncClosure)).backward(
	1742	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
	1743	) - @sizeOf(Group.AsyncClosure));
	1744	}
	1745
	1746	fn contextPointer(
	1747	closure: *Group.AsyncClosure,
	1748	) [*]align(Fiber.max_context_align.toByteUnits()) u8 {
	1749	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(Group.AsyncClosure));
	1750	}
	1751
	1752	fn entry() callconv(.naked) void {
	1753	switch (builtin.cpu.arch) {
	1754	.aarch64 => asm volatile (
	1755	\\ mov x0, sp
	1756	\\ b %[call]
	1757	:
	1758	: [call] "X" (&call),
	1759	),
	1760	.x86_64 => asm volatile (
	1761	\\ leaq 8(%%rsp), %%rdi
	1762	\\ jmp %[call:P]
	1763	:
	1764	: [call] "X" (&call),
	1765	),
	1766	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
	1767	}
	1768	}
	1769
	1770	fn call(
	1771	closure: *Group.AsyncClosure,
	1772	message: *const SwitchMessage,
	1773	) callconv(.withStackAlign(.c, @alignOf(Group.AsyncClosure))) noreturn {
	1774	message.handle(closure.ev);
	1775	assert(closure.fiber.status.queue_next == null);
	1776	const result = closure.start(closure.contextPointer());
	1777	const ev = closure.ev;
	1778	const group = closure.group;
	1779	const fiber = closure.fiber;
	1780	const cancel_acknowledged = fiber.cancel_protection.acknowledged;
	1781	if (result) {
	1782	assert(!cancel_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
	1783	} else \|err\| switch (err) {
	1784	error.Canceled => assert(cancel_acknowledged), // group task returned `error.Canceled` but was never canceled
	1785	}
	1786	ev.yield(group.removeFiber(ev, fiber), .destroy);
	1787	unreachable; // switched to dead fiber
	1788	}
	1789	};
	1790	};
	1791
	1792	fn groupAsync(
	1793	userdata: ?*anyopaque,
	1794	type_erased: *Io.Group,
	1795	context: []const u8,
	1796	context_alignment: Alignment,
	1797	start: const fn (context: const anyopaque) Io.Cancelable!void,
	1798	) void {
	1799	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1800	return groupConcurrent(ev, type_erased, context, context_alignment, start) catch {
	1801	const fiber = Thread.current().currentFiber();
	1802	const pre_acknowledged = fiber.cancel_protection.acknowledged;
	1803	const result = start(context.ptr);
	1804	const post_acknowledged = fiber.cancel_protection.acknowledged;
	1805	if (result) {
	1806	if (pre_acknowledged) {
	1807	assert(post_acknowledged); // group task called `recancel` but was not canceled
	1808	} else {
	1809	assert(!post_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
	1810	}
	1811	} else \|err\| switch (err) {
	1812	// Don't swallow the cancelation: make it visible to the `Group.async` caller.
	1813	error.Canceled => {
	1814	assert(!pre_acknowledged); // group task called `recancel` but was not canceled
	1815	assert(post_acknowledged); // group task returned `error.Canceled` but was never canceled
	1816	fiber.cancel_protection.recancel();
	1817	},
	1818	}
	1819	};
	1820	}
	1821
	1822	fn groupConcurrent(
	1823	userdata: ?*anyopaque,
	1824	type_erased: *Io.Group,
	1825	context: []const u8,
	1826	context_alignment: Alignment,
	1827	start: const fn (context: const anyopaque) Io.Cancelable!void,
	1828	) Io.ConcurrentError!void {
	1829	assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
	1830	assert(context.len <= Fiber.max_context_size); // TODO
	1831
	1832	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1833	const group: Group = .{ .ptr = type_erased };
	1834	const fiber = Fiber.create(ev) catch \|err\| switch (err) {
	1835	error.OutOfMemory => return error.ConcurrencyUnavailable,
	1836	};
	1837
	1838	const closure: *Group.AsyncClosure = .fromFiber(fiber);
	1839	fiber.* = .{
	1840	.required_align = {},
	1841	.context = switch (builtin.cpu.arch) {
	1842	.aarch64 => .{
	1843	.sp = @intFromPtr(closure),
	1844	.fp = 0,
	1845	.pc = @intFromPtr(&Group.AsyncClosure.entry),
	1846	},
	1847	.x86_64 => .{
	1848	.rsp = @intFromPtr(closure) - 8,
	1849	.rbp = 0,
	1850	.rip = @intFromPtr(&Group.AsyncClosure.entry),
	1851	},
	1852	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
	1853	},
	1854	.await_count = 0,
	1855	.link = .{ .group = .{ .prev = null, .next = null } },
	1856	.status = .{ .queue_next = null },
	1857	.cancel_status = .unrequested,
	1858	.cancel_protection = .unblocked,
	1859	.name = if (tracy.enable) name: {
	1860	const thread: *Thread = .current();
	1861	var name_arena = thread.name_arena.promote(std.heap.page_allocator);
	1862	defer thread.name_arena = name_arena.state;
	1863	break :name std.fmt.allocPrintSentinel(
	1864	name_arena.allocator(),
	1865	"group task {d}",
	1866	.{@atomicRmw(u64, &Fiber.next_name, .Add, 1, .monotonic)},
	1867	0,
	1868	) catch return error.ConcurrencyUnavailable;
	1869	},
	1870	};
	1871	closure.* = .{
	1872	.ev = ev,
	1873	.group = group,
	1874	.fiber = fiber,
	1875	.start = start,
	1876	};
	1877	@memcpy(closure.contextPointer(), context);
	1878	group.addFiber(ev, fiber);
	1879	const thread: *Thread = .current();
	1880	if (ev.schedule(thread, .{ .head = fiber, .tail = fiber })) thread.submit();
	1881	}
	1882
	1883	fn groupAwait(
	1884	userdata: ?*anyopaque,
	1885	type_erased: *Io.Group,
	1886	initial_token: *anyopaque,
	1887	) Io.Cancelable!void {
	1888	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1889	_ = initial_token;
	1890	ev.yield(null, .{ .group_await = .{ .ptr = type_erased } });
	1891	}
	1892
	1893	fn groupCancel(userdata: ?anyopaque, type_erased: Io.Group, initial_token: *anyopaque) void {
	1894	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1895	_ = initial_token;
	1896	ev.yield(null, .{ .group_cancel = .{ .ptr = type_erased } });
	1897	}
	1898
	1899	fn recancel(userdata: ?*anyopaque) void {
	1900	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1901	_ = ev;
	1902	Thread.current().currentFiber().cancel_protection.recancel();
	1903	}
	1904
	1905	fn swapCancelProtection(userdata: ?*anyopaque, new: Io.CancelProtection) Io.CancelProtection {
	1906	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1907	_ = ev;
	1908	const cancel_protection = &Thread.current().currentFiber().cancel_protection;
	1909	defer cancel_protection.user = new;
	1910	return cancel_protection.user;
	1911	}
	1912
	1913	fn checkCancel(userdata: ?*anyopaque) Io.Cancelable!void {
	1914	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1915	_ = ev;
	1916	const fiber = Thread.current().currentFiber();
	1917	switch (fiber.cancel_protection.check()) {
	1918	.unblocked => {
	1919	const cancel_status = @atomicLoad(Fiber.CancelStatus, &fiber.cancel_status, .monotonic);
	1920	assert(cancel_status.awaiting == .nothing);
	1921	if (cancel_status.requested) {
	1922	@branchHint(.unlikely);
	1923	fiber.cancel_protection.acknowledge();
	1924	return error.Canceled;
	1925	}
	1926	},
	1927	.blocked => {},
	1928	}
	1929	}
	1930
	1931	fn select(userdata: ?anyopaque, futures: []const Io.AnyFuture) Io.Cancelable!usize {
	1932	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1933	var cancel_region: CancelRegion = .init();
	1934	defer cancel_region.deinit();
	1935	var await_count: u31, var result = for (futures, 0..) \|future, future_index\| {
	1936	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
	1937	if (@atomicRmw(
	1938	?*Fiber,
	1939	&future_fiber.link.awaiter,
	1940	.Xchg,
	1941	cancel_region.fiber,
	1942	.acq_rel,
	1943	)) \|awaiter\| {
	1944	assert(awaiter == Fiber.finished);
	1945	break .{ @intCast(future_index), future_index };
	1946	}
	1947	} else result: {
	1948	const await_count: u31 = @intCast(futures.len);
	1949	cancel_region.await(.select) catch \|err\| switch (err) {
	1950	error.Canceled => \|e\| break :result .{ await_count + 1, e },
	1951	};
	1952	ev.yield(null, .{ .await = 1 });
	1953	cancel_region.await(.nothing) catch \|err\| switch (err) {
	1954	error.Canceled => \|e\| break :result .{ await_count, e },
	1955	};
	1956	break :result .{ await_count - 1, futures.len };
	1957	};
	1958	for (futures[0 .. result catch futures.len], 0..) \|future, future_index\| {
	1959	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
	1960	const awaiter = @atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, null, .monotonic);
	1961	if (awaiter == Fiber.finished) {
	1962	@atomicStore(?*Fiber, &future_fiber.link.awaiter, Fiber.finished, .monotonic);
	1963	result = if (result) \|finished_index\| @min(future_index, finished_index) else \|e\| e;
	1964	} else {
	1965	assert(awaiter == cancel_region.fiber);
	1966	await_count -= 1;
	1967	}
	1968	}
	1969	// Equivalent to `ev.yield(null, .{ .await = await_count });`,
	1970	// but avoiding a context switch in the common case.
	1971	switch (std.math.order(
	1972	@atomicRmw(i32, &cancel_region.fiber.await_count, .Sub, await_count, .monotonic),
	1973	await_count,
	1974	)) {
	1975	.lt => ev.yield(null, .{ .await = 0 }),
	1976	.eq => {},
	1977	.gt => unreachable,
	1978	}
	1979	return result;
	1980	}
	1981
	1982	fn futexWait(
	1983	userdata: ?*anyopaque,
	1984	ptr: *const u32,
	1985	expected: u32,
	1986	timeout: Io.Timeout,
	1987	) Io.Cancelable!void {
	1988	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1989	const timespec: ?linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = timespec: switch (timeout) {
	1990	.none => .{
	1991	null,
	1992	.awake,
	1993	linux.IORING_TIMEOUT_ABS,
	1994	},
	1995	.duration => \|duration\| {
	1996	const ns = duration.raw.toNanoseconds();
	1997	break :timespec .{
	1998	.{
	1999	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	2000	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	2001	},
	2002	duration.clock,
	2003	0,
	2004	};
	2005	},
	2006	.deadline => \|deadline\| {
	2007	const ns = deadline.raw.toNanoseconds();
	2008	break :timespec .{
	2009	.{
	2010	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	2011	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	2012	},
	2013	deadline.clock,
	2014	linux.IORING_TIMEOUT_ABS,
	2015	};
	2016	},
	2017	};
	2018	var cancel_region: CancelRegion = .init();
	2019	defer cancel_region.deinit();
	2020	const thread = try cancel_region.awaitIoUring();
	2021	thread.enqueue().* = .{
	2022	.opcode = .FUTEX_WAIT,
	2023	.flags = if (timespec) \|_\| linux.IOSQE_IO_LINK else 0,
	2024	.ioprio = 0,
	2025	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
	2026	.off = expected,
	2027	.addr = @intFromPtr(ptr),
	2028	.len = 0,
	2029	.rw_flags = 0,
	2030	.user_data = @intFromPtr(cancel_region.fiber),
	2031	.buf_index = 0,
	2032	.personality = 0,
	2033	.splice_fd_in = 0,
	2034	.addr3 = std.math.maxInt(u32),
	2035	.resv = 0,
	2036	};
	2037	if (timespec) \|timespec_ptr\| thread.enqueue(). = .{
	2038	.opcode = .LINK_TIMEOUT,
	2039	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	2040	.ioprio = 0,
	2041	.fd = 0,
	2042	.off = 0,
	2043	.addr = @intFromPtr(timespec_ptr),
	2044	.len = 1,
	2045	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
	2046	.real => linux.IORING_TIMEOUT_REALTIME,
	2047	else => 0,
	2048	.boot => linux.IORING_TIMEOUT_BOOTTIME,
	2049	}),
	2050	.user_data = @intFromEnum(Completion.Userdata.wakeup),
	2051	.buf_index = 0,
	2052	.personality = 0,
	2053	.splice_fd_in = 0,
	2054	.addr3 = 0,
	2055	.resv = 0,
	2056	};
	2057	ev.yield(null, .nothing);
	2058	switch (cancel_region.errno()) {
	2059	.SUCCESS => {}, // notified by `wake()`
	2060	.INTR, .CANCELED => {}, // caller's responsibility to retry
	2061	.AGAIN => {}, // ptr.* != expect
	2062	.INVAL => {}, // possibly timeout overflow
	2063	.TIMEDOUT => unreachable,
	2064	.FAULT => recoverableOsBugDetected(), // ptr was invalid
	2065	else => recoverableOsBugDetected(),
	2066	}
	2067	}
	2068
	2069	fn futexWaitUncancelable(userdata: ?anyopaque, ptr: const u32, expected: u32) void {
	2070	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2071	var cancel_region: CancelRegion = .initBlocked();
	2072	defer cancel_region.deinit();
	2073	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
	2074	error.Canceled => unreachable, // blocked
	2075	};
	2076	thread.enqueue().* = .{
	2077	.opcode = .FUTEX_WAIT,
	2078	.flags = 0,
	2079	.ioprio = 0,
	2080	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
	2081	.off = expected,
	2082	.addr = @intFromPtr(ptr),
	2083	.len = 0,
	2084	.rw_flags = 0,
	2085	.user_data = @intFromPtr(cancel_region.fiber),
	2086	.buf_index = 0,
	2087	.personality = 0,
	2088	.splice_fd_in = 0,
	2089	.addr3 = std.math.maxInt(u32),
	2090	.resv = 0,
	2091	};
	2092	ev.yield(null, .nothing);
	2093	switch (cancel_region.errno()) {
	2094	.SUCCESS => {}, // notified by `wake()`
	2095	.INTR, .CANCELED => {}, // caller's responsibility to retry
	2096	.AGAIN => {}, // ptr.* != expect
	2097	.INVAL => {}, // possibly timeout overflow
	2098	.FAULT => recoverableOsBugDetected(), // ptr was invalid
	2099	else => recoverableOsBugDetected(),
	2100	}
	2101	}
	2102
	2103	fn futexWake(userdata: ?anyopaque, ptr: const u32, max_waiters: u32) void {
	2104	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2105	_ = ev;
	2106	const thread: *Thread = .current();
	2107	thread.enqueue().* = .{
	2108	.opcode = .FUTEX_WAKE,
	2109	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	2110	.ioprio = 0,
	2111	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
	2112	.off = max_waiters,
	2113	.addr = @intFromPtr(ptr),
	2114	.len = 0,
	2115	.rw_flags = 0,
	2116	.user_data = @intFromEnum(Completion.Userdata.futex_wake),
	2117	.buf_index = 0,
	2118	.personality = 0,
	2119	.splice_fd_in = 0,
	2120	.addr3 = std.math.maxInt(u32),
	2121	.resv = 0,
	2122	};
	2123	thread.submit();
	2124	}
	2125
	2126	fn operate(userdata: ?*anyopaque, operation: Io.Operation) Io.Cancelable!Io.Operation.Result {
	2127	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2128	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
	2129	defer maybe_sync.deinit(ev);
	2130	return switch (operation) {
	2131	.file_read_streaming => \|o\| .{
	2132	.file_read_streaming = ev.fileReadStreaming(
	2133	&maybe_sync.cancel_region,
	2134	o.file,
	2135	o.data,
	2136	) catch \|err\| switch (err) {
	2137	error.Canceled => \|e\| return e,
	2138	else => \|e\| e,
	2139	},
	2140	},
	2141	.file_write_streaming => \|o\| .{
	2142	.file_write_streaming = ev.fileWriteStreaming(
	2143	&maybe_sync.cancel_region,
	2144	o.file,
	2145	o.header,
	2146	o.data,
	2147	o.splat,
	2148	) catch \|err\| switch (err) {
	2149	error.Canceled => \|e\| return e,
	2150	else => \|e\| e,
	2151	},
	2152	},
	2153	.device_io_control => \|o\| .{
	2154	.device_io_control = try ev.deviceIoControl(try maybe_sync.enterSync(ev), o),
	2155	},
	2156	};
	2157	}
	2158
	2159	fn fileReadStreaming(
	2160	ev: *Evented,
	2161	cancel_region: *CancelRegion,
	2162	file: File,
	2163	data: []const []u8,
	2164	) File.ReadStreamingError!usize {
	2165	var iovecs_buffer: [max_iovecs_len]iovec = undefined;
	2166	var i: usize = 0;
	2167	for (data) \|buf\| {
	2168	if (iovecs_buffer.len - i == 0) break;
	2169	if (buf.len > 0) {
	2170	iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len };
	2171	i += 1;
	2172	}
	2173	}
	2174	const dest = iovecs_buffer[0..i];
	2175	assert(dest[0].len > 0);
	2176
	2177	const n = try ev.preadv(cancel_region, file.handle, dest, null);
	2178	return if (n == 0) error.EndOfStream else n;
	2179	}
	2180
	2181	fn fileWriteStreaming(
	2182	ev: *Evented,
	2183	cancel_region: *CancelRegion,
	2184	file: File,
	2185	header: []const u8,
	2186	data: []const []const u8,
	2187	splat: usize,
	2188	) File.Writer.Error!usize {
	2189	var iovecs: [max_iovecs_len]iovec_const = undefined;
	2190	var iovlen: iovlen_t = 0;
	2191	addBuf(&iovecs, &iovlen, header);
	2192	for (data[0 .. data.len - 1]) \|bytes\| addBuf(&iovecs, &iovlen, bytes);
	2193	const pattern = data[data.len - 1];
	2194	var backup_buffer: [splat_buffer_size]u8 = undefined;
	2195	if (iovecs.len - iovlen != 0) switch (splat) {
	2196	0 => {},
	2197	1 => addBuf(&iovecs, &iovlen, pattern),
	2198	else => switch (pattern.len) {
	2199	0 => {},
	2200	1 => {
	2201	const splat_buffer = &backup_buffer;
	2202	const memset_len = @min(splat_buffer.len, splat);
	2203	const buf = splat_buffer[0..memset_len];
	2204	@memset(buf, pattern[0]);
	2205	addBuf(&iovecs, &iovlen, buf);
	2206	var remaining_splat = splat - buf.len;
	2207	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0) {
	2208	assert(buf.len == splat_buffer.len);
	2209	addBuf(&iovecs, &iovlen, splat_buffer);
	2210	remaining_splat -= splat_buffer.len;
	2211	}
	2212	addBuf(&iovecs, &iovlen, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
	2213	},
	2214	else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
	2215	addBuf(&iovecs, &iovlen, pattern);
	2216	},
	2217	},
	2218	};
	2219	return ev.pwritev(cancel_region, file.handle, iovecs[0..iovlen], null);
	2220	}
	2221
	2222	fn deviceIoControl(
	2223	ev: *Evented,
	2224	sync: *CancelRegion.Sync,
	2225	o: Io.Operation.DeviceIoControl,
	2226	) Io.Cancelable!i32 {
	2227	_ = ev;
	2228	while (true) {
	2229	try sync.cancel_region.await(.nothing);
	2230	const rc = linux.ioctl(o.file.handle, @bitCast(o.code), @intFromPtr(o.arg));
	2231	switch (linux.errno(rc)) {
	2232	.SUCCESS => return @bitCast(@as(u32, @truncate(rc))),
	2233	.INTR => continue,
	2234	else => \|err\| return -@as(i32, @intFromEnum(err)),
	2235	}
	2236	}
	2237	}
	2238
	2239	fn batchAwaitAsync(userdata: ?anyopaque, batch: Io.Batch) Io.Cancelable!void {
	2240	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2241	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
	2242	defer maybe_sync.deinit(ev);
	2243	ev.batchDrainSubmitted(&maybe_sync, batch, false) catch \|err\| switch (err) {
	2244	error.ConcurrencyUnavailable => unreachable, // passed concurrency=false
	2245	error.Canceled => \|e\| return e,
	2246	};
	2247	maybe_sync.leaveSync(ev);
	2248	while (true) {
	2249	batchDrainReady(batch) catch \|err\| switch (err) {
	2250	error.Timeout => unreachable, // no timeout
	2251	};
	2252	if (batch.completed.head != .none or batch.pending.head == .none) return;
	2253	ev.yield(null, .{ .batch_await = batch });
	2254	}
	2255	}
	2256
	2257	fn batchAwaitConcurrent(
	2258	userdata: ?*anyopaque,
	2259	batch: *Io.Batch,
	2260	timeout: Io.Timeout,
	2261	) Io.Batch.AwaitConcurrentError!void {
	2262	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2263	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
	2264	defer maybe_sync.deinit(ev);
	2265	try ev.batchDrainSubmitted(&maybe_sync, batch, true);
	2266	maybe_sync.leaveSync(ev);
	2267	const timespec: linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = while (true) {
	2268	batchDrainReady(batch) catch \|err\| switch (err) {
	2269	error.Timeout => unreachable, // no timeout
	2270	};
	2271	if (batch.completed.head != .none or batch.pending.head == .none) return;
	2272	switch (timeout) {
	2273	.none => ev.yield(null, .{ .batch_await = batch }),
	2274	.duration => \|duration\| {
	2275	const ns = duration.raw.toNanoseconds();
	2276	break .{
	2277	.{
	2278	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	2279	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	2280	},
	2281	duration.clock,
	2282	0,
	2283	};
	2284	},
	2285	.deadline => \|deadline\| {
	2286	const ns = deadline.raw.toNanoseconds();
	2287	break .{
	2288	.{
	2289	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	2290	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	2291	},
	2292	deadline.clock,
	2293	linux.IORING_TIMEOUT_ABS,
	2294	};
	2295	},
	2296	}
	2297	};
	2298	{
	2299	const thread = try maybe_sync.cancel_region.awaitIoUring();
	2300	thread.enqueue().* = .{
	2301	.opcode = .TIMEOUT,
	2302	.flags = 0,
	2303	.ioprio = 0,
	2304	.fd = 0,
	2305	.off = 0,
	2306	.addr = @intFromPtr(&timespec),
	2307	.len = 1,
	2308	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
	2309	.real => linux.IORING_TIMEOUT_REALTIME,
	2310	else => 0,
	2311	.boot => linux.IORING_TIMEOUT_BOOTTIME,
	2312	}),
	2313	.user_data = @intFromPtr(&batch.userdata) \| 0b11,
	2314	.buf_index = 0,
	2315	.personality = 0,
	2316	.splice_fd_in = 0,
	2317	.addr3 = 0,
	2318	.resv = 0,
	2319	};
	2320	}
	2321	while (batch.completed.head == .none and batch.pending.head != .none) {
	2322	ev.yield(null, .{ .batch_await = batch });
	2323	batchDrainReady(batch) catch \|err\| switch (err) {
	2324	error.Timeout => \|e\| return if (batch.completed.head == .none and
	2325	batch.pending.head != .none) e,
	2326	};
	2327	}
	2328	const thread = try maybe_sync.cancel_region.awaitIoUring();
	2329	thread.enqueue().* = .{
	2330	.opcode = .TIMEOUT_REMOVE,
	2331	.flags = 0,
	2332	.ioprio = 0,
	2333	.fd = 0,
	2334	.off = 0,
	2335	.addr = @intFromPtr(&batch.userdata) \| 0b11,
	2336	.len = 0,
	2337	.rw_flags = 0,
	2338	.user_data = @intFromPtr(maybe_sync.cancel_region.fiber),
	2339	.buf_index = 0,
	2340	.personality = 0,
	2341	.splice_fd_in = 0,
	2342	.addr3 = 0,
	2343	.resv = 0,
	2344	};
	2345	ev.yield(null, .nothing);
	2346	switch (maybe_sync.cancel_region.errno()) {
	2347	.SUCCESS => return,
	2348	.BUSY, .NOENT => {},
	2349	else => \|err\| unexpectedErrno(err) catch {},
	2350	}
	2351	while (true) {
	2352	batchDrainReady(batch) catch \|err\| switch (err) {
	2353	error.Timeout => return,
	2354	};
	2355	ev.yield(null, .{ .batch_await = batch });
	2356	}
	2357	}
	2358
	2359	/// If `concurrency` is false, `error.ConcurrencyUnavailable` is unreachable.
	2360	fn batchDrainSubmitted(
	2361	ev: *Evented,
	2362	maybe_sync: *CancelRegion.Sync.Maybe,
	2363	batch: *Io.Batch,
	2364	concurrency: bool,
	2365	) (Io.ConcurrentError \|\| Io.Cancelable)!void {
	2366	var index = batch.submitted.head;
	2367	if (index == .none) return;
	2368	const thread = try maybe_sync.cancelRegion().awaitIoUring();
	2369	errdefer batch.submitted.head = index;
	2370	while (index != .none) {
	2371	const storage = &batch.storage[index.toIndex()];
	2372	const next_index = storage.submission.node.next;
	2373	if (@as(?Io.Operation.Result, result: switch (storage.submission.operation) {
	2374	.file_read_streaming => \|o\| {
	2375	const buffer = for (o.data) \|buffer\| {
	2376	if (buffer.len > 0) break buffer;
	2377	} else break :result .{ .file_read_streaming = 0 };
	2378	const fd = o.file.handle;
	2379	storage.* = .{ .pending = .{
	2380	.node = .{ .prev = batch.pending.tail, .next = .none },
	2381	.tag = .file_read_streaming,
	2382	.userdata = undefined,
	2383	} };
	2384	thread.enqueue().* = .{
	2385	.opcode = .READ,
	2386	.flags = 0,
	2387	.ioprio = 0,
	2388	.fd = fd,
	2389	.off = std.math.maxInt(u64),
	2390	.addr = @intFromPtr(buffer.ptr),
	2391	.len = @min(buffer.len, 0xfffff000),
	2392	.rw_flags = 0,
	2393	.user_data = @intFromPtr(&storage.pending.userdata) \| 0b10,
	2394	.buf_index = 0,
	2395	.personality = 0,
	2396	.splice_fd_in = 0,
	2397	.addr3 = 0,
	2398	.resv = 0,
	2399	};
	2400	break :result null;
	2401	},
	2402	.file_write_streaming => \|o\| {
	2403	const buffer = buffer: {
	2404	if (o.header.len != 0) break :buffer o.header;
	2405	for (o.data[0 .. o.data.len - 1]) \|buffer\| {
	2406	if (buffer.len > 0) break :buffer buffer;
	2407	}
	2408	if (o.splat > 0) break :buffer o.data[o.data.len - 1];
	2409	break :result .{ .file_write_streaming = 0 };
	2410	};
	2411	const fd = o.file.handle;
	2412	storage.* = .{ .pending = .{
	2413	.node = .{ .prev = batch.pending.tail, .next = .none },
	2414	.tag = .file_write_streaming,
	2415	.userdata = undefined,
	2416	} };
	2417	thread.enqueue().* = .{
	2418	.opcode = .WRITE,
	2419	.flags = 0,
	2420	.ioprio = 0,
	2421	.fd = fd,
	2422	.off = std.math.maxInt(u64),
	2423	.addr = @intFromPtr(buffer.ptr),
	2424	.len = @min(buffer.len, 0xfffff000),
	2425	.rw_flags = 0,
	2426	.user_data = @intFromPtr(&storage.pending.userdata) \| 0b10,
	2427	.buf_index = 0,
	2428	.personality = 0,
	2429	.splice_fd_in = 0,
	2430	.addr3 = 0,
	2431	.resv = 0,
	2432	};
	2433	break :result null;
	2434	},
	2435	.device_io_control => \|o\| if (concurrency)
	2436	return error.ConcurrencyUnavailable
	2437	else
	2438	.{ .device_io_control = try ev.deviceIoControl(try maybe_sync.enterSync(ev), o) },
	2439	})) \|result\| {
	2440	switch (batch.completed.tail) {
	2441	.none => batch.completed.head = index,
	2442	else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next = index,
	2443	}
	2444	batch.completed.tail = index;
	2445	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
	2446	} else {
	2447	switch (batch.pending.tail) {
	2448	.none => batch.pending.head = index,
	2449	else => \|tail_index\| batch.storage[tail_index.toIndex()].pending.node.next = index,
	2450	}
	2451	batch.pending.tail = index;
	2452	storage.pending.userdata[0] = @intFromPtr(batch);
	2453	}
	2454	index = next_index;
	2455	}
	2456	batch.submitted = .{ .head = .none, .tail = .none };
	2457	}
	2458
	2459	fn batchDrainReady(batch: *Io.Batch) Io.Timeout.Error!void {
	2460	while (@atomicRmw(?*anyopaque, &batch.userdata, .Xchg, null, .acquire)) \|head\| {
	2461	var next: usize = @intFromPtr(head);
	2462	var timeout = false;
	2463	while (cond: switch (@as(u2, @truncate(next))) {
	2464	0b00 => if (timeout) return error.Timeout else false,
	2465	0b01 => {
	2466	assert(!timeout);
	2467	return error.Timeout;
	2468	},
	2469	0b10 => true,
	2470	0b11 => {
	2471	assert(!timeout);
	2472	timeout = true;
	2473	break :cond true;
	2474	},
	2475	}) {
	2476	var operation_userdata: *Io.Operation.Storage.Pending.Userdata =
	2477	@ptrFromInt(next & ~@as(usize, 0b11));
	2478	next = operation_userdata[0];
	2479	const completion: Completion = .{
	2480	.result = @bitCast(@as(u32, @intCast(operation_userdata[1]))),
	2481	.flags = @intCast(operation_userdata[2]),
	2482	};
	2483	const pending: *Io.Operation.Storage.Pending =
	2484	@fieldParentPtr("userdata", operation_userdata);
	2485	const storage: *Io.Operation.Storage = @fieldParentPtr("pending", pending);
	2486	const index: Io.Operation.OptionalIndex = .fromIndex(storage - batch.storage.ptr);
	2487	assert(completion.flags & linux.IORING_CQE_F_SKIP == 0);
	2488	switch (pending.node.prev) {
	2489	.none => batch.pending.head = pending.node.next,
	2490	else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.next =
	2491	pending.node.next,
	2492	}
	2493	switch (pending.node.next) {
	2494	.none => batch.pending.tail = pending.node.prev,
	2495	else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.prev =
	2496	pending.node.prev,
	2497	}
	2498	if (@as(?Io.Operation.Result, result: switch (pending.tag) {
	2499	.file_read_streaming => .{
	2500	.file_read_streaming = switch (completion.errno()) {
	2501	.SUCCESS => @as(u32, @bitCast(completion.result)),
	2502	.INTR => 0,
	2503	.CANCELED => break :result null,
	2504	.INVAL => \|err\| errnoBug(err),
	2505	.FAULT => \|err\| errnoBug(err),
	2506	.AGAIN => error.WouldBlock,
	2507	.BADF => \|err\| errnoBug(err), // File descriptor used after closed
	2508	.IO => error.InputOutput,
	2509	.ISDIR => error.IsDir,
	2510	.NOBUFS => error.SystemResources,
	2511	.NOMEM => error.SystemResources,
	2512	.NOTCONN => error.SocketUnconnected,
	2513	.CONNRESET => error.ConnectionResetByPeer,
	2514	else => \|err\| unexpectedErrno(err),
	2515	},
	2516	},
	2517	.file_write_streaming => .{
	2518	.file_write_streaming = switch (completion.errno()) {
	2519	.SUCCESS => @as(u32, @bitCast(completion.result)),
	2520	.INTR => 0,
	2521	.CANCELED => break :result null,
	2522	.INVAL => \|err\| errnoBug(err),
	2523	.FAULT => \|err\| errnoBug(err),
	2524	.AGAIN => error.WouldBlock,
	2525	.BADF => error.NotOpenForWriting, // Can be a race condition.
	2526	.DESTADDRREQ => \|err\| errnoBug(err), // `connect` was never called.
	2527	.DQUOT => error.DiskQuota,
	2528	.FBIG => error.FileTooBig,
	2529	.IO => error.InputOutput,
	2530	.NOSPC => error.NoSpaceLeft,
	2531	.PERM => error.PermissionDenied,
	2532	.PIPE => error.BrokenPipe,
	2533	.CONNRESET => \|err\| errnoBug(err), // Not a socket handle.
	2534	.BUSY => error.DeviceBusy,
	2535	else => \|err\| unexpectedErrno(err),
	2536	},
	2537	},
	2538	.device_io_control => unreachable,
	2539	})) \|result\| {
	2540	switch (batch.completed.tail) {
	2541	.none => batch.completed.head = index,
	2542	else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next =
	2543	index,
	2544	}
	2545	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
	2546	batch.completed.tail = index;
	2547	} else {
	2548	switch (batch.unused.tail) {
	2549	.none => batch.unused.head = index,
	2550	else => \|tail_index\| batch.storage[tail_index.toIndex()].unused.next = index,
	2551	}
	2552	storage.* = .{ .unused = .{ .prev = batch.unused.tail, .next = .none } };
	2553	batch.unused.tail = index;
	2554	}
	2555	}
	2556	}
	2557	}
	2558
	2559	fn batchCancel(userdata: ?anyopaque, batch: Io.Batch) void {
	2560	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2561	_ = ev;
	2562	batchDrainReady(batch) catch \|err\| switch (err) {
	2563	error.Timeout => unreachable, // no timeout
	2564	};
	2565	var index = batch.pending.head;
	2566	if (index == .none) return;
	2567	var cancel_region: CancelRegion = .initBlocked();
	2568	defer cancel_region.deinit();
	2569	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
	2570	error.Canceled => unreachable, // blocked
	2571	};
	2572	while (index != .none) {
	2573	const pending = &batch.storage[index.toIndex()].pending;
	2574	thread.enqueue().* = .{
	2575	.opcode = .ASYNC_CANCEL,
	2576	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	2577	.ioprio = 0,
	2578	.fd = 0,
	2579	.off = 0,
	2580	.addr = @intFromPtr(&pending.userdata) \| 0b10,
	2581	.len = 0,
	2582	.rw_flags = 0,
	2583	.user_data = @intFromEnum(Completion.Userdata.wakeup),
	2584	.buf_index = 0,
	2585	.personality = 0,
	2586	.splice_fd_in = 0,
	2587	.addr3 = 0,
	2588	.resv = 0,
	2589	};
	2590	index = pending.node.next;
	2591	}
	2592	while (batch.pending.head != .none) batchDrainReady(batch) catch \|err\| switch (err) {
	2593	error.Timeout => unreachable, // no timeout
	2594	};
	2595	}
	2596
	2597	fn dirCreateDir(
	2598	userdata: ?*anyopaque,
	2599	dir: Dir,
	2600	sub_path: []const u8,
	2601	permissions: Dir.Permissions,
	2602	) Dir.CreateDirError!void {
	2603	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2604
	2605	var path_buffer: [PATH_MAX]u8 = undefined;
	2606	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2607
	2608	var cancel_region: CancelRegion = .init();
	2609	defer cancel_region.deinit();
	2610	while (true) {
	2611	const thread = try cancel_region.awaitIoUring();
	2612	thread.enqueue().* = .{
	2613	.opcode = .MKDIRAT,
	2614	.flags = 0,
	2615	.ioprio = 0,
	2616	.fd = dir.handle,
	2617	.off = 0,
	2618	.addr = @intFromPtr(sub_path_posix.ptr),
	2619	.len = permissions.toMode(),
	2620	.rw_flags = 0,
	2621	.user_data = @intFromPtr(cancel_region.fiber),
	2622	.buf_index = 0,
	2623	.personality = 0,
	2624	.splice_fd_in = 0,
	2625	.addr3 = 0,
	2626	.resv = 0,
	2627	};
	2628	ev.yield(null, .nothing);
	2629	switch (cancel_region.errno()) {
	2630	.SUCCESS => return,
	2631	.INTR, .CANCELED => continue,
	2632	.ACCES => return error.AccessDenied,
	2633	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	2634	.PERM => return error.PermissionDenied,
	2635	.DQUOT => return error.DiskQuota,
	2636	.EXIST => return error.PathAlreadyExists,
	2637	.FAULT => \|err\| return errnoBug(err),
	2638	.LOOP => return error.SymLinkLoop,
	2639	.MLINK => return error.LinkQuotaExceeded,
	2640	.NAMETOOLONG => return error.NameTooLong,
	2641	.NOENT => return error.FileNotFound,
	2642	.NOMEM => return error.SystemResources,
	2643	.NOSPC => return error.NoSpaceLeft,
	2644	.NOTDIR => return error.NotDir,
	2645	.ROFS => return error.ReadOnlyFileSystem,
	2646	.ILSEQ => return error.BadPathName,
	2647	else => \|err\| return unexpectedErrno(err),
	2648	}
	2649	}
	2650	}
	2651
	2652	fn dirCreateDirPath(
	2653	userdata: ?*anyopaque,
	2654	dir: Dir,
	2655	sub_path: []const u8,
	2656	permissions: Dir.Permissions,
	2657	) Dir.CreateDirPathError!Dir.CreatePathStatus {
	2658	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2659
	2660	var it = Dir.path.componentIterator(sub_path);
	2661	var status: Dir.CreatePathStatus = .existed;
	2662	var component = it.last() orelse return error.BadPathName;
	2663	while (true) {
	2664	if (dirCreateDir(ev, dir, component.path, permissions)) \|_\| {
	2665	status = .created;
	2666	} else \|err\| switch (err) {
	2667	error.PathAlreadyExists => {
	2668	// stat the file and return an error if it's not a directory
	2669	// this is important because otherwise a dangling symlink
	2670	// could cause an infinite loop
	2671	const kind = try ev.filePathKind(dir, component.path);
	2672	if (kind != .directory) return error.NotDir;
	2673	},
	2674	error.FileNotFound => \|e\| {
	2675	component = it.previous() orelse return e;
	2676	continue;
	2677	},
	2678	else => \|e\| return e,
	2679	}
	2680	component = it.next() orelse return status;
	2681	}
	2682	}
	2683
	2684	fn filePathKind(ev: *Evented, dir: Dir, sub_path: []const u8) !File.Kind {
	2685	var path_buffer: [PATH_MAX]u8 = undefined;
	2686	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2687	var cancel_region: CancelRegion = .init();
	2688	defer cancel_region.deinit();
	2689	while (true) {
	2690	var statx_buf = std.mem.zeroes(linux.Statx);
	2691	const thread = try cancel_region.awaitIoUring();
	2692	thread.enqueue().* = .{
	2693	.opcode = .STATX,
	2694	.flags = 0,
	2695	.ioprio = 0,
	2696	.fd = dir.handle,
	2697	.off = @intFromPtr(&statx_buf),
	2698	.addr = @intFromPtr(sub_path_posix.ptr),
	2699	.len = @bitCast(linux.STATX{ .TYPE = true }),
	2700	.rw_flags = linux.AT.NO_AUTOMOUNT \| linux.AT.SYMLINK_NOFOLLOW,
	2701	.user_data = @intFromPtr(cancel_region.fiber),
	2702	.buf_index = 0,
	2703	.personality = 0,
	2704	.splice_fd_in = 0,
	2705	.addr3 = 0,
	2706	.resv = 0,
	2707	};
	2708	ev.yield(null, .nothing);
	2709	switch (cancel_region.errno()) {
	2710	.SUCCESS => {
	2711	if (!statx_buf.mask.TYPE) return error.Unexpected;
	2712	return statxKind(statx_buf.mode);
	2713	},
	2714	.INTR, .CANCELED => continue,
	2715	.ACCES => \|err\| return errnoBug(err),
	2716	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	2717	.FAULT => \|err\| return errnoBug(err),
	2718	.INVAL => \|err\| return errnoBug(err),
	2719	.LOOP => \|err\| return errnoBug(err),
	2720	.NAMETOOLONG => \|err\| return errnoBug(err),
	2721	.NOENT => \|err\| return errnoBug(err),
	2722	.NOMEM => return error.SystemResources,
	2723	.NOTDIR => \|err\| return errnoBug(err),
	2724	else => \|err\| return unexpectedErrno(err),
	2725	}
	2726	}
	2727	}
	2728
	2729	fn dirCreateDirPathOpen(
	2730	userdata: ?*anyopaque,
	2731	dir: Dir,
	2732	sub_path: []const u8,
	2733	permissions: Dir.Permissions,
	2734	options: Dir.OpenOptions,
	2735	) Dir.CreateDirPathOpenError!Dir {
	2736	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2737	return dirOpenDir(ev, dir, sub_path, options) catch \|err\| switch (err) {
	2738	error.FileNotFound => {
	2739	_ = try dirCreateDirPath(ev, dir, sub_path, permissions);
	2740	return dirOpenDir(ev, dir, sub_path, options);
	2741	},
	2742	else => \|e\| return e,
	2743	};
	2744	}
	2745
	2746	fn dirOpenDir(
	2747	userdata: ?*anyopaque,
	2748	dir: Dir,
	2749	sub_path: []const u8,
	2750	options: Dir.OpenOptions,
	2751	) Dir.OpenError!Dir {
	2752	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2753
	2754	var path_buffer: [PATH_MAX]u8 = undefined;
	2755	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2756
	2757	var cancel_region: CancelRegion = .init();
	2758	defer cancel_region.deinit();
	2759	return .{
	2760	.handle = ev.openat(&cancel_region, dir.handle, sub_path_posix, .{
	2761	.ACCMODE = .RDONLY,
	2762	.DIRECTORY = true,
	2763	.NOFOLLOW = !options.follow_symlinks,
	2764	.CLOEXEC = true,
	2765	.PATH = !options.iterate,
	2766	}, 0) catch \|err\| switch (err) {
	2767	error.IsDir => return errnoBug(.ISDIR),
	2768	error.WouldBlock => return errnoBug(.AGAIN),
	2769	error.FileTooBig => return errnoBug(.FBIG),
	2770	error.NoSpaceLeft => return errnoBug(.NOSPC),
	2771	error.DeviceBusy => return errnoBug(.BUSY), // O_EXCL not passed
	2772	error.FileBusy => return errnoBug(.TXTBSY),
	2773	error.PathAlreadyExists => return errnoBug(.EXIST), // Not creating.
	2774	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
	2775	error.AntivirusInterference => unreachable, // Windows-only
	2776	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
	2777	else => \|e\| return e,
	2778	},
	2779	};
	2780	}
	2781
	2782	fn dirStat(userdata: ?*anyopaque, dir: Dir) Dir.StatError!Dir.Stat {
	2783	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2784	var cancel_region: CancelRegion = .init();
	2785	defer cancel_region.deinit();
	2786	return ev.stat(&cancel_region, dir.handle);
	2787	}
	2788
	2789	fn dirStatFile(
	2790	userdata: ?*anyopaque,
	2791	dir: Dir,
	2792	sub_path: []const u8,
	2793	options: Dir.StatFileOptions,
	2794	) Dir.StatFileError!File.Stat {
	2795	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2796	var path_buffer: [PATH_MAX]u8 = undefined;
	2797	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2798	var cancel_region: CancelRegion = .init();
	2799	defer cancel_region.deinit();
	2800	return ev.statx(&cancel_region, dir.handle, sub_path_posix, linux.AT.NO_AUTOMOUNT \|
	2801	@as(u32, if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW));
	2802	}
	2803
	2804	fn dirAccess(
	2805	userdata: ?*anyopaque,
	2806	dir: Dir,
	2807	sub_path: []const u8,
	2808	options: Dir.AccessOptions,
	2809	) Dir.AccessError!void {
	2810	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2811
	2812	var path_buffer: [PATH_MAX]u8 = undefined;
	2813	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2814
	2815	const mode: u32 =
	2816	@as(u32, if (options.read) linux.R_OK else 0) \|
	2817	@as(u32, if (options.write) linux.W_OK else 0) \|
	2818	@as(u32, if (options.execute) linux.X_OK else 0);
	2819	const flags: u32 = if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW;
	2820
	2821	var sync: CancelRegion.Sync = try .init(ev);
	2822	defer sync.deinit(ev);
	2823	while (true) {
	2824	try sync.cancel_region.await(.nothing);
	2825	switch (linux.errno(linux.faccessat(dir.handle, sub_path_posix, mode, flags))) {
	2826	.SUCCESS => return,
	2827	.INTR => continue,
	2828	.ACCES => return error.AccessDenied,
	2829	.PERM => return error.PermissionDenied,
	2830	.ROFS => return error.ReadOnlyFileSystem,
	2831	.LOOP => return error.SymLinkLoop,
	2832	.TXTBSY => return error.FileBusy,
	2833	.NOTDIR => return error.FileNotFound,
	2834	.NOENT => return error.FileNotFound,
	2835	.NAMETOOLONG => return error.NameTooLong,
	2836	.INVAL => \|err\| return errnoBug(err),
	2837	.FAULT => \|err\| return errnoBug(err),
	2838	.IO => return error.InputOutput,
	2839	.NOMEM => return error.SystemResources,
	2840	.ILSEQ => return error.BadPathName,
	2841	else => \|err\| return unexpectedErrno(err),
	2842	}
	2843	}
	2844	}
	2845
	2846	fn dirCreateFile(
	2847	userdata: ?*anyopaque,
	2848	dir: Dir,
	2849	sub_path: []const u8,
	2850	flags: File.CreateFlags,
	2851	) File.OpenError!File {
	2852	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2853
	2854	var path_buffer: [PATH_MAX]u8 = undefined;
	2855	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2856
	2857	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
	2858	defer maybe_sync.deinit(ev);
	2859	const fd = try ev.openat(&maybe_sync.cancel_region, dir.handle, sub_path_posix, .{
	2860	.ACCMODE = if (flags.read) .RDWR else .WRONLY,
	2861	.CREAT = true,
	2862	.TRUNC = flags.truncate,
	2863	.EXCL = flags.exclusive,
	2864	.CLOEXEC = true,
	2865	}, flags.permissions.toMode());
	2866	errdefer ev.close(fd);
	2867
	2868	switch (flags.lock) {
	2869	.none => {},
	2870	.shared, .exclusive => try ev.flock(
	2871	try maybe_sync.enterSync(ev),
	2872	fd,
	2873	flags.lock,
	2874	if (flags.lock_nonblocking) .nonblocking else .blocking,
	2875	),
	2876	}
	2877
	2878	return .{ .handle = fd, .flags = .{ .nonblocking = false } };
	2879	}
	2880
	2881	fn dirCreateFileAtomic(
	2882	userdata: ?*anyopaque,
	2883	dir: Dir,
	2884	dest_path: []const u8,
	2885	options: Dir.CreateFileAtomicOptions,
	2886	) Dir.CreateFileAtomicError!File.Atomic {
	2887	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2888	// Linux has O_TMPFILE, but linkat() does not support AT_REPLACE, so it's
	2889	// useless when we have to make up a bogus path name to do the rename()
	2890	// anyway.
	2891	if (!options.replace) tmpfile: {
	2892	const flags: linux.O = if (@hasField(linux.O, "TMPFILE")) .{
	2893	.ACCMODE = .RDWR,
	2894	.TMPFILE = true,
	2895	.DIRECTORY = true,
	2896	.CLOEXEC = true,
	2897	} else if (@hasField(linux.O, "TMPFILE0") and !@hasField(linux.O, "TMPFILE2")) .{
	2898	.ACCMODE = .RDWR,
	2899	.TMPFILE0 = true,
	2900	.TMPFILE1 = true,
	2901	.DIRECTORY = true,
	2902	.CLOEXEC = true,
	2903	} else break :tmpfile;
	2904
	2905	const dest_dirname = Dir.path.dirname(dest_path);
	2906	if (dest_dirname) \|dirname\| {
	2907	// This has a nice side effect of preemptively triggering EISDIR or
	2908	// ENOENT, avoiding the ambiguity below.
	2909	_ = dirCreateDirPath(ev, dir, dirname, .default_dir) catch \|err\| switch (err) {
	2910	// None of these make sense in this context.
	2911	error.IsDir,
	2912	error.Streaming,
	2913	error.DiskQuota,
	2914	error.PathAlreadyExists,
	2915	error.LinkQuotaExceeded,
	2916	error.PipeBusy,
	2917	error.FileTooBig,
	2918	error.DeviceBusy,
	2919	error.FileLocksUnsupported,
	2920	error.FileBusy,
	2921	=> return error.Unexpected,
	2922
	2923	else => \|e\| return e,
	2924	};
	2925	}
	2926
	2927	var path_buffer: [PATH_MAX]u8 = undefined;
	2928	const sub_path_posix = try pathToPosix(dest_dirname orelse ".", &path_buffer);
	2929
	2930	var cancel_region: CancelRegion = .init();
	2931	defer cancel_region.deinit();
	2932	return .{
	2933	.file = .{
	2934	.handle = ev.openat(
	2935	&cancel_region,
	2936	dir.handle,
	2937	sub_path_posix,
	2938	flags,
	2939	options.permissions.toMode(),
	2940	) catch \|err\| switch (err) {
	2941	error.IsDir, error.FileNotFound => {
	2942	// Ambiguous error code. It might mean the file system
	2943	// does not support O_TMPFILE. Therefore, we must fall
	2944	// back to not using O_TMPFILE.
	2945	break :tmpfile;
	2946	},
	2947	error.FileTooBig => return errnoBug(.FBIG),
	2948	error.DeviceBusy => return errnoBug(.BUSY), // O_EXCL not passed
	2949	error.PathAlreadyExists => return errnoBug(.EXIST), // Not creating.
	2950	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
	2951	error.AntivirusInterference => unreachable, // Windows-only
	2952	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
	2953	else => \|e\| return e,
	2954	},
	2955	.flags = .{ .nonblocking = false },
	2956	},
	2957	.file_basename_hex = 0,
	2958	.dest_sub_path = dest_path,
	2959	.file_open = true,
	2960	.file_exists = false,
	2961	.close_dir_on_deinit = false,
	2962	.dir = dir,
	2963	};
	2964	}
	2965
	2966	if (Dir.path.dirname(dest_path)) \|dirname\| {
	2967	const new_dir = if (options.make_path)
	2968	dirCreateDirPathOpen(ev, dir, dirname, .default_dir, .{}) catch \|err\| switch (err) {
	2969	// None of these make sense in this context.
	2970	error.IsDir,
	2971	error.Streaming,
	2972	error.DiskQuota,
	2973	error.PathAlreadyExists,
	2974	error.LinkQuotaExceeded,
	2975	error.PipeBusy,
	2976	error.FileTooBig,
	2977	error.FileLocksUnsupported,
	2978	error.DeviceBusy,
	2979	=> return error.Unexpected,
	2980
	2981	else => \|e\| return e,
	2982	}
	2983	else
	2984	try dirOpenDir(ev, dir, dirname, .{});
	2985
	2986	return ev.atomicFileInit(Dir.path.basename(dest_path), options.permissions, new_dir, true);
	2987	}
	2988
	2989	return ev.atomicFileInit(dest_path, options.permissions, dir, false);
	2990	}
	2991
	2992	fn atomicFileInit(
	2993	ev: *Evented,
	2994	dest_basename: []const u8,
	2995	permissions: File.Permissions,
	2996	dir: Dir,
	2997	close_dir_on_deinit: bool,
	2998	) Dir.CreateFileAtomicError!File.Atomic {
	2999	while (true) {
	3000	var random_integer: u64 = undefined;
	3001	random(ev, @ptrCast(&random_integer));
	3002	const tmp_sub_path = std.fmt.hex(random_integer);
	3003	const file = dirCreateFile(ev, dir, &tmp_sub_path, .{
	3004	.permissions = permissions,
	3005	.exclusive = true,
	3006	}) catch \|err\| switch (err) {
	3007	error.PathAlreadyExists => continue,
	3008	error.DeviceBusy => continue,
	3009	error.FileBusy => continue,
	3010
	3011	error.IsDir => return error.Unexpected, // No path components.
	3012	error.FileTooBig => return error.Unexpected, // Creating, not opening.
	3013	error.FileLocksUnsupported => return error.Unexpected, // Not asking for locks.
	3014	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
	3015
	3016	else => \|e\| return e,
	3017	};
	3018	return .{
	3019	.file = file,
	3020	.file_basename_hex = random_integer,
	3021	.dest_sub_path = dest_basename,
	3022	.file_open = true,
	3023	.file_exists = true,
	3024	.close_dir_on_deinit = close_dir_on_deinit,
	3025	.dir = dir,
	3026	};
	3027	}
	3028	}
	3029
	3030	fn dirOpenFile(
	3031	userdata: ?*anyopaque,
	3032	dir: Dir,
	3033	sub_path: []const u8,
	3034	flags: File.OpenFlags,
	3035	) File.OpenError!File {
	3036	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3037
	3038	var path_buffer: [PATH_MAX]u8 = undefined;
	3039	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3040
	3041	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
	3042	defer maybe_sync.deinit(ev);
	3043	const fd = try ev.openat(&maybe_sync.cancel_region, dir.handle, sub_path_posix, .{
	3044	.ACCMODE = switch (flags.mode) {
	3045	.read_only => .RDONLY,
	3046	.write_only => .WRONLY,
	3047	.read_write => .RDWR,
	3048	},
	3049	.NOCTTY = !flags.allow_ctty,
	3050	.NOFOLLOW = !flags.follow_symlinks,
	3051	.CLOEXEC = true,
	3052	.PATH = flags.path_only,
	3053	}, 0);
	3054	errdefer ev.close(fd);
	3055
	3056	if (!flags.allow_directory) {
	3057	const is_dir = is_dir: {
	3058	const s = ev.stat(&maybe_sync.cancel_region, fd) catch \|err\| switch (err) {
	3059	// The directory-ness is either unknown or unknowable
	3060	error.Streaming => break :is_dir false,
	3061	else => \|e\| return e,
	3062	};
	3063	break :is_dir s.kind == .directory;
	3064	};
	3065	if (is_dir) return error.IsDir;
	3066	}
	3067
	3068	switch (flags.lock) {
	3069	.none => {},
	3070	.shared, .exclusive => try ev.flock(
	3071	try maybe_sync.enterSync(ev),
	3072	fd,
	3073	flags.lock,
	3074	if (flags.lock_nonblocking) .nonblocking else .blocking,
	3075	),
	3076	}
	3077
	3078	return .{ .handle = fd, .flags = .{ .nonblocking = false } };
	3079	}
	3080
	3081	fn dirClose(userdata: ?*anyopaque, dirs: []const Dir) void {
	3082	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3083	for (dirs) \|dir\| ev.close(dir.handle);
	3084	}
	3085
	3086	fn dirRead(userdata: ?anyopaque, dr: Dir.Reader, buffer: []Dir.Entry) Dir.Reader.Error!usize {
	3087	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3088	var buffer_index: usize = 0;
	3089	while (buffer.len - buffer_index != 0) {
	3090	if (dr.end - dr.index == 0) {
	3091	// Refill the buffer, unless we've already created references to
	3092	// buffered data.
	3093	if (buffer_index != 0) break;
	3094	var sync: CancelRegion.Sync = try .init(ev);
	3095	defer sync.deinit(ev);
	3096	if (dr.state == .reset) {
	3097	ev.lseek(&sync, dr.dir.handle, 0, linux.SEEK.SET) catch \|err\| switch (err) {
	3098	error.Unseekable => return error.Unexpected,
	3099	else => \|e\| return e,
	3100	};
	3101	dr.state = .reading;
	3102	}
	3103	const n = while (true) {
	3104	try sync.cancel_region.await(.nothing);
	3105	const rc = linux.getdents64(dr.dir.handle, dr.buffer.ptr, dr.buffer.len);
	3106	switch (linux.errno(rc)) {
	3107	.SUCCESS => break rc,
	3108	.INTR => continue,
	3109	.BADF => \|err\| return errnoBug(err), // Dir is invalid or was opened without iteration ability.
	3110	.FAULT => \|err\| return errnoBug(err),
	3111	.NOTDIR => \|err\| return errnoBug(err),
	3112	// To be consistent across platforms, iteration
	3113	// ends if the directory being iterated is deleted
	3114	// during iteration. This matches the behavior of
	3115	// non-Linux, non-WASI UNIX platforms.
	3116	.NOENT => {
	3117	dr.state = .finished;
	3118	return 0;
	3119	},
	3120	// This can occur when reading /proc/$PID/net, or
	3121	// if the provided buffer is too small. Neither
	3122	// scenario is intended to be handled by this API.
	3123	.INVAL => return error.Unexpected,
	3124	.ACCES => return error.AccessDenied, // Lacking permission to iterate this directory.
	3125	else => \|err\| return unexpectedErrno(err),
	3126	}
	3127	};
	3128	if (n == 0) {
	3129	dr.state = .finished;
	3130	return 0;
	3131	}
	3132	dr.index = 0;
	3133	dr.end = n;
	3134	}
	3135	// Linux aligns the header by padding after the null byte of the name
	3136	// to align the next entry. This means we can find the end of the name
	3137	// by looking at only the 8 bytes before the next record. However since
	3138	// file names are usually short it's better to keep the machine code
	3139	// simpler.
	3140	//
	3141	// Furthermore, I observed qemu user mode to not align this struct, so
	3142	// this code makes the conservative choice to not assume alignment.
	3143	const linux_entry: *align(1) linux.dirent64 = @ptrCast(&dr.buffer[dr.index]);
	3144	const next_index = dr.index + linux_entry.reclen;
	3145	dr.index = next_index;
	3146	const name_ptr: [*]u8 = &linux_entry.name;
	3147	const padded_name = name_ptr[0 .. linux_entry.reclen - @offsetOf(linux.dirent64, "name")];
	3148	const name_len = std.mem.findScalar(u8, padded_name, 0).?;
	3149	const name = name_ptr[0..name_len :0];
	3150
	3151	if (std.mem.eql(u8, name, ".") or std.mem.eql(u8, name, "..")) continue;
	3152
	3153	const entry_kind: File.Kind = switch (linux_entry.type) {
	3154	linux.DT.BLK => .block_device,
	3155	linux.DT.CHR => .character_device,
	3156	linux.DT.DIR => .directory,
	3157	linux.DT.FIFO => .named_pipe,
	3158	linux.DT.LNK => .sym_link,
	3159	linux.DT.REG => .file,
	3160	linux.DT.SOCK => .unix_domain_socket,
	3161	else => .unknown,
	3162	};
	3163	buffer[buffer_index] = .{
	3164	.name = name,
	3165	.kind = entry_kind,
	3166	.inode = linux_entry.ino,
	3167	};
	3168	buffer_index += 1;
	3169	}
	3170	return buffer_index;
	3171	}
	3172
	3173	fn dirRealPath(userdata: ?*anyopaque, dir: Dir, out_buffer: []u8) Dir.RealPathError!usize {
	3174	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3175	var sync: CancelRegion.Sync = try .init(ev);
	3176	defer sync.deinit(ev);
	3177	return ev.realPath(&sync, dir.handle, out_buffer);
	3178	}
	3179
	3180	fn dirRealPathFile(
	3181	userdata: ?*anyopaque,
	3182	dir: Dir,
	3183	sub_path: []const u8,
	3184	out_buffer: []u8,
	3185	) Dir.RealPathFileError!usize {
	3186	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3187
	3188	var path_buffer: [PATH_MAX]u8 = undefined;
	3189	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3190
	3191	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
	3192	defer maybe_sync.deinit(ev);
	3193	const fd = ev.openat(&maybe_sync.cancel_region, dir.handle, sub_path_posix, .{
	3194	.CLOEXEC = true,
	3195	.PATH = true,
	3196	}, 0) catch \|err\| switch (err) {
	3197	error.WouldBlock => return errnoBug(.AGAIN),
	3198	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
	3199	else => \|e\| return e,
	3200	};
	3201	defer ev.close(fd);
	3202	return ev.realPath(try maybe_sync.enterSync(ev), fd, out_buffer);
	3203	}
	3204
	3205	fn dirDeleteFile(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteFileError!void {
	3206	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3207
	3208	var path_buffer: [PATH_MAX]u8 = undefined;
	3209	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3210
	3211	var cancel_region: CancelRegion = .init();
	3212	defer cancel_region.deinit();
	3213	while (true) {
	3214	const thread = try cancel_region.awaitIoUring();
	3215	thread.enqueue().* = .{
	3216	.opcode = .UNLINKAT,
	3217	.flags = 0,
	3218	.ioprio = 0,
	3219	.fd = dir.handle,
	3220	.off = 0,
	3221	.addr = @intFromPtr(sub_path_posix.ptr),
	3222	.len = 0,
	3223	.rw_flags = 0,
	3224	.user_data = @intFromPtr(cancel_region.fiber),
	3225	.buf_index = 0,
	3226	.personality = 0,
	3227	.splice_fd_in = 0,
	3228	.addr3 = 0,
	3229	.resv = 0,
	3230	};
	3231	ev.yield(null, .nothing);
	3232	switch (cancel_region.errno()) {
	3233	.SUCCESS => return,
	3234	.INTR, .CANCELED => continue,
	3235	.PERM => return error.PermissionDenied,
	3236	.ACCES => return error.AccessDenied,
	3237	.BUSY => return error.FileBusy,
	3238	.FAULT => \|err\| return errnoBug(err),
	3239	.IO => return error.FileSystem,
	3240	.ISDIR => return error.IsDir,
	3241	.LOOP => return error.SymLinkLoop,
	3242	.NAMETOOLONG => return error.NameTooLong,
	3243	.NOENT => return error.FileNotFound,
	3244	.NOTDIR => return error.NotDir,
	3245	.NOMEM => return error.SystemResources,
	3246	.ROFS => return error.ReadOnlyFileSystem,
	3247	.EXIST => \|err\| return errnoBug(err),
	3248	.NOTEMPTY => \|err\| return errnoBug(err), // Not passing AT.REMOVEDIR
	3249	.ILSEQ => return error.BadPathName,
	3250	.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
	3251	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	3252	else => \|err\| return unexpectedErrno(err),
	3253	}
	3254	}
	3255	}
	3256
	3257	fn dirDeleteDir(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteDirError!void {
	3258	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3259
	3260	var path_buffer: [PATH_MAX]u8 = undefined;
	3261	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3262
	3263	var cancel_region: CancelRegion = .init();
	3264	defer cancel_region.deinit();
	3265	while (true) {
	3266	const thread = try cancel_region.awaitIoUring();
	3267	thread.enqueue().* = .{
	3268	.opcode = .UNLINKAT,
	3269	.flags = 0,
	3270	.ioprio = 0,
	3271	.fd = dir.handle,
	3272	.off = 0,
	3273	.addr = @intFromPtr(sub_path_posix.ptr),
	3274	.len = 0,
	3275	.rw_flags = linux.AT.REMOVEDIR,
	3276	.user_data = @intFromPtr(cancel_region.fiber),
	3277	.buf_index = 0,
	3278	.personality = 0,
	3279	.splice_fd_in = 0,
	3280	.addr3 = 0,
	3281	.resv = 0,
	3282	};
	3283	ev.yield(null, .nothing);
	3284	switch (cancel_region.errno()) {
	3285	.SUCCESS => return,
	3286	.INTR, .CANCELED => continue,
	3287	.ACCES => return error.AccessDenied,
	3288	.PERM => return error.PermissionDenied,
	3289	.BUSY => return error.FileBusy,
	3290	.FAULT => \|err\| return errnoBug(err),
	3291	.IO => return error.FileSystem,
	3292	.ISDIR => \|err\| return errnoBug(err),
	3293	.LOOP => return error.SymLinkLoop,
	3294	.NAMETOOLONG => return error.NameTooLong,
	3295	.NOENT => return error.FileNotFound,
	3296	.NOTDIR => return error.NotDir,
	3297	.NOMEM => return error.SystemResources,
	3298	.ROFS => return error.ReadOnlyFileSystem,
	3299	.EXIST => \|err\| return errnoBug(err),
	3300	.NOTEMPTY => return error.DirNotEmpty,
	3301	.ILSEQ => return error.BadPathName,
	3302	.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
	3303	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	3304	else => \|err\| return unexpectedErrno(err),
	3305	}
	3306	}
	3307	}
	3308
	3309	fn dirRename(
	3310	userdata: ?*anyopaque,
	3311	old_dir: Dir,
	3312	old_sub_path: []const u8,
	3313	new_dir: Dir,
	3314	new_sub_path: []const u8,
	3315	) Dir.RenameError!void {
	3316	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3317
	3318	var old_path_buffer: [PATH_MAX]u8 = undefined;
	3319	var new_path_buffer: [PATH_MAX]u8 = undefined;
	3320
	3321	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
	3322	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
	3323
	3324	var cancel_region: CancelRegion = .init();
	3325	defer cancel_region.deinit();
	3326	return ev.renameat(
	3327	&cancel_region,
	3328	old_dir.handle,
	3329	old_sub_path_posix,
	3330	new_dir.handle,
	3331	new_sub_path_posix,
	3332	.{},
	3333	);
	3334	}
	3335
	3336	fn dirRenamePreserve(
	3337	userdata: ?*anyopaque,
	3338	old_dir: Dir,
	3339	old_sub_path: []const u8,
	3340	new_dir: Dir,
	3341	new_sub_path: []const u8,
	3342	) Dir.RenamePreserveError!void {
	3343	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3344
	3345	var old_path_buffer: [PATH_MAX]u8 = undefined;
	3346	var new_path_buffer: [PATH_MAX]u8 = undefined;
	3347
	3348	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
	3349	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
	3350
	3351	var cancel_region: CancelRegion = .init();
	3352	defer cancel_region.deinit();
	3353	return ev.renameat(
	3354	&cancel_region,
	3355	old_dir.handle,
	3356	old_sub_path_posix,
	3357	new_dir.handle,
	3358	new_sub_path_posix,
	3359	.{ .NOREPLACE = true },
	3360	);
	3361	}
	3362
	3363	fn dirSymLink(
	3364	userdata: ?*anyopaque,
	3365	dir: Dir,
	3366	target_path: []const u8,
	3367	sym_link_path: []const u8,
	3368	flags: Dir.SymLinkFlags,
	3369	) Dir.SymLinkError!void {
	3370	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3371	_ = flags;
	3372
	3373	var target_path_buffer: [PATH_MAX]u8 = undefined;
	3374	var sym_link_path_buffer: [PATH_MAX]u8 = undefined;
	3375
	3376	const target_path_posix = try pathToPosix(target_path, &target_path_buffer);
	3377	const sym_link_path_posix = try pathToPosix(sym_link_path, &sym_link_path_buffer);
	3378
	3379	var cancel_region: CancelRegion = .init();
	3380	defer cancel_region.deinit();
	3381	while (true) {
	3382	const thread = try cancel_region.awaitIoUring();
	3383	thread.enqueue().* = .{
	3384	.opcode = .SYMLINKAT,
	3385	.flags = 0,
	3386	.ioprio = 0,
	3387	.fd = dir.handle,
	3388	.off = @intFromPtr(sym_link_path_posix.ptr),
	3389	.addr = @intFromPtr(target_path_posix.ptr),
	3390	.len = 0,
	3391	.rw_flags = 0,
	3392	.user_data = @intFromPtr(cancel_region.fiber),
	3393	.buf_index = 0,
	3394	.personality = 0,
	3395	.splice_fd_in = 0,
	3396	.addr3 = 0,
	3397	.resv = 0,
	3398	};
	3399	ev.yield(null, .nothing);
	3400	switch (cancel_region.errno()) {
	3401	.SUCCESS => return,
	3402	.INTR, .CANCELED => continue,
	3403	.FAULT => \|err\| return errnoBug(err),
	3404	.INVAL => \|err\| return errnoBug(err),
	3405	.ACCES => return error.AccessDenied,
	3406	.PERM => return error.PermissionDenied,
	3407	.DQUOT => return error.DiskQuota,
	3408	.EXIST => return error.PathAlreadyExists,
	3409	.IO => return error.FileSystem,
	3410	.LOOP => return error.SymLinkLoop,
	3411	.NAMETOOLONG => return error.NameTooLong,
	3412	.NOENT => return error.FileNotFound,
	3413	.NOTDIR => return error.NotDir,
	3414	.NOMEM => return error.SystemResources,
	3415	.NOSPC => return error.NoSpaceLeft,
	3416	.ROFS => return error.ReadOnlyFileSystem,
	3417	.ILSEQ => return error.BadPathName,
	3418	else => \|err\| return unexpectedErrno(err),
	3419	}
	3420	}
	3421	}
	3422
	3423	fn dirReadLink(
	3424	userdata: ?*anyopaque,
	3425	dir: Dir,
	3426	sub_path: []const u8,
	3427	buffer: []u8,
	3428	) Dir.ReadLinkError!usize {
	3429	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3430
	3431	var sub_path_buffer: [PATH_MAX]u8 = undefined;
	3432	const sub_path_posix = try pathToPosix(sub_path, &sub_path_buffer);
	3433
	3434	var sync: CancelRegion.Sync = try .init(ev);
	3435	defer sync.deinit(ev);
	3436	while (true) {
	3437	try sync.cancel_region.await(.nothing);
	3438	const rc = linux.readlinkat(dir.handle, sub_path_posix, buffer.ptr, buffer.len);
	3439	switch (linux.errno(rc)) {
	3440	.SUCCESS => return @bitCast(rc),
	3441	.INTR => continue,
	3442	.ACCES => return error.AccessDenied,
	3443	.FAULT => \|err\| return errnoBug(err),
	3444	.INVAL => return error.NotLink,
	3445	.IO => return error.FileSystem,
	3446	.LOOP => return error.SymLinkLoop,
	3447	.NAMETOOLONG => return error.NameTooLong,
	3448	.NOENT => return error.FileNotFound,
	3449	.NOMEM => return error.SystemResources,
	3450	.NOTDIR => return error.NotDir,
	3451	.ILSEQ => return error.BadPathName,
	3452	else => \|err\| return unexpectedErrno(err),
	3453	}
	3454	}
	3455	}
	3456
	3457	fn dirSetOwner(
	3458	userdata: ?*anyopaque,
	3459	dir: Dir,
	3460	owner: ?File.Uid,
	3461	group: ?File.Gid,
	3462	) Dir.SetOwnerError!void {
	3463	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3464	var sync: CancelRegion.Sync = try .init(ev);
	3465	defer sync.deinit(ev);
	3466	try ev.fchownat(
	3467	&sync,
	3468	dir.handle,
	3469	"",
	3470	owner orelse std.math.maxInt(linux.uid_t),
	3471	group orelse std.math.maxInt(linux.gid_t),
	3472	linux.AT.EMPTY_PATH,
	3473	);
	3474	}
	3475
	3476	fn dirSetFileOwner(
	3477	userdata: ?*anyopaque,
	3478	dir: Dir,
	3479	sub_path: []const u8,
	3480	owner: ?File.Uid,
	3481	group: ?File.Gid,
	3482	options: Dir.SetFileOwnerOptions,
	3483	) Dir.SetFileOwnerError!void {
	3484	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3485	var path_buffer: [PATH_MAX]u8 = undefined;
	3486	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3487	var sync: CancelRegion.Sync = try .init(ev);
	3488	defer sync.deinit(ev);
	3489	try ev.fchownat(
	3490	&sync,
	3491	dir.handle,
	3492	sub_path_posix,
	3493	owner orelse std.math.maxInt(linux.uid_t),
	3494	group orelse std.math.maxInt(linux.gid_t),
	3495	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
	3496	);
	3497	}
	3498
	3499	fn dirSetPermissions(
	3500	userdata: ?*anyopaque,
	3501	dir: Dir,
	3502	permissions: Dir.Permissions,
	3503	) Dir.SetPermissionsError!void {
	3504	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3505	var sync: CancelRegion.Sync = try .init(ev);
	3506	defer sync.deinit(ev);
	3507	ev.fchmodat(
	3508	&sync,
	3509	dir.handle,
	3510	"",
	3511	permissions.toMode(),
	3512	linux.AT.EMPTY_PATH,
	3513	) catch \|err\| switch (err) {
	3514	error.NameTooLong => return errnoBug(.NAMETOOLONG),
	3515	error.BadPathName => return errnoBug(.ILSEQ),
	3516	error.ProcessFdQuotaExceeded => return errnoBug(.MFILE),
	3517	error.SystemFdQuotaExceeded => return errnoBug(.NFILE),
	3518	error.OperationUnsupported => return errnoBug(.OPNOTSUPP),
	3519	else => \|e\| return e,
	3520	};
	3521	}
	3522
	3523	fn dirSetFilePermissions(
	3524	userdata: ?*anyopaque,
	3525	dir: Dir,
	3526	sub_path: []const u8,
	3527	permissions: Dir.Permissions,
	3528	options: Dir.SetFilePermissionsOptions,
	3529	) Dir.SetFilePermissionsError!void {
	3530	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3531	var path_buffer: [PATH_MAX]u8 = undefined;
	3532	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3533	var sync: CancelRegion.Sync = try .init(ev);
	3534	defer sync.deinit(ev);
	3535	try ev.fchmodat(
	3536	&sync,
	3537	dir.handle,
	3538	sub_path_posix,
	3539	permissions.toMode(),
	3540	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
	3541	);
	3542	}
	3543
	3544	fn dirSetTimestamps(
	3545	userdata: ?*anyopaque,
	3546	dir: Dir,
	3547	sub_path: []const u8,
	3548	options: Dir.SetTimestampsOptions,
	3549	) Dir.SetTimestampsError!void {
	3550	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3551	var path_buffer: [PATH_MAX]u8 = undefined;
	3552	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3553	var cancel_region: CancelRegion.Sync = try .init(ev);
	3554	defer cancel_region.deinit(ev);
	3555	try ev.utimensat(
	3556	&cancel_region,
	3557	dir.handle,
	3558	sub_path_posix,
	3559	if (options.modify_timestamp != .now or options.access_timestamp != .now) &.{
	3560	setTimestampToPosix(options.access_timestamp),
	3561	setTimestampToPosix(options.modify_timestamp),
	3562	} else null,
	3563	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
	3564	);
	3565	}
	3566
	3567	fn dirHardLink(
	3568	userdata: ?*anyopaque,
	3569	old_dir: Dir,
	3570	old_sub_path: []const u8,
	3571	new_dir: Dir,
	3572	new_sub_path: []const u8,
	3573	options: Dir.HardLinkOptions,
	3574	) Dir.HardLinkError!void {
	3575	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3576
	3577	var old_path_buffer: [PATH_MAX]u8 = undefined;
	3578	var new_path_buffer: [PATH_MAX]u8 = undefined;
	3579
	3580	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
	3581	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
	3582
	3583	var cancel_region: CancelRegion = .init();
	3584	defer cancel_region.deinit();
	3585	return ev.linkat(
	3586	&cancel_region,
	3587	old_dir.handle,
	3588	old_sub_path_posix,
	3589	new_dir.handle,
	3590	new_sub_path_posix,
	3591	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
	3592	);
	3593	}
	3594
	3595	fn fileStat(userdata: ?*anyopaque, file: File) File.StatError!File.Stat {
	3596	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3597	var cancel_region: CancelRegion = .init();
	3598	defer cancel_region.deinit();
	3599	return ev.stat(&cancel_region, file.handle);
	3600	}
	3601
	3602	fn fileLength(userdata: ?*anyopaque, file: File) File.LengthError!u64 {
	3603	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3604	var cancel_region: CancelRegion = .init();
	3605	defer cancel_region.deinit();
	3606	while (true) {
	3607	var statx_buf = std.mem.zeroes(linux.Statx);
	3608	const thread = try cancel_region.awaitIoUring();
	3609	thread.enqueue().* = .{
	3610	.opcode = .STATX,
	3611	.flags = 0,
	3612	.ioprio = 0,
	3613	.fd = file.handle,
	3614	.off = @intFromPtr(&statx_buf),
	3615	.addr = @intFromPtr(""),
	3616	.len = @bitCast(linux.STATX{ .SIZE = true }),
	3617	.rw_flags = linux.AT.EMPTY_PATH,
	3618	.user_data = @intFromPtr(cancel_region.fiber),
	3619	.buf_index = 0,
	3620	.personality = 0,
	3621	.splice_fd_in = 0,
	3622	.addr3 = 0,
	3623	.resv = 0,
	3624	};
	3625	ev.yield(null, .nothing);
	3626	switch (cancel_region.errno()) {
	3627	.SUCCESS => {
	3628	if (!statx_buf.mask.SIZE) return error.Unexpected;
	3629	return statx_buf.size;
	3630	},
	3631	.INTR, .CANCELED => continue,
	3632	.ACCES => \|err\| return errnoBug(err),
	3633	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	3634	.FAULT => \|err\| return errnoBug(err),
	3635	.INVAL => \|err\| return errnoBug(err),
	3636	.LOOP => \|err\| return errnoBug(err),
	3637	.NAMETOOLONG => \|err\| return errnoBug(err),
	3638	.NOENT => \|err\| return errnoBug(err),
	3639	.NOMEM => return error.SystemResources,
	3640	.NOTDIR => \|err\| return errnoBug(err),
	3641	else => \|err\| return unexpectedErrno(err),
	3642	}
	3643	}
	3644	}
	3645
	3646	fn fileClose(userdata: ?*anyopaque, files: []const File) void {
	3647	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3648	var cancel_region: CancelRegion = .init();
	3649	defer cancel_region.deinit();
	3650	for (files) \|file\| ev.close(file.handle);
	3651	}
	3652
	3653	fn fileWritePositional(
	3654	userdata: ?*anyopaque,
	3655	file: File,
	3656	header: []const u8,
	3657	data: []const []const u8,
	3658	splat: usize,
	3659	offset: u64,
	3660	) File.WritePositionalError!usize {
	3661	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3662
	3663	var iovecs: [max_iovecs_len]iovec_const = undefined;
	3664	var iovlen: iovlen_t = 0;
	3665	addBuf(&iovecs, &iovlen, header);
	3666	for (data[0 .. data.len - 1]) \|bytes\| addBuf(&iovecs, &iovlen, bytes);
	3667	const pattern = data[data.len - 1];
	3668	var backup_buffer: [splat_buffer_size]u8 = undefined;
	3669	if (iovecs.len - iovlen != 0) switch (splat) {
	3670	0 => {},
	3671	1 => addBuf(&iovecs, &iovlen, pattern),
	3672	else => switch (pattern.len) {
	3673	0 => {},
	3674	1 => {
	3675	const splat_buffer = &backup_buffer;
	3676	const memset_len = @min(splat_buffer.len, splat);
	3677	const buf = splat_buffer[0..memset_len];
	3678	@memset(buf, pattern[0]);
	3679	addBuf(&iovecs, &iovlen, buf);
	3680	var remaining_splat = splat - buf.len;
	3681	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0) {
	3682	assert(buf.len == splat_buffer.len);
	3683	addBuf(&iovecs, &iovlen, splat_buffer);
	3684	remaining_splat -= splat_buffer.len;
	3685	}
	3686	addBuf(&iovecs, &iovlen, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
	3687	},
	3688	else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
	3689	addBuf(&iovecs, &iovlen, pattern);
	3690	},
	3691	},
	3692	};
	3693
	3694	var cancel_region: CancelRegion = .init();
	3695	defer cancel_region.deinit();
	3696	return ev.pwritev(&cancel_region, file.handle, iovecs[0..iovlen], offset);
	3697	}
	3698
	3699	/// This is either usize or u32. Since, either is fine, let's use the same
	3700	/// `addBuf` function for both writing to a file and sending network messages.
	3701	const iovlen_t = @FieldType(linux.msghdr_const, "iovlen");
	3702
	3703	fn addBuf(v: []iovec_const, i: *iovlen_t, bytes: []const u8) void {
	3704	// OS checks ptr addr before length so zero length vectors must be omitted.
	3705	if (bytes.len == 0) return;
	3706	if (v.len - i.* == 0) return;
	3707	v[i.*] = .{ .base = bytes.ptr, .len = bytes.len };
	3708	i.* += 1;
	3709	}
	3710
	3711	fn fileWriteFileStreaming(
	3712	userdata: ?*anyopaque,
	3713	file: File,
	3714	header: []const u8,
	3715	file_reader: *File.Reader,
	3716	limit: Io.Limit,
	3717	) File.Writer.WriteFileError!usize {
	3718	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3719	_ = ev;
	3720	_ = file;
	3721	_ = header;
	3722	_ = file_reader;
	3723	_ = limit;
	3724	return error.Unimplemented;
	3725	}
	3726
	3727	fn fileWriteFilePositional(
	3728	userdata: ?*anyopaque,
	3729	file: File,
	3730	header: []const u8,
	3731	file_reader: *File.Reader,
	3732	limit: Io.Limit,
	3733	offset: u64,
	3734	) File.WriteFilePositionalError!usize {
	3735	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3736	_ = ev;
	3737	_ = file;
	3738	_ = header;
	3739	_ = file_reader;
	3740	_ = limit;
	3741	_ = offset;
	3742	return error.Unimplemented;
	3743	}
	3744
	3745	fn fileReadPositional(
	3746	userdata: ?*anyopaque,
	3747	file: File,
	3748	data: []const []u8,
	3749	offset: u64,
	3750	) File.ReadPositionalError!usize {
	3751	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3752
	3753	var iovecs_buffer: [max_iovecs_len]iovec = undefined;
	3754	var i: usize = 0;
	3755	for (data) \|buf\| {
	3756	if (iovecs_buffer.len - i == 0) break;
	3757	if (buf.len > 0) {
	3758	iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len };
	3759	i += 1;
	3760	}
	3761	}
	3762	if (i == 0) return 0;
	3763	const dest = iovecs_buffer[0..i];
	3764	assert(dest[0].len > 0);
	3765
	3766	var cancel_region: CancelRegion = .init();
	3767	defer cancel_region.deinit();
	3768	return ev.preadv(&cancel_region, file.handle, dest, offset) catch \|err\| switch (err) {
	3769	error.SocketUnconnected => return errnoBug(.NOTCONN), // not a socket
	3770	error.ConnectionResetByPeer => return errnoBug(.CONNRESET), // not a socket
	3771	else => \|e\| return e,
	3772	};
	3773	}
	3774
	3775	fn fileSeekBy(userdata: ?*anyopaque, file: File, offset: i64) File.SeekError!void {
	3776	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3777	var sync: CancelRegion.Sync = try .init(ev);
	3778	defer sync.deinit(ev);
	3779	try ev.lseek(&sync, file.handle, @bitCast(offset), linux.SEEK.CUR);
	3780	}
	3781
	3782	fn fileSeekTo(userdata: ?*anyopaque, file: File, offset: u64) File.SeekError!void {
	3783	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3784	var sync: CancelRegion.Sync = try .init(ev);
	3785	defer sync.deinit(ev);
	3786	try ev.lseek(&sync, file.handle, offset, linux.SEEK.SET);
	3787	}
	3788
	3789	fn fileSync(userdata: ?*anyopaque, file: File) File.SyncError!void {
	3790	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3791	var cancel_region: CancelRegion = .init();
	3792	defer cancel_region.deinit();
	3793	while (true) {
	3794	const thread = try cancel_region.awaitIoUring();
	3795	thread.enqueue().* = .{
	3796	.opcode = .FSYNC,
	3797	.flags = 0,
	3798	.ioprio = 0,
	3799	.fd = file.handle,
	3800	.off = 0,
	3801	.addr = 0,
	3802	.len = 0,
	3803	.rw_flags = 0,
	3804	.user_data = @intFromPtr(cancel_region.fiber),
	3805	.buf_index = 0,
	3806	.personality = 0,
	3807	.splice_fd_in = 0,
	3808	.addr3 = 0,
	3809	.resv = 0,
	3810	};
	3811	ev.yield(null, .nothing);
	3812	switch (cancel_region.errno()) {
	3813	.SUCCESS => return,
	3814	.INTR, .CANCELED => continue,
	3815	.BADF => \|err\| return errnoBug(err),
	3816	.INVAL => \|err\| return errnoBug(err),
	3817	.ROFS => \|err\| return errnoBug(err),
	3818	.IO => return error.InputOutput,
	3819	.NOSPC => return error.NoSpaceLeft,
	3820	.DQUOT => return error.DiskQuota,
	3821	else => \|err\| return unexpectedErrno(err),
	3822	}
	3823	}
	3824	}
	3825
	3826	fn fileIsTty(userdata: ?*anyopaque, file: File) Io.Cancelable!bool {
	3827	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3828	var sync: CancelRegion.Sync = try .init(ev);
	3829	defer sync.deinit(ev);
	3830	while (true) {
	3831	try sync.cancel_region.await(.nothing);
	3832	var wsz: winsize = undefined;
	3833	const rc = linux.ioctl(file.handle, linux.T.IOCGWINSZ, @intFromPtr(&wsz));
	3834	switch (linux.errno(rc)) {
	3835	.SUCCESS => return true,
	3836	.INTR => continue,
	3837	else => return false,
	3838	}
	3839	}
	3840	}
	3841
	3842	fn fileEnableAnsiEscapeCodes(userdata: ?*anyopaque, file: File) File.EnableAnsiEscapeCodesError!void {
	3843	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3844	if (!try fileIsTty(ev, file)) return error.NotTerminalDevice;
	3845	}
	3846
	3847	fn fileSetLength(userdata: ?*anyopaque, file: File, length: u64) File.SetLengthError!void {
	3848	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3849	var cancel_region: CancelRegion = .init();
	3850	defer cancel_region.deinit();
	3851	while (true) {
	3852	const thread = try cancel_region.awaitIoUring();
	3853	thread.enqueue().* = .{
	3854	.opcode = .FTRUNCATE,
	3855	.flags = 0,
	3856	.ioprio = 0,
	3857	.fd = file.handle,
	3858	.off = length,
	3859	.addr = 0,
	3860	.len = 0,
	3861	.rw_flags = 0,
	3862	.user_data = @intFromPtr(cancel_region.fiber),
	3863	.buf_index = 0,
	3864	.personality = 0,
	3865	.splice_fd_in = 0,
	3866	.addr3 = 0,
	3867	.resv = 0,
	3868	};
	3869	ev.yield(null, .nothing);
	3870	switch (cancel_region.errno()) {
	3871	.SUCCESS => return,
	3872	.INTR, .CANCELED => continue,
	3873	.FBIG => return error.FileTooBig,
	3874	.IO => return error.InputOutput,
	3875	.PERM => return error.PermissionDenied,
	3876	.TXTBSY => return error.FileBusy,
	3877	.BADF => \|err\| return errnoBug(err), // Handle not open for writing.
	3878	.INVAL => return error.NonResizable, // This is returned for /dev/null for example.
	3879	else => \|err\| return unexpectedErrno(err),
	3880	}
	3881	}
	3882	}
	3883
	3884	fn fileSetOwner(
	3885	userdata: ?*anyopaque,
	3886	file: File,
	3887	owner: ?File.Uid,
	3888	group: ?File.Gid,
	3889	) File.SetOwnerError!void {
	3890	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3891	var sync: CancelRegion.Sync = try .init(ev);
	3892	defer sync.deinit(ev);
	3893	try ev.fchownat(
	3894	&sync,
	3895	file.handle,
	3896	"",
	3897	owner orelse std.math.maxInt(linux.uid_t),
	3898	group orelse std.math.maxInt(linux.gid_t),
	3899	linux.AT.EMPTY_PATH,
	3900	);
	3901	}
	3902
	3903	fn fileSetPermissions(
	3904	userdata: ?*anyopaque,
	3905	file: File,
	3906	permissions: File.Permissions,
	3907	) File.SetPermissionsError!void {
	3908	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3909	var sync: CancelRegion.Sync = try .init(ev);
	3910	defer sync.deinit(ev);
	3911	ev.fchmodat(
	3912	&sync,
	3913	file.handle,
	3914	"",
	3915	permissions.toMode(),
	3916	linux.AT.EMPTY_PATH,
	3917	) catch \|err\| switch (err) {
	3918	error.NameTooLong => return errnoBug(.NAMETOOLONG),
	3919	error.BadPathName => return errnoBug(.ILSEQ),
	3920	error.ProcessFdQuotaExceeded => return errnoBug(.MFILE),
	3921	error.SystemFdQuotaExceeded => return errnoBug(.NFILE),
	3922	error.OperationUnsupported => return errnoBug(.OPNOTSUPP),
	3923	else => \|e\| return e,
	3924	};
	3925	}
	3926
	3927	fn fileSetTimestamps(
	3928	userdata: ?*anyopaque,
	3929	file: File,
	3930	options: File.SetTimestampsOptions,
	3931	) File.SetTimestampsError!void {
	3932	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3933	var sync: CancelRegion.Sync = try .init(ev);
	3934	defer sync.deinit(ev);
	3935	try ev.utimensat(
	3936	&sync,
	3937	file.handle,
	3938	"",
	3939	if (options.modify_timestamp != .now or options.access_timestamp != .now) &.{
	3940	setTimestampToPosix(options.access_timestamp),
	3941	setTimestampToPosix(options.modify_timestamp),
	3942	} else null,
	3943	linux.AT.EMPTY_PATH,
	3944	);
	3945	}
	3946
	3947	fn fileLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!void {
	3948	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3949	var sync: CancelRegion.Sync = try .init(ev);
	3950	defer sync.deinit(ev);
	3951	ev.flock(&sync, file.handle, lock, .blocking) catch \|err\| switch (err) {
	3952	error.WouldBlock => unreachable, // blocking
	3953	else => \|e\| return e,
	3954	};
	3955	}
	3956
	3957	fn fileTryLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!bool {
	3958	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3959	var sync: CancelRegion.Sync = try .init(ev);
	3960	defer sync.deinit(ev);
	3961	ev.flock(&sync, file.handle, lock, switch (lock) {
	3962	.none => .blocking,
	3963	.shared, .exclusive => .nonblocking,
	3964	}) catch \|err\| switch (err) {
	3965	error.WouldBlock => return false,
	3966	else => \|e\| return e,
	3967	};
	3968	return true;
	3969	}
	3970
	3971	fn fileUnlock(userdata: ?*anyopaque, file: File) void {
	3972	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3973	var sync: CancelRegion.Sync = .initBlocked(ev);
	3974	defer sync.deinit(ev);
	3975	ev.flock(&sync, file.handle, .none, .blocking) catch \|err\| switch (err) {
	3976	error.Canceled => unreachable, // blocked
	3977	error.WouldBlock => unreachable, // blocking
	3978	error.SystemResources => return recoverableOsBugDetected(), // Resource deallocation.
	3979	error.FileLocksUnsupported => return recoverableOsBugDetected(), // We already got the lock.
	3980	error.Unexpected => return recoverableOsBugDetected(), // Resource deallocation must succeed.
	3981	};
	3982	}
	3983
	3984	fn fileDowngradeLock(userdata: ?*anyopaque, file: File) File.DowngradeLockError!void {
	3985	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3986	var sync: CancelRegion.Sync = try .init(ev);
	3987	defer sync.deinit(ev);
	3988	ev.flock(&sync, file.handle, .shared, .nonblocking) catch \|err\| switch (err) {
	3989	error.WouldBlock => return errnoBug(.AGAIN), // File was not locked in exclusive mode.
	3990	error.SystemResources => return errnoBug(.NOLCK), // Lock already obtained.
	3991	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Lock already obtained.
	3992	else => \|e\| return e,
	3993	};
	3994	}
	3995
	3996	fn fileRealPath(userdata: ?*anyopaque, file: File, out_buffer: []u8) File.RealPathError!usize {
	3997	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3998	var sync: CancelRegion.Sync = try .init(ev);
	3999	defer sync.deinit(ev);
	4000	return ev.realPath(&sync, file.handle, out_buffer);
	4001	}
	4002
	4003	fn fileHardLink(
	4004	userdata: ?*anyopaque,
	4005	file: File,
	4006	new_dir: Dir,
	4007	new_sub_path: []const u8,
	4008	options: File.HardLinkOptions,
	4009	) File.HardLinkError!void {
	4010	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4011
	4012	var new_path_buffer: [PATH_MAX]u8 = undefined;
	4013	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
	4014
	4015	var cancel_region: CancelRegion = .init();
	4016	defer cancel_region.deinit();
	4017	return ev.linkat(
	4018	&cancel_region,
	4019	file.handle,
	4020	"",
	4021	new_dir.handle,
	4022	new_sub_path_posix,
	4023	linux.AT.EMPTY_PATH \| @as(u32, if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW),
	4024	);
	4025	}
	4026
	4027	fn fileMemoryMapCreate(
	4028	userdata: ?*anyopaque,
	4029	file: File,
	4030	options: File.MemoryMap.CreateOptions,
	4031	) File.MemoryMap.CreateError!File.MemoryMap {
	4032	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4033
	4034	const prot: linux.PROT = .{
	4035	.READ = options.protection.read,
	4036	.WRITE = options.protection.write,
	4037	.EXEC = options.protection.execute,
	4038	};
	4039	const flags: linux.MAP = .{
	4040	.TYPE = .SHARED_VALIDATE,
	4041	.POPULATE = options.populate,
	4042	};
	4043
	4044	const page_align = std.heap.page_size_min;
	4045
	4046	var sync: CancelRegion.Sync = try .init(ev);
	4047	defer sync.deinit(ev);
	4048	const contents = while (true) {
	4049	try sync.cancel_region.await(.nothing);
	4050	const casted_offset = std.math.cast(i64, options.offset) orelse return error.Unseekable;
	4051	const rc = linux.mmap(null, options.len, prot, flags, file.handle, casted_offset);
	4052	switch (linux.errno(rc)) {
	4053	.SUCCESS => break @as([*]align(page_align) u8, @ptrFromInt(rc))[0..options.len],
	4054	.INTR => continue,
	4055	.ACCES => return error.AccessDenied,
	4056	.AGAIN => return error.LockedMemoryLimitExceeded,
	4057	.MFILE => return error.ProcessFdQuotaExceeded,
	4058	.NFILE => return error.SystemFdQuotaExceeded,
	4059	.NOMEM => return error.OutOfMemory,
	4060	.PERM => return error.PermissionDenied,
	4061	.OVERFLOW => return error.Unseekable,
	4062	.BADF => \|err\| return errnoBug(err), // Always a race condition.
	4063	.INVAL => \|err\| return errnoBug(err), // Invalid parameters to mmap()
	4064	.OPNOTSUPP => \|err\| return errnoBug(err), // Bad flags with MAP.SHARED_VALIDATE on Linux.
	4065	else => \|err\| return unexpectedErrno(err),
	4066	}
	4067	};
	4068	return .{
	4069	.file = file,
	4070	.offset = options.offset,
	4071	.memory = contents,
	4072	.section = {},
	4073	};
	4074	}
	4075
	4076	fn fileMemoryMapDestroy(userdata: ?anyopaque, mm: File.MemoryMap) void {
	4077	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4078	_ = ev;
	4079	const memory = mm.memory;
	4080	if (memory.len == 0) return;
	4081	switch (linux.errno(linux.munmap(memory.ptr, memory.len))) {
	4082	.SUCCESS => {},
	4083	else => \|err\| if (builtin.mode == .Debug)
	4084	std.log.err("failed to unmap {d} bytes at {*}: {t}", .{ memory.len, memory.ptr, err }),
	4085	}
	4086	mm.* = undefined;
	4087	}
	4088
	4089	fn fileMemoryMapSetLength(
	4090	userdata: ?*anyopaque,
	4091	mm: *File.MemoryMap,
	4092	new_len: usize,
	4093	) File.MemoryMap.SetLengthError!void {
	4094	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4095
	4096	const page_size = std.heap.pageSize();
	4097	const alignment: Alignment = .fromByteUnits(page_size);
	4098	const page_align = std.heap.page_size_min;
	4099	const old_memory = mm.memory;
	4100
	4101	if (alignment.forward(new_len) == alignment.forward(old_memory.len)) {
	4102	mm.memory.len = new_len;
	4103	return;
	4104	}
	4105	const flags: linux.MREMAP = .{ .MAYMOVE = true };
	4106	const addr_hint: ?[*]const u8 = null;
	4107	var sync: CancelRegion.Sync = try .init(ev);
	4108	defer sync.deinit(ev);
	4109	const new_memory = while (true) {
	4110	try sync.cancel_region.await(.nothing);
	4111	const rc = linux.mremap(old_memory.ptr, old_memory.len, new_len, flags, addr_hint);
	4112	switch (linux.errno(rc)) {
	4113	.SUCCESS => break @as([*]align(page_align) u8, @ptrFromInt(rc))[0..new_len],
	4114	.INTR => continue,
	4115	.AGAIN => return error.LockedMemoryLimitExceeded,
	4116	.NOMEM => return error.OutOfMemory,
	4117	.INVAL => \|err\| return errnoBug(err),
	4118	.FAULT => \|err\| return errnoBug(err),
	4119	else => \|err\| return unexpectedErrno(err),
	4120	}
	4121	};
	4122	mm.memory = new_memory;
	4123	}
	4124
	4125	fn fileMemoryMapRead(userdata: ?anyopaque, mm: File.MemoryMap) File.ReadPositionalError!void {
	4126	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4127	_ = ev;
	4128	_ = mm;
	4129	}
	4130
	4131	fn fileMemoryMapWrite(userdata: ?anyopaque, mm: File.MemoryMap) File.WritePositionalError!void {
	4132	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4133	_ = ev;
	4134	_ = mm;
	4135	}
	4136
	4137	fn processExecutableOpen(
	4138	userdata: ?*anyopaque,
	4139	flags: File.OpenFlags,
	4140	) process.OpenExecutableError!File {
	4141	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4142	return dirOpenFile(ev, .{ .handle = linux.AT.FDCWD }, "/proc/self/exe", flags);
	4143	}
	4144
	4145	fn processExecutablePath(userdata: ?*anyopaque, out_buffer: []u8) process.ExecutablePathError!usize {
	4146	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4147	return dirReadLink(ev, .cwd(), "/proc/self/exe", out_buffer) catch \|err\| switch (err) {
	4148	error.UnsupportedReparsePointType => unreachable, // Windows-only
	4149	error.NetworkNotFound => unreachable, // Windows-only
	4150	error.FileBusy => unreachable, // Windows-only
	4151	else => \|e\| return e,
	4152	};
	4153	}
	4154
	4155	fn lockStderr(userdata: ?*anyopaque, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
	4156	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4157	const ev_io = ev.io();
	4158	ev.stderr_mutex.lockUncancelable(ev_io);
	4159	errdefer ev.stderr_mutex.unlock(ev_io);
	4160	return ev.initLockedStderr(terminal_mode);
	4161	}
	4162
	4163	fn tryLockStderr(
	4164	userdata: ?*anyopaque,
	4165	terminal_mode: ?Io.Terminal.Mode,
	4166	) Io.Cancelable!?Io.LockedStderr {
	4167	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4168	const ev_io = ev.io();
	4169	if (!ev.stderr_mutex.tryLock()) return null;
	4170	errdefer ev.stderr_mutex.unlock(ev_io);
	4171	return try ev.initLockedStderr(terminal_mode);
	4172	}
	4173
	4174	fn initLockedStderr(ev: *Evented, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
	4175	if (!ev.stderr_writer_initialized) {
	4176	const ev_io = ev.io();
	4177	const cancel_protection = swapCancelProtection(ev, .blocked);
	4178	defer assert(swapCancelProtection(ev, cancel_protection) == .blocked);
	4179	ev.scanEnviron() catch \|err\| switch (err) {
	4180	error.Canceled => unreachable, // blocked
	4181	};
	4182	const NO_COLOR = ev.environ.exist.NO_COLOR;
	4183	const CLICOLOR_FORCE = ev.environ.exist.CLICOLOR_FORCE;
	4184	ev.stderr_mode = Io.Terminal.Mode.detect(
	4185	ev_io,
	4186	ev.stderr_writer.file,
	4187	NO_COLOR,
	4188	CLICOLOR_FORCE,
	4189	) catch \|err\| switch (err) {
	4190	error.Canceled => unreachable, // blocked
	4191	};
	4192	ev.stderr_writer_initialized = true;
	4193	}
	4194	return .{
	4195	.file_writer = &ev.stderr_writer,
	4196	.terminal_mode = terminal_mode orelse ev.stderr_mode,
	4197	};
	4198	}
	4199
	4200	fn unlockStderr(userdata: ?*anyopaque) void {
	4201	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4202	if (ev.stderr_writer.err == null) ev.stderr_writer.interface.flush() catch {};
	4203	if (ev.stderr_writer.err) \|err\| {
	4204	switch (err) {
	4205	error.Canceled => Thread.current().currentFiber().cancel_protection.recancel(),
	4206	else => {},
	4207	}
	4208	ev.stderr_writer.err = null;
	4209	}
	4210	ev.stderr_writer.interface.end = 0;
	4211	ev.stderr_writer.interface.buffer = &.{};
	4212	ev.stderr_mutex.unlock(ev.io());
	4213	}
	4214
	4215	fn processCurrentPath(userdata: ?*anyopaque, buffer: []u8) process.CurrentPathError!usize {
	4216	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4217	var sync: CancelRegion.Sync = try .init(ev);
	4218	defer sync.deinit(ev);
	4219	while (true) {
	4220	try sync.cancel_region.await(.nothing);
	4221	switch (linux.errno(linux.getcwd(buffer.ptr, buffer.len))) {
	4222	.SUCCESS => return std.mem.findScalar(u8, buffer, 0).?,
	4223	.INTR => continue,
	4224	.NOENT => return error.CurrentDirUnlinked,
	4225	.RANGE => return error.NameTooLong,
	4226	.FAULT => \|err\| return errnoBug(err),
	4227	.INVAL => \|err\| return errnoBug(err),
	4228	else => \|err\| return unexpectedErrno(err),
	4229	}
	4230	}
	4231	}
	4232
	4233	fn processSetCurrentDir(userdata: ?*anyopaque, dir: Dir) process.SetCurrentDirError!void {
	4234	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4235	if (dir.handle == linux.AT.FDCWD) return;
	4236	var sync: CancelRegion.Sync = try .init(ev);
	4237	defer sync.deinit(ev);
	4238	return ev.fchdir(&sync, dir.handle);
	4239	}
	4240
	4241	fn processSetCurrentPath(userdata: ?*anyopaque, dir_path: []const u8) ChdirError!void {
	4242	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4243	var path_buffer: [PATH_MAX]u8 = undefined;
	4244	const dir_path_posix = try pathToPosix(dir_path, &path_buffer);
	4245	var sync: CancelRegion.Sync = try .init(ev);
	4246	defer sync.deinit(ev);
	4247	return ev.chdir(&sync, dir_path_posix);
	4248	}
	4249
	4250	fn processReplace(userdata: ?*anyopaque, options: process.ReplaceOptions) process.ReplaceError {
	4251	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4252
	4253	try ev.scanEnviron(); // for PATH
	4254	const PATH = ev.environ.string.PATH orelse default_PATH;
	4255
	4256	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
	4257	defer arena_allocator.deinit();
	4258	const arena = arena_allocator.allocator();
	4259
	4260	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
	4261	for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
	4262
	4263	const env_block = env_block: {
	4264	const prog_fd: i32 = -1;
	4265	if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
	4266	.zig_progress_fd = prog_fd,
	4267	});
	4268	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
	4269	.zig_progress_fd = prog_fd,
	4270	});
	4271	};
	4272
	4273	var sync: CancelRegion.Sync = try .init(ev);
	4274	defer sync.deinit(ev);
	4275	return ev.execv(&sync, options.expand_arg0, argv_buf.ptr[0].?, argv_buf.ptr, env_block, PATH);
	4276	}
	4277
	4278	fn processReplacePath(
	4279	userdata: ?*anyopaque,
	4280	dir: Dir,
	4281	options: process.ReplaceOptions,
	4282	) process.ReplaceError {
	4283	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4284	_ = ev;
	4285	_ = dir;
	4286	_ = options;
	4287	@panic("TODO processReplacePath");
	4288	}
	4289
	4290	fn processSpawn(userdata: ?*anyopaque, options: process.SpawnOptions) process.SpawnError!process.Child {
	4291	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4292	const spawned = try ev.spawn(options);
	4293	var cancel_region: CancelRegion = .initBlocked();
	4294	defer cancel_region.deinit();
	4295	defer ev.close(spawned.err_fd);
	4296
	4297	// Wait for the child to report any errors in or before `execvpe`.
	4298	var child_err: ForkBailError = undefined;
	4299	ev.readAll(&cancel_region, spawned.err_fd, @ptrCast(&child_err)) catch \|read_err\| {
	4300	switch (read_err) {
	4301	error.Canceled => unreachable, // blocked
	4302	error.EndOfStream => {
	4303	// Write end closed by CLOEXEC at the time of the `execvpe` call,
	4304	// indicating success.
	4305	},
	4306	else => {
	4307	// Problem reading the error from the error reporting pipe. We
	4308	// don't know if the child is alive or dead. Better to assume it is
	4309	// alive so the resource does not risk being leaked.
	4310	},
	4311	}
	4312	return .{
	4313	.id = spawned.pid,
	4314	.thread_handle = {},
	4315	.stdin = spawned.stdin,
	4316	.stdout = spawned.stdout,
	4317	.stderr = spawned.stderr,
	4318	.request_resource_usage_statistics = options.request_resource_usage_statistics,
	4319	};
	4320	};
	4321	return child_err;
	4322	}
	4323
	4324	fn processSpawnPath(
	4325	userdata: ?*anyopaque,
	4326	dir: Dir,
	4327	options: process.SpawnOptions,
	4328	) process.SpawnError!process.Child {
	4329	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4330	_ = ev;
	4331	_ = dir;
	4332	_ = options;
	4333	@panic("TODO processSpawnPath");
	4334	}
	4335
	4336	const prog_fileno = @max(linux.STDIN_FILENO, linux.STDOUT_FILENO, linux.STDERR_FILENO);
	4337
	4338	const Spawned = struct {
	4339	pid: pid_t,
	4340	err_fd: fd_t,
	4341	stdin: ?File,
	4342	stdout: ?File,
	4343	stderr: ?File,
	4344	};
	4345	fn spawn(ev: *Evented, options: process.SpawnOptions) process.SpawnError!Spawned {
	4346	var cancel_region: CancelRegion = .init();
	4347	defer cancel_region.deinit();
	4348
	4349	// The child process does need to access (one end of) these pipes. However,
	4350	// we must initially set CLOEXEC to avoid a race condition. If another thread
	4351	// is racing to spawn a different child process, we don't want it to inherit
	4352	// these FDs in any scenario; that would mean that, for instance, calls to
	4353	// `poll` from the parent would not report the child's stdout as closing when
	4354	// expected, since the other child may retain a reference to the write end of
	4355	// the pipe. So, we create the pipes with CLOEXEC initially. After fork, we
	4356	// need to do something in the new child to make sure we preserve the reference
	4357	// we want. We could use `fcntl` to remove CLOEXEC from the FD, but as it
	4358	// turns out, we `dup2` everything anyway, so there's no need!
	4359	const pipe_flags: linux.O = .{ .CLOEXEC = true };
	4360
	4361	const stdin_pipe = if (options.stdin == .pipe) try pipe2(pipe_flags) else undefined;
	4362	errdefer if (options.stdin == .pipe) {
	4363	ev.destroyPipe(stdin_pipe);
	4364	};
	4365
	4366	const stdout_pipe = if (options.stdout == .pipe) try pipe2(pipe_flags) else undefined;
	4367	errdefer if (options.stdout == .pipe) {
	4368	ev.destroyPipe(stdout_pipe);
	4369	};
	4370
	4371	const stderr_pipe = if (options.stderr == .pipe) try pipe2(pipe_flags) else undefined;
	4372	errdefer if (options.stderr == .pipe) {
	4373	ev.destroyPipe(stderr_pipe);
	4374	};
	4375
	4376	const any_ignore =
	4377	options.stdin == .ignore or options.stdout == .ignore or options.stderr == .ignore;
	4378	const dev_null_fd = if (any_ignore) try ev.null_fd.open(ev, &cancel_region, "/dev/null", .{
	4379	.ACCMODE = .RDWR,
	4380	}) else undefined;
	4381
	4382	const prog_pipe: [2]fd_t = if (options.progress_node.index != .none) pipe: {
	4383	// We use CLOEXEC for the same reason as in `pipe_flags`.
	4384	const pipe = try pipe2(.{ .NONBLOCK = true, .CLOEXEC = true });
	4385	_ = linux.fcntl(pipe[0], linux.F.SETPIPE_SZ, @as(u32, std.Progress.max_packet_len * 2));
	4386	break :pipe pipe;
	4387	} else .{ -1, -1 };
	4388	errdefer ev.destroyPipe(prog_pipe);
	4389
	4390	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
	4391	defer arena_allocator.deinit();
	4392	const arena = arena_allocator.allocator();
	4393
	4394	// The POSIX standard does not allow malloc() between fork() and execve(),
	4395	// and this allocator may be a libc allocator.
	4396	// I have personally observed the child process deadlocking when it tries
	4397	// to call malloc() due to a heap allocation between fork() and execve(),
	4398	// in musl v1.1.24.
	4399	// Additionally, we want to reduce the number of possible ways things
	4400	// can fail between fork() and execve().
	4401	// Therefore, we do all the allocation for the execve() before the fork().
	4402	// This means we must do the null-termination of argv and env vars here.
	4403	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
	4404	for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
	4405
	4406	const env_block = env_block: {
	4407	const prog_fd: i32 = if (prog_pipe[1] == -1) -1 else prog_fileno;
	4408	if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
	4409	.zig_progress_fd = prog_fd,
	4410	});
	4411	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
	4412	.zig_progress_fd = prog_fd,
	4413	});
	4414	};
	4415
	4416	// This pipe communicates to the parent errors in the child between `fork` and `execvpe`.
	4417	// It is closed by the child (via CLOEXEC) without writing if `execvpe` succeeds.
	4418	const err_pipe: [2]fd_t = try pipe2(.{ .CLOEXEC = true });
	4419	errdefer ev.destroyPipe(err_pipe);
	4420
	4421	try ev.scanEnviron(); // for PATH
	4422	const PATH = ev.environ.string.PATH orelse default_PATH;
	4423
	4424	const pid_result: pid_t = fork: {
	4425	const rc = linux.fork();
	4426	switch (linux.errno(rc)) {
	4427	.SUCCESS => break :fork @intCast(rc),
	4428	.AGAIN => return error.SystemResources,
	4429	.NOMEM => return error.SystemResources,
	4430	.NOSYS => return error.OperationUnsupported,
	4431	else => \|err\| return unexpectedErrno(err),
	4432	}
	4433	};
	4434
	4435	if (pid_result == 0) {
	4436	defer comptime unreachable; // We are the child.
	4437	var sync: CancelRegion.Sync = .{ .cancel_region = .initBlocked() };
	4438	const err = ev.setUpChild(&sync, .{
	4439	.stdin_pipe = stdin_pipe[0],
	4440	.stdout_pipe = stdout_pipe[1],
	4441	.stderr_pipe = stderr_pipe[1],
	4442	.dev_null_fd = dev_null_fd,
	4443	.prog_pipe = prog_pipe[1],
	4444	.argv_buf = argv_buf,
	4445	.env_block = env_block,
	4446	.PATH = PATH,
	4447	.spawn = options,
	4448	});
	4449	ev.writeAll(&sync.cancel_region, err_pipe[1], @ptrCast(&err)) catch {};
	4450	const exit = if (builtin.single_threaded) linux.exit else linux.exit_group;
	4451	exit(1);
	4452	}
	4453
	4454	const pid: pid_t = @intCast(pid_result); // We are the parent.
	4455	errdefer comptime unreachable; // The child is forked; we must not error from now on
	4456
	4457	ev.close(err_pipe[1]); // make sure only the child holds the write end open
	4458
	4459	if (options.stdin == .pipe) ev.close(stdin_pipe[0]);
	4460	if (options.stdout == .pipe) ev.close(stdout_pipe[1]);
	4461	if (options.stderr == .pipe) ev.close(stderr_pipe[1]);
	4462
	4463	if (prog_pipe[1] != -1) ev.close(prog_pipe[1]);
	4464
	4465	options.progress_node.setIpcFile(ev, .{ .handle = prog_pipe[0], .flags = .{ .nonblocking = true } });
	4466
	4467	return .{
	4468	.pid = pid,
	4469	.err_fd = err_pipe[0],
	4470	.stdin = switch (options.stdin) {
	4471	.pipe => .{ .handle = stdin_pipe[1], .flags = .{ .nonblocking = false } },
	4472	else => null,
	4473	},
	4474	.stdout = switch (options.stdout) {
	4475	.pipe => .{ .handle = stdout_pipe[0], .flags = .{ .nonblocking = false } },
	4476	else => null,
	4477	},
	4478	.stderr = switch (options.stderr) {
	4479	.pipe => .{ .handle = stderr_pipe[0], .flags = .{ .nonblocking = false } },
	4480	else => null,
	4481	},
	4482	};
	4483	}
	4484
	4485	pub const PipeError = error{
	4486	SystemFdQuotaExceeded,
	4487	ProcessFdQuotaExceeded,
	4488	} \|\| Io.UnexpectedError;
	4489	pub fn pipe2(flags: linux.O) PipeError![2]fd_t {
	4490	var fds: [2]fd_t = undefined;
	4491	switch (linux.errno(linux.pipe2(&fds, flags))) {
	4492	.SUCCESS => return fds,
	4493	.INVAL => \|err\| return errnoBug(err), // Invalid flags
	4494	.NFILE => return error.SystemFdQuotaExceeded,
	4495	.MFILE => return error.ProcessFdQuotaExceeded,
	4496	else => \|err\| return unexpectedErrno(err),
	4497	}
	4498	}
	4499	fn destroyPipe(ev: *Evented, pipe: [2]fd_t) void {
	4500	if (pipe[0] != -1) ev.close(pipe[0]);
	4501	if (pipe[0] != pipe[1]) ev.close(pipe[1]);
	4502	}
	4503
	4504	/// Errors that can occur between fork() and execv()
	4505	const ForkBailError = process.SetCurrentDirError \|\| ChdirError \|\|
	4506	process.SpawnError \|\| process.ReplaceError;
	4507	fn setUpChild(ev: Evented, sync: CancelRegion.Sync, options: struct {
	4508	stdin_pipe: fd_t,
	4509	stdout_pipe: fd_t,
	4510	stderr_pipe: fd_t,
	4511	dev_null_fd: fd_t,
	4512	prog_pipe: fd_t,
	4513	argv_buf: [:null]?[*:0]const u8,
	4514	env_block: process.Environ.Block,
	4515	PATH: []const u8,
	4516	spawn: process.SpawnOptions,
	4517	}) ForkBailError {
	4518	try ev.setUpChildIo(
	4519	sync,
	4520	options.spawn.stdin,
	4521	options.stdin_pipe,
	4522	linux.STDIN_FILENO,
	4523	options.dev_null_fd,
	4524	);
	4525	try ev.setUpChildIo(
	4526	sync,
	4527	options.spawn.stdout,
	4528	options.stdout_pipe,
	4529	linux.STDOUT_FILENO,
	4530	options.dev_null_fd,
	4531	);
	4532	try ev.setUpChildIo(
	4533	sync,
	4534	options.spawn.stderr,
	4535	options.stderr_pipe,
	4536	linux.STDERR_FILENO,
	4537	options.dev_null_fd,
	4538	);
	4539
	4540	switch (options.spawn.cwd) {
	4541	.inherit => {},
	4542	.dir => \|cwd_dir\| try ev.fchdir(sync, cwd_dir.handle),
	4543	.path => \|cwd_path\| {
	4544	var cwd_path_buffer: [PATH_MAX]u8 = undefined;
	4545	const cwd_path_posix = try pathToPosix(cwd_path, &cwd_path_buffer);
	4546	try ev.chdir(sync, cwd_path_posix);
	4547	},
	4548	}
	4549
	4550	// Must happen after fchdir above, the cwd file descriptor might be
	4551	// equal to prog_fileno and be clobbered by this dup2 call.
	4552	if (options.prog_pipe != -1) try ev.dup2(sync, options.prog_pipe, prog_fileno);
	4553
	4554	if (options.spawn.gid) \|gid\| {
	4555	switch (linux.errno(linux.setregid(gid, gid))) {
	4556	.SUCCESS => {},
	4557	.AGAIN => return error.ResourceLimitReached,
	4558	.INVAL => return error.InvalidUserId,
	4559	.PERM => return error.PermissionDenied,
	4560	else => return error.Unexpected,
	4561	}
	4562	}
	4563
	4564	if (options.spawn.uid) \|uid\| {
	4565	switch (linux.errno(linux.setreuid(uid, uid))) {
	4566	.SUCCESS => {},
	4567	.AGAIN => return error.ResourceLimitReached,
	4568	.INVAL => return error.InvalidUserId,
	4569	.PERM => return error.PermissionDenied,
	4570	else => return error.Unexpected,
	4571	}
	4572	}
	4573
	4574	if (options.spawn.pgid) \|pid\| {
	4575	switch (linux.errno(linux.setpgid(0, pid))) {
	4576	.SUCCESS => {},
	4577	.ACCES => return error.ProcessAlreadyExec,
	4578	.INVAL => return error.InvalidProcessGroupId,
	4579	.PERM => return error.PermissionDenied,
	4580	else => return error.Unexpected,
	4581	}
	4582	}
	4583
	4584	if (options.spawn.start_suspended) {
	4585	switch (linux.errno(linux.kill(0, .STOP))) {
	4586	.SUCCESS => {},
	4587	.PERM => return error.PermissionDenied,
	4588	else => return error.Unexpected,
	4589	}
	4590	}
	4591
	4592	return ev.execv(
	4593	sync,
	4594	options.spawn.expand_arg0,
	4595	options.argv_buf.ptr[0].?,
	4596	options.argv_buf.ptr,
	4597	options.env_block,
	4598	options.PATH,
	4599	);
	4600	}
	4601
	4602	fn setUpChildIo(
	4603	ev: *Evented,
	4604	sync: *CancelRegion.Sync,
	4605	stdio: process.SpawnOptions.StdIo,
	4606	pipe_fd: fd_t,
	4607	std_fileno: i32,
	4608	dev_null_fd: fd_t,
	4609	) !void {
	4610	switch (stdio) {
	4611	.pipe => try ev.dup2(sync, pipe_fd, std_fileno),
	4612	.close => _ = linux.close(std_fileno),
	4613	.inherit => {},
	4614	.ignore => try ev.dup2(sync, dev_null_fd, std_fileno),
	4615	.file => \|file\| {
	4616	if (file.flags.nonblocking) @panic("TODO implement setUpChildIo when nonblocking file is used");
	4617	try ev.dup2(sync, file.handle, std_fileno);
	4618	},
	4619	}
	4620	}
	4621
	4622	pub const DupError = error{
	4623	ProcessFdQuotaExceeded,
	4624	SystemResources,
	4625	} \|\| Io.UnexpectedError \|\| Io.Cancelable;
	4626	pub fn dup2(ev: Evented, sync: CancelRegion.Sync, old_fd: fd_t, new_fd: fd_t) DupError!void {
	4627	_ = ev;
	4628	while (true) {
	4629	try sync.cancel_region.await(.nothing);
	4630	switch (linux.errno(linux.dup2(old_fd, new_fd))) {
	4631	.SUCCESS => {},
	4632	.BUSY, .INTR => continue,
	4633	.INVAL => \|err\| return errnoBug(err), // invalid parameters
	4634	.BADF => \|err\| return errnoBug(err), // use after free
	4635	.MFILE => return error.ProcessFdQuotaExceeded,
	4636	.NOMEM => return error.SystemResources,
	4637	else => \|err\| return unexpectedErrno(err),
	4638	}
	4639	}
	4640	}
	4641
	4642	fn execv(
	4643	ev: *Evented,
	4644	sync: *CancelRegion.Sync,
	4645	arg0_expand: process.ArgExpansion,
	4646	file: [*:0]const u8,
	4647	child_argv: [:null]?[:0]const u8,
	4648	env_block: process.Environ.PosixBlock,
	4649	PATH: []const u8,
	4650	) process.ReplaceError {
	4651	const file_slice = std.mem.sliceTo(file, 0);
	4652	if (std.mem.findScalar(u8, file_slice, '/') != null) return ev.execvPath(sync, file, child_argv, env_block);
	4653
	4654	// Use of PATH_MAX here is valid as the path_buf will be passed
	4655	// directly to the operating system in posixExecvPath.
	4656	var path_buf: [PATH_MAX]u8 = undefined;
	4657	var it = std.mem.tokenizeScalar(u8, PATH, ':');
	4658	var seen_eacces = false;
	4659	var err: process.ReplaceError = error.FileNotFound;
	4660
	4661	// In case of expanding arg0 we must put it back if we return with an error.
	4662	const prev_arg0 = child_argv[0];
	4663	defer switch (arg0_expand) {
	4664	.expand => child_argv[0] = prev_arg0,
	4665	.no_expand => {},
	4666	};
	4667
	4668	while (it.next()) \|search_path\| {
	4669	const path_len = search_path.len + file_slice.len + 1;
	4670	if (path_buf.len < path_len + 1) return error.NameTooLong;
	4671	@memcpy(path_buf[0..search_path.len], search_path);
	4672	path_buf[search_path.len] = '/';
	4673	@memcpy(path_buf[search_path.len + 1 ..][0..file_slice.len], file_slice);
	4674	path_buf[path_len] = 0;
	4675	const full_path = path_buf[0..path_len :0].ptr;
	4676	switch (arg0_expand) {
	4677	.expand => child_argv[0] = full_path,
	4678	.no_expand => {},
	4679	}
	4680	err = ev.execvPath(sync, full_path, child_argv, env_block);
	4681	switch (err) {
	4682	error.AccessDenied => seen_eacces = true,
	4683	error.FileNotFound, error.NotDir => {},
	4684	else => \|e\| return e,
	4685	}
	4686	}
	4687	if (seen_eacces) return error.AccessDenied;
	4688	return err;
	4689	}
	4690	/// This function ignores PATH environment variable.
	4691	pub fn execvPath(
	4692	ev: *Evented,
	4693	sync: *CancelRegion.Sync,
	4694	path: [*:0]const u8,
	4695	child_argv: [:null]const ?[:0]const u8,
	4696	env_block: process.Environ.PosixBlock,
	4697	) process.ReplaceError {
	4698	_ = ev;
	4699	try sync.cancel_region.await(.nothing);
	4700	switch (linux.errno(linux.execve(path, child_argv, env_block.slice.ptr))) {
	4701	.FAULT => \|err\| return errnoBug(err), // Bad pointer parameter.
	4702	.@"2BIG" => return error.SystemResources,
	4703	.MFILE => return error.ProcessFdQuotaExceeded,
	4704	.NAMETOOLONG => return error.NameTooLong,
	4705	.NFILE => return error.SystemFdQuotaExceeded,
	4706	.NOMEM => return error.SystemResources,
	4707	.ACCES => return error.AccessDenied,
	4708	.PERM => return error.PermissionDenied,
	4709	.INVAL => return error.InvalidExe,
	4710	.NOEXEC => return error.InvalidExe,
	4711	.IO => return error.FileSystem,
	4712	.LOOP => return error.FileSystem,
	4713	.ISDIR => return error.IsDir,
	4714	.NOENT => return error.FileNotFound,
	4715	.NOTDIR => return error.NotDir,
	4716	.TXTBSY => return error.FileBusy,
	4717	.LIBBAD => return error.InvalidExe,
	4718	else => \|err\| return unexpectedErrno(err),
	4719	}
	4720	}
	4721
	4722	fn childWait(userdata: ?anyopaque, child: process.Child) process.Child.WaitError!process.Child.Term {
	4723	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4724
	4725	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
	4726	defer maybe_sync.deinit(ev);
	4727	defer ev.childCleanup(child);
	4728
	4729	const pid = child.id.?;
	4730	var info: linux.siginfo_t = undefined;
	4731	while (true) {
	4732	const thread = try maybe_sync.cancel_region.awaitIoUring();
	4733	thread.enqueue().* = .{
	4734	.opcode = .WAITID,
	4735	.flags = 0,
	4736	.ioprio = 0,
	4737	.fd = pid,
	4738	.off = @intFromPtr(&info),
	4739	.addr = 0,
	4740	.len = @intFromEnum(linux.P.PID),
	4741	.rw_flags = 0,
	4742	.user_data = @intFromPtr(maybe_sync.cancel_region.fiber),
	4743	.buf_index = 0,
	4744	.personality = 0,
	4745	.splice_fd_in = linux.W.EXITED \|
	4746	@as(i32, if (child.request_resource_usage_statistics) linux.W.NOWAIT else 0),
	4747	.addr3 = 0,
	4748	.resv = 0,
	4749	};
	4750	ev.yield(null, .nothing);
	4751	switch (maybe_sync.cancel_region.errno()) {
	4752	.SUCCESS => {
	4753	if (child.request_resource_usage_statistics) {
	4754	const sync = try maybe_sync.enterSync(ev);
	4755	while (true) {
	4756	try sync.cancel_region.await(.nothing);
	4757	var rusage: linux.rusage = undefined;
	4758	switch (linux.errno(linux.waitid(
	4759	.PID,
	4760	pid,
	4761	&info,
	4762	linux.W.EXITED \| linux.W.NOHANG,
	4763	&rusage,
	4764	))) {
	4765	.SUCCESS => {
	4766	child.resource_usage_statistics.rusage = rusage;
	4767	break;
	4768	},
	4769	.INTR, .CANCELED => continue,
	4770	.CHILD => \|err\| return errnoBug(err), // Double-free.
	4771	else => \|err\| return unexpectedErrno(err),
	4772	}
	4773	}
	4774	}
	4775	const status: u32 = @bitCast(info.fields.common.second.sigchld.status);
	4776	const code: linux.CLD = @enumFromInt(info.code);
	4777	return switch (code) {
	4778	.EXITED => .{ .exited = @truncate(status) },
	4779	.KILLED, .DUMPED => .{ .signal = @enumFromInt(status) },
	4780	.TRAPPED, .STOPPED => .{ .stopped = status },
	4781	_, .CONTINUED => .{ .unknown = status },
	4782	};
	4783	},
	4784	.INTR, .CANCELED => continue,
	4785	.CHILD => \|err\| return errnoBug(err), // Double-free.
	4786	else => \|err\| return unexpectedErrno(err),
	4787	}
	4788	}
	4789	}
	4790
	4791	fn childKill(userdata: ?anyopaque, child: process.Child) void {
	4792	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4793
	4794	var maybe_sync: CancelRegion.Sync.Maybe = .{ .sync = .initBlocked(ev) };
	4795	defer maybe_sync.deinit(ev);
	4796	defer ev.childCleanup(child);
	4797
	4798	const pid = child.id.?;
	4799	while (true) switch (linux.errno(linux.kill(pid, .TERM))) {
	4800	.SUCCESS => break,
	4801	.INTR => continue,
	4802	.PERM => return,
	4803	.INVAL => \|err\| return errnoBug(err) catch {},
	4804	.SRCH => \|err\| return errnoBug(err) catch {},
	4805	else => \|err\| return unexpectedErrno(err) catch {},
	4806	};
	4807	maybe_sync.leaveSync(ev);
	4808
	4809	var info: linux.siginfo_t = undefined;
	4810	while (true) {
	4811	const thread = maybe_sync.cancel_region.awaitIoUring() catch \|err\| switch (err) {
	4812	error.Canceled => unreachable, // blocked
	4813	};
	4814	thread.enqueue().* = .{
	4815	.opcode = .WAITID,
	4816	.flags = 0,
	4817	.ioprio = 0,
	4818	.fd = pid,
	4819	.off = @intFromPtr(&info),
	4820	.addr = 0,
	4821	.len = @intFromEnum(linux.P.PID),
	4822	.rw_flags = 0,
	4823	.user_data = @intFromPtr(maybe_sync.cancel_region.fiber),
	4824	.buf_index = 0,
	4825	.personality = 0,
	4826	.splice_fd_in = linux.W.EXITED,
	4827	.addr3 = 0,
	4828	.resv = 0,
	4829	};
	4830	ev.yield(null, .nothing);
	4831	switch (maybe_sync.cancel_region.errno()) {
	4832	.SUCCESS => return,
	4833	.INTR, .CANCELED => continue,
	4834	.CHILD => \|err\| return errnoBug(err) catch {}, // Double-free.
	4835	else => \|err\| return unexpectedErrno(err) catch {},
	4836	}
	4837	}
	4838	}
	4839
	4840	fn childCleanup(ev: Evented, child: process.Child) void {
	4841	if (child.stdin) \|*stdin\| {
	4842	ev.close(stdin.handle);
	4843	child.stdin = null;
	4844	}
	4845	if (child.stdout) \|*stdout\| {
	4846	ev.close(stdout.handle);
	4847	child.stdout = null;
	4848	}
	4849	if (child.stderr) \|*stderr\| {
	4850	ev.close(stderr.handle);
	4851	child.stderr = null;
	4852	}
	4853	child.id = null;
	4854	}
	4855
	4856	fn progressParentFile(userdata: ?*anyopaque) std.Progress.ParentFileError!File {
	4857	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4858	const cancel_protection = swapCancelProtection(ev, .blocked);
	4859	defer assert(swapCancelProtection(ev, cancel_protection) == .blocked);
	4860	ev.scanEnviron() catch \|err\| switch (err) {
	4861	error.Canceled => unreachable, // blocked
	4862	};
	4863	return ev.environ.zig_progress_file;
	4864	}
	4865
	4866	fn scanEnviron(ev: *Evented) Io.Cancelable!void {
	4867	const ev_io = ev.io();
	4868	try ev.environ_mutex.lock(ev_io);
	4869	defer ev.environ_mutex.unlock(ev_io);
	4870	if (ev.environ_initialized) return;
	4871	ev.environ.scan(ev.allocator());
	4872	ev.environ_initialized = true;
	4873	}
	4874
	4875	fn clockResolution(userdata: ?*anyopaque, clock: Io.Clock) Io.Clock.ResolutionError!Io.Duration {
	4876	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4877	_ = ev;
	4878	const clock_id = clockToPosix(clock);
	4879	var timespec: linux.timespec = undefined;
	4880	return switch (linux.errno(linux.clock_getres(clock_id, &timespec))) {
	4881	.SUCCESS => .fromNanoseconds(nanosecondsFromPosix(&timespec)),
	4882	.INVAL => return error.ClockUnavailable,
	4883	else => \|err\| return unexpectedErrno(err),
	4884	};
	4885	}
	4886
	4887	fn now(userdata: ?*anyopaque, clock: Io.Clock) Io.Timestamp {
	4888	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4889	_ = ev;
	4890	var tp: linux.timespec = undefined;
	4891	switch (linux.errno(linux.clock_gettime(clockToPosix(clock), &tp))) {
	4892	.SUCCESS => return timestampFromPosix(&tp),
	4893	else => return .zero,
	4894	}
	4895	}
	4896
	4897	fn sleep(userdata: ?*anyopaque, timeout: Io.Timeout) Io.Cancelable!void {
	4898	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4899
	4900	const timespec: linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = timespec: switch (timeout) {
	4901	.none => .{
	4902	.{
	4903	.sec = std.math.maxInt(i64),
	4904	.nsec = std.time.ns_per_s - 1,
	4905	},
	4906	.awake,
	4907	linux.IORING_TIMEOUT_ABS,
	4908	},
	4909	.duration => \|duration\| {
	4910	const ns = duration.raw.toNanoseconds();
	4911	break :timespec .{
	4912	.{
	4913	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	4914	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	4915	},
	4916	duration.clock,
	4917	0,
	4918	};
	4919	},
	4920	.deadline => \|deadline\| {
	4921	const ns = deadline.raw.toNanoseconds();
	4922	break :timespec .{
	4923	.{
	4924	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	4925	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	4926	},
	4927	deadline.clock,
	4928	linux.IORING_TIMEOUT_ABS,
	4929	};
	4930	},
	4931	};
	4932	var cancel_region: CancelRegion = .init();
	4933	defer cancel_region.deinit();
	4934	const thread = try cancel_region.awaitIoUring();
	4935	thread.enqueue().* = .{
	4936	.opcode = .TIMEOUT,
	4937	.flags = 0,
	4938	.ioprio = 0,
	4939	.fd = 0,
	4940	.off = 0,
	4941	.addr = @intFromPtr(&timespec),
	4942	.len = 1,
	4943	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
	4944	.real => linux.IORING_TIMEOUT_REALTIME,
	4945	else => 0,
	4946	.boot => linux.IORING_TIMEOUT_BOOTTIME,
	4947	}),
	4948	.user_data = @intFromPtr(cancel_region.fiber),
	4949	.buf_index = 0,
	4950	.personality = 0,
	4951	.splice_fd_in = 0,
	4952	.addr3 = 0,
	4953	.resv = 0,
	4954	};
	4955	ev.yield(null, .nothing);
	4956	switch (cancel_region.errno()) {
	4957	// Handles SUCCESS as well as clock not available and unexpected
	4958	// errors. The user had a chance to check clock resolution before
	4959	// getting here, which would have reported 0, making this a legal
	4960	// amount of time to sleep.
	4961	else => return,
	4962	.INTR, .CANCELED => return error.Canceled,
	4963	}
	4964	}
	4965
	4966	fn random(userdata: ?*anyopaque, buffer: []u8) void {
	4967	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4968	var thread: *Thread = .current();
	4969	if (!thread.csprng.isInitialized()) {
	4970	@branchHint(.unlikely);
	4971	var seed: [Csprng.seed_len]u8 = undefined;
	4972	{
	4973	const ev_io = ev.io();
	4974	ev.csprng_mutex.lockUncancelable(ev_io);
	4975	defer ev.csprng_mutex.unlock(ev_io);
	4976	if (!ev.csprng.isInitialized()) {
	4977	@branchHint(.unlikely);
	4978	var cancel_region: CancelRegion = .initBlocked();
	4979	defer cancel_region.deinit();
	4980	ev.urandomReadAll(&cancel_region, &seed) catch \|err\| switch (err) {
	4981	error.Canceled => unreachable, // blocked
	4982	else => fallbackSeed(ev, &seed),
	4983	};
	4984	ev.csprng.rng = .init(seed);
	4985	thread = .current();
	4986	}
	4987	ev.csprng.rng.fill(&seed);
	4988	}
	4989	if (!thread.csprng.isInitialized()) {
	4990	@branchHint(.likely);
	4991	thread.csprng.rng = .init(seed);
	4992	} else thread.csprng.rng.addEntropy(&seed);
	4993	}
	4994	thread.csprng.rng.fill(buffer);
	4995	}
	4996
	4997	fn randomSecure(userdata: ?*anyopaque, buffer: []u8) Io.RandomSecureError!void {
	4998	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4999	if (buffer.len == 0) return;
	5000	var cancel_region: CancelRegion = .init();
	5001	defer cancel_region.deinit();
	5002	ev.urandomReadAll(&cancel_region, buffer) catch \|err\| switch (err) {
	5003	error.Canceled => return error.Canceled,
	5004	else => return error.EntropyUnavailable,
	5005	};
	5006	}
	5007
	5008	fn netListenIpUnavailable(
	5009	userdata: ?*anyopaque,
	5010	address: net.IpAddress,
	5011	options: net.IpAddress.ListenOptions,
	5012	) net.IpAddress.ListenError!net.Server {
	5013	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5014	_ = ev;
	5015	_ = address;
	5016	_ = options;
	5017	return error.NetworkDown;
	5018	}
	5019
	5020	fn netAcceptUnavailable(
	5021	userdata: ?*anyopaque,
	5022	listen_handle: net.Socket.Handle,
	5023	) net.Server.AcceptError!net.Stream {
	5024	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5025	_ = ev;
	5026	_ = listen_handle;
	5027	return error.NetworkDown;
	5028	}
	5029
	5030	fn netBindIp(
	5031	userdata: ?*anyopaque,
	5032	address: *const net.IpAddress,
	5033	options: net.IpAddress.BindOptions,
	5034	) net.IpAddress.BindError!net.Socket {
	5035	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5036	const family = posixAddressFamily(address);
	5037	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
	5038	defer maybe_sync.deinit(ev);
	5039	const socket_fd = try ev.socket(&maybe_sync.cancel_region, family, options);
	5040	errdefer ev.close(socket_fd);
	5041	var storage: PosixAddress = undefined;
	5042	var addr_len = addressToPosix(address, &storage);
	5043	try ev.bind(&maybe_sync.cancel_region, socket_fd, &storage.any, addr_len);
	5044	try ev.getsockname(try maybe_sync.enterSync(ev), socket_fd, &storage.any, &addr_len);
	5045	return .{
	5046	.handle = socket_fd,
	5047	.address = addressFromPosix(&storage),
	5048	};
	5049	}
	5050
	5051	fn netConnectIpUnavailable(
	5052	userdata: ?*anyopaque,
	5053	address: *const net.IpAddress,
	5054	options: net.IpAddress.ConnectOptions,
	5055	) net.IpAddress.ConnectError!net.Stream {
	5056	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5057	_ = ev;
	5058	_ = address;
	5059	_ = options;
	5060	return error.NetworkDown;
	5061	}
	5062
	5063	fn netListenUnixUnavailable(
	5064	userdata: ?*anyopaque,
	5065	address: *const net.UnixAddress,
	5066	options: net.UnixAddress.ListenOptions,
	5067	) net.UnixAddress.ListenError!net.Socket.Handle {
	5068	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5069	_ = ev;
	5070	_ = address;
	5071	_ = options;
	5072	return error.AddressFamilyUnsupported;
	5073	}
	5074
	5075	fn netConnectUnixUnavailable(
	5076	userdata: ?*anyopaque,
	5077	address: *const net.UnixAddress,
	5078	) net.UnixAddress.ConnectError!net.Socket.Handle {
	5079	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5080	_ = ev;
	5081	_ = address;
	5082	return error.AddressFamilyUnsupported;
	5083	}
	5084
	5085	fn netSocketCreatePairUnavailable(
	5086	userdata: ?*anyopaque,
	5087	options: net.Socket.CreatePairOptions,
	5088	) net.Socket.CreatePairError![2]net.Socket {
	5089	_ = userdata;
	5090	_ = options;
	5091	return error.OperationUnsupported;
	5092	}
	5093
	5094	fn netSendUnavailable(
	5095	userdata: ?*anyopaque,
	5096	handle: net.Socket.Handle,
	5097	messages: []net.OutgoingMessage,
	5098	flags: net.SendFlags,
	5099	) struct { ?net.Socket.SendError, usize } {
	5100	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5101	_ = ev;
	5102	_ = handle;
	5103	_ = messages;
	5104	_ = flags;
	5105	return .{ error.NetworkDown, 0 };
	5106	}
	5107
	5108	fn netReceive(
	5109	userdata: ?*anyopaque,
	5110	handle: net.Socket.Handle,
	5111	message_buffer: []net.IncomingMessage,
	5112	data_buffer: []u8,
	5113	flags: net.ReceiveFlags,
	5114	timeout: Io.Timeout,
	5115	) struct { ?net.Socket.ReceiveTimeoutError, usize } {
	5116	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5117	const ev_io = ev.io();
	5118
	5119	var message_i: usize = 0;
	5120	var data_i: usize = 0;
	5121
	5122	const deadline: ?struct {
	5123	raw: Io.Timestamp,
	5124	timespec: linux.kernel_timespec,
	5125	clock: Io.Clock,
	5126	} = if (timeout.toTimestamp(ev_io)) \|deadline\| deadline: {
	5127	const ns = deadline.raw.toNanoseconds();
	5128	break :deadline .{
	5129	.raw = deadline.raw,
	5130	.timespec = .{
	5131	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	5132	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	5133	},
	5134	.clock = deadline.clock,
	5135	};
	5136	} else null;
	5137
	5138	var cancel_region: CancelRegion = .init();
	5139	defer cancel_region.deinit();
	5140	while (true) {
	5141	if (message_buffer.len - message_i == 0) return .{ null, message_i };
	5142	const message = &message_buffer[message_i];
	5143	const remaining_data_buffer = data_buffer[data_i..];
	5144	var storage: PosixAddress = undefined;
	5145	var iov: iovec = .{ .base = remaining_data_buffer.ptr, .len = remaining_data_buffer.len };
	5146	var msg: linux.msghdr = .{
	5147	.name = &storage.any,
	5148	.namelen = @sizeOf(PosixAddress),
	5149	.iov = (&iov)[0..1],
	5150	.iovlen = 1,
	5151	.control = message.control.ptr,
	5152	.controllen = @intCast(message.control.len),
	5153	.flags = undefined,
	5154	};
	5155
	5156	const thread = cancel_region.awaitIoUring() catch \|err\| return .{ err, message_i };
	5157	thread.enqueue().* = .{
	5158	.opcode = .RECVMSG,
	5159	.flags = if (deadline) \|_\| linux.IOSQE_IO_LINK else 0,
	5160	.ioprio = 0,
	5161	.fd = handle,
	5162	.off = 0,
	5163	.addr = @intFromPtr(&msg),
	5164	.len = 0,
	5165	.rw_flags = linux.MSG.NOSIGNAL \|
	5166	@as(u32, if (flags.oob) linux.MSG.OOB else 0) \|
	5167	@as(u32, if (flags.peek) linux.MSG.PEEK else 0) \|
	5168	@as(u32, if (flags.trunc) linux.MSG.TRUNC else 0),
	5169	.user_data = @intFromPtr(cancel_region.fiber),
	5170	.buf_index = 0,
	5171	.personality = 0,
	5172	.splice_fd_in = 0,
	5173	.addr3 = 0,
	5174	.resv = 0,
	5175	};
	5176	if (deadline) \|deadline_ptr\| thread.enqueue(). = .{
	5177	.opcode = .LINK_TIMEOUT,
	5178	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	5179	.ioprio = 0,
	5180	.fd = 0,
	5181	.off = 0,
	5182	.addr = @intFromPtr(&deadline_ptr.timespec),
	5183	.len = 1,
	5184	.rw_flags = linux.IORING_TIMEOUT_ABS \| @as(u32, switch (deadline_ptr.clock) {
	5185	.real => linux.IORING_TIMEOUT_REALTIME,
	5186	else => 0,
	5187	.boot => linux.IORING_TIMEOUT_BOOTTIME,
	5188	}),
	5189	.user_data = @intFromEnum(Completion.Userdata.wakeup),
	5190	.buf_index = 0,
	5191	.personality = 0,
	5192	.splice_fd_in = 0,
	5193	.addr3 = 0,
	5194	.resv = 0,
	5195	};
	5196	ev.yield(null, .nothing);
	5197	const completion = cancel_region.completion();
	5198	switch (completion.errno()) {
	5199	.SUCCESS => {
	5200	const data = remaining_data_buffer[0..@intCast(completion.result)];
	5201	data_i += data.len;
	5202	message.* = .{
	5203	.from = addressFromPosix(&storage),
	5204	.data = data,
	5205	.control = if (msg.control) \|ptr\| @as([*]u8, @ptrCast(ptr))[0..msg.controllen] else message.control,
	5206	.flags = .{
	5207	.eor = (msg.flags & linux.MSG.EOR) != 0,
	5208	.trunc = (msg.flags & linux.MSG.TRUNC) != 0,
	5209	.ctrunc = (msg.flags & linux.MSG.CTRUNC) != 0,
	5210	.oob = (msg.flags & linux.MSG.OOB) != 0,
	5211	.errqueue = if (@hasDecl(linux.MSG, "ERRQUEUE")) (msg.flags & linux.MSG.ERRQUEUE) != 0 else false,
	5212	},
	5213	};
	5214	message_i += 1;
	5215	continue;
	5216	},
	5217	.AGAIN => unreachable,
	5218	.INTR, .CANCELED => {
	5219	if (deadline) \|d\| {
	5220	if (now(ev, d.clock).nanoseconds >= d.raw.nanoseconds) return .{ error.Timeout, message_i };
	5221	}
	5222	continue;
	5223	},
	5224
	5225	.BADF => \|err\| return .{ errnoBug(err), message_i },
	5226	.NFILE => return .{ error.SystemFdQuotaExceeded, message_i },
	5227	.MFILE => return .{ error.ProcessFdQuotaExceeded, message_i },
	5228	.FAULT => \|err\| return .{ errnoBug(err), message_i },
	5229	.INVAL => \|err\| return .{ errnoBug(err), message_i },
	5230	.NOBUFS => return .{ error.SystemResources, message_i },
	5231	.NOMEM => return .{ error.SystemResources, message_i },
	5232	.NOTCONN => return .{ error.SocketUnconnected, message_i },
	5233	.NOTSOCK => \|err\| return .{ errnoBug(err), message_i },
	5234	.MSGSIZE => return .{ error.MessageOversize, message_i },
	5235	.PIPE => return .{ error.SocketUnconnected, message_i },
	5236	.OPNOTSUPP => \|err\| return .{ errnoBug(err), message_i },
	5237	.CONNRESET => return .{ error.ConnectionResetByPeer, message_i },
	5238	.NETDOWN => return .{ error.NetworkDown, message_i },
	5239	else => \|err\| return .{ unexpectedErrno(err), message_i },
	5240	}
	5241	}
	5242	}
	5243
	5244	fn netReadUnavailable(
	5245	userdata: ?*anyopaque,
	5246	fd: net.Socket.Handle,
	5247	data: [][]u8,
	5248	) net.Stream.Reader.Error!usize {
	5249	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5250	_ = ev;
	5251	_ = fd;
	5252	_ = data;
	5253	return error.NetworkDown;
	5254	}
	5255
	5256	fn netWriteUnavailable(
	5257	userdata: ?*anyopaque,
	5258	handle: net.Socket.Handle,
	5259	header: []const u8,
	5260	data: []const []const u8,
	5261	splat: usize,
	5262	) net.Stream.Writer.Error!usize {
	5263	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5264	_ = ev;
	5265	_ = handle;
	5266	_ = header;
	5267	_ = data;
	5268	_ = splat;
	5269	return error.NetworkDown;
	5270	}
	5271
	5272	fn netWriteFileUnavailable(
	5273	userdata: ?*anyopaque,
	5274	socket_handle: net.Socket.Handle,
	5275	header: []const u8,
	5276	file_reader: *File.Reader,
	5277	limit: Io.Limit,
	5278	) net.Stream.Writer.WriteFileError!usize {
	5279	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5280	_ = ev;
	5281	_ = socket_handle;
	5282	_ = header;
	5283	_ = file_reader;
	5284	_ = limit;
	5285	return error.NetworkDown;
	5286	}
	5287
	5288	fn netClose(userdata: ?*anyopaque, handles: []const net.Socket.Handle) void {
	5289	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5290	for (handles) \|handle\| ev.close(handle);
	5291	}
	5292
	5293	fn netShutdown(
	5294	userdata: ?*anyopaque,
	5295	handle: net.Socket.Handle,
	5296	how: net.ShutdownHow,
	5297	) net.ShutdownError!void {
	5298	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5299	var cancel_region: CancelRegion = .init();
	5300	defer cancel_region.deinit();
	5301	while (true) {
	5302	const thread = try cancel_region.awaitIoUring();
	5303	thread.enqueue().* = .{
	5304	.opcode = .SHUTDOWN,
	5305	.flags = 0,
	5306	.ioprio = 0,
	5307	.fd = handle,
	5308	.off = 0,
	5309	.addr = 0,
	5310	.len = switch (how) {
	5311	.recv => linux.SHUT.RD,
	5312	.send => linux.SHUT.WR,
	5313	.both => linux.SHUT.RDWR,
	5314	},
	5315	.rw_flags = 0,
	5316	.user_data = @intFromPtr(cancel_region.fiber),
	5317	.buf_index = 0,
	5318	.personality = 0,
	5319	.splice_fd_in = 0,
	5320	.addr3 = 0,
	5321	.resv = 0,
	5322	};
	5323	ev.yield(null, .nothing);
	5324	switch (cancel_region.errno()) {
	5325	.SUCCESS => return,
	5326	.INTR, .CANCELED => continue,
	5327	.BADF, .NOTSOCK, .INVAL => \|err\| return errnoBug(err),
	5328	.NOTCONN => return error.SocketUnconnected,
	5329	.NOBUFS => return error.SystemResources,
	5330	else => \|err\| return unexpectedErrno(err),
	5331	}
	5332	}
	5333	}
	5334
	5335	fn netInterfaceNameResolveUnavailable(
	5336	userdata: ?*anyopaque,
	5337	name: *const net.Interface.Name,
	5338	) net.Interface.Name.ResolveError!net.Interface {
	5339	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5340	_ = ev;
	5341	_ = name;
	5342	return error.InterfaceNotFound;
	5343	}
	5344
	5345	fn netInterfaceNameUnavailable(
	5346	userdata: ?*anyopaque,
	5347	interface: net.Interface,
	5348	) net.Interface.NameError!net.Interface.Name {
	5349	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5350	_ = ev;
	5351	_ = interface;
	5352	return error.Unexpected;
	5353	}
	5354
	5355	fn netLookupUnavailable(
	5356	userdata: ?*anyopaque,
	5357	host_name: net.HostName,
	5358	resolved: *Io.Queue(net.HostName.LookupResult),
	5359	options: net.HostName.LookupOptions,
	5360	) net.HostName.LookupError!void {
	5361	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5362	_ = host_name;
	5363	_ = options;
	5364	resolved.close(ev.io());
	5365	return error.NetworkDown;
	5366	}
	5367
	5368	fn bind(
	5369	ev: *Evented,
	5370	cancel_region: *CancelRegion,
	5371	socket_fd: fd_t,
	5372	addr: *const linux.sockaddr,
	5373	addr_len: linux.socklen_t,
	5374	) !void {
	5375	while (true) {
	5376	const thread = try cancel_region.awaitIoUring();
	5377	thread.enqueue().* = .{
	5378	.opcode = .BIND,
	5379	.flags = 0,
	5380	.ioprio = 0,
	5381	.fd = socket_fd,
	5382	.off = addr_len,
	5383	.addr = @intFromPtr(addr),
	5384	.len = 0,
	5385	.rw_flags = 0,
	5386	.user_data = @intFromPtr(cancel_region.fiber),
	5387	.buf_index = 0,
	5388	.personality = 0,
	5389	.splice_fd_in = 0,
	5390	.addr3 = 0,
	5391	.resv = 0,
	5392	};
	5393	ev.yield(null, .nothing);
	5394	switch (cancel_region.errno()) {
	5395	.SUCCESS => return,
	5396	.INTR, .CANCELED => continue,
	5397	.ADDRINUSE => return error.AddressInUse,
	5398	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	5399	.INVAL => \|err\| return errnoBug(err), // invalid parameters
	5400	.NOTSOCK => \|err\| return errnoBug(err), // invalid `sockfd`
	5401	.AFNOSUPPORT => return error.AddressFamilyUnsupported,
	5402	.ADDRNOTAVAIL => return error.AddressUnavailable,
	5403	.FAULT => \|err\| return errnoBug(err), // invalid `addr` pointer
	5404	.NOMEM => return error.SystemResources,
	5405	else => \|err\| return unexpectedErrno(err),
	5406	}
	5407	}
	5408	}
	5409
	5410	fn chdir(ev: Evented, sync: CancelRegion.Sync, path: [*:0]const u8) ChdirError!void {
	5411	_ = ev;
	5412	while (true) {
	5413	try sync.cancel_region.await(.nothing);
	5414	switch (linux.errno(linux.chdir(path))) {
	5415	.SUCCESS => return,
	5416	.INTR => continue,
	5417	.ACCES => return error.AccessDenied,
	5418	.IO => return error.FileSystem,
	5419	.LOOP => return error.SymLinkLoop,
	5420	.NAMETOOLONG => return error.NameTooLong,
	5421	.NOENT => return error.FileNotFound,
	5422	.NOMEM => return error.SystemResources,
	5423	.NOTDIR => return error.NotDir,
	5424	.ILSEQ => return error.BadPathName,
	5425	.FAULT => \|err\| return errnoBug(err),
	5426	else => \|err\| return unexpectedErrno(err),
	5427	}
	5428	}
	5429	}
	5430
	5431	fn close(ev: *Evented, fd: fd_t) void {
	5432	_ = ev;
	5433	const thread: *Thread = .current();
	5434	thread.enqueue().* = .{
	5435	.opcode = .CLOSE,
	5436	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	5437	.ioprio = 0,
	5438	.fd = fd,
	5439	.off = 0,
	5440	.addr = 0,
	5441	.len = 0,
	5442	.rw_flags = 0,
	5443	.user_data = @intFromEnum(Completion.Userdata.close),
	5444	.buf_index = 0,
	5445	.personality = 0,
	5446	.splice_fd_in = 0,
	5447	.addr3 = 0,
	5448	.resv = 0,
	5449	};
	5450	}
	5451
	5452	fn fchdir(ev: Evented, sync: CancelRegion.Sync, dir: fd_t) process.SetCurrentDirError!void {
	5453	_ = ev;
	5454	if (dir == linux.AT.FDCWD) return;
	5455	while (true) {
	5456	try sync.cancel_region.await(.nothing);
	5457	switch (linux.errno(linux.fchdir(dir))) {
	5458	.SUCCESS => return,
	5459	.INTR => continue,
	5460	.ACCES => return error.AccessDenied,
	5461	.NOTDIR => return error.NotDir,
	5462	.IO => return error.FileSystem,
	5463	.BADF => \|err\| return errnoBug(err),
	5464	else => \|err\| return unexpectedErrno(err),
	5465	}
	5466	}
	5467	}
	5468
	5469	fn fchmodat(
	5470	ev: *Evented,
	5471	sync: *CancelRegion.Sync,
	5472	dir: fd_t,
	5473	path: [*:0]const u8,
	5474	mode: linux.mode_t,
	5475	flags: u32,
	5476	) Dir.SetFilePermissionsError!void {
	5477	_ = ev;
	5478	while (true) {
	5479	try sync.cancel_region.await(.nothing);
	5480	switch (linux.errno(linux.fchmodat2(dir, path, mode, flags))) {
	5481	.SUCCESS => return,
	5482	.INTR => continue,
	5483	.BADF => \|err\| return errnoBug(err),
	5484	.FAULT => \|err\| return errnoBug(err),
	5485	.INVAL => \|err\| return errnoBug(err),
	5486	.ACCES => return error.AccessDenied,
	5487	.IO => return error.InputOutput,
	5488	.LOOP => return error.SymLinkLoop,
	5489	.NOENT => return error.FileNotFound,
	5490	.NOMEM => return error.SystemResources,
	5491	.NOTDIR => return error.FileNotFound,
	5492	.OPNOTSUPP => return error.OperationUnsupported,
	5493	.PERM => return error.PermissionDenied,
	5494	.ROFS => return error.ReadOnlyFileSystem,
	5495	else => \|err\| return unexpectedErrno(err),
	5496	}
	5497	}
	5498	}
	5499
	5500	fn fchownat(
	5501	ev: *Evented,
	5502	sync: *CancelRegion.Sync,
	5503	dir: fd_t,
	5504	path: [*:0]const u8,
	5505	owner: linux.uid_t,
	5506	group: linux.gid_t,
	5507	flags: u32,
	5508	) File.SetOwnerError!void {
	5509	_ = ev;
	5510	while (true) {
	5511	try sync.cancel_region.await(.nothing);
	5512	switch (linux.errno(linux.fchownat(dir, path, owner, group, flags))) {
	5513	.SUCCESS => return,
	5514	.INTR => continue,
	5515	.BADF => \|err\| return errnoBug(err), // likely fd refers to directory opened without `Dir.OpenOptions.iterate`
	5516	.FAULT => \|err\| return errnoBug(err),
	5517	.INVAL => \|err\| return errnoBug(err),
	5518	.ACCES => return error.AccessDenied,
	5519	.IO => return error.InputOutput,
	5520	.LOOP => return error.SymLinkLoop,
	5521	.NOENT => return error.FileNotFound,
	5522	.NOMEM => return error.SystemResources,
	5523	.NOTDIR => return error.FileNotFound,
	5524	.PERM => return error.PermissionDenied,
	5525	.ROFS => return error.ReadOnlyFileSystem,
	5526	else => \|err\| return unexpectedErrno(err),
	5527	}
	5528	}
	5529	}
	5530
	5531	fn flock(
	5532	ev: *Evented,
	5533	sync: *CancelRegion.Sync,
	5534	fd: fd_t,
	5535	op: File.Lock,
	5536	blocking: enum { blocking, nonblocking },
	5537	) (File.LockError \|\| error{WouldBlock})!void {
	5538	while (true) {
	5539	try sync.cancel_region.await(.nothing);
	5540	switch (linux.errno(linux.flock(fd, LOCK.NB \| @as(i32, switch (op) {
	5541	.none => LOCK.UN,
	5542	.shared => LOCK.SH,
	5543	.exclusive => LOCK.EX,
	5544	})))) {
	5545	.SUCCESS => return,
	5546	.INTR => continue,
	5547	.BADF => \|err\| return errnoBug(err),
	5548	.INVAL => \|err\| return errnoBug(err), // invalid parameters
	5549	.NOLCK => return error.SystemResources,
	5550	.AGAIN => {
	5551	const thread = try sync.cancel_region.awaitIoUring();
	5552	thread.enqueue().* = .{
	5553	.opcode = .NOP,
	5554	.flags = 0,
	5555	.ioprio = 0,
	5556	.fd = 0,
	5557	.off = 0,
	5558	.addr = 0,
	5559	.len = 0,
	5560	.rw_flags = 0,
	5561	.user_data = @intFromPtr(sync.cancel_region.fiber),
	5562	.buf_index = 0,
	5563	.personality = 0,
	5564	.splice_fd_in = 0,
	5565	.addr3 = 0,
	5566	.resv = 0,
	5567	};
	5568	ev.yield(null, .nothing);
	5569	switch (sync.cancel_region.errno()) {
	5570	.SUCCESS, .INTR, .CANCELED => {},
	5571	else => unreachable,
	5572	}
	5573	switch (blocking) {
	5574	.blocking => continue,
	5575	.nonblocking => return error.WouldBlock,
	5576	}
	5577	},
	5578	.OPNOTSUPP => return error.FileLocksUnsupported,
	5579	else => \|err\| return unexpectedErrno(err),
	5580	}
	5581	}
	5582	}
	5583
	5584	fn getsockname(
	5585	ev: *Evented,
	5586	sync: *CancelRegion.Sync,
	5587	socket_fd: fd_t,
	5588	addr: *linux.sockaddr,
	5589	addr_len: *linux.socklen_t,
	5590	) !void {
	5591	_ = ev;
	5592	while (true) {
	5593	try sync.cancel_region.await(.nothing);
	5594	switch (linux.errno(linux.getsockname(socket_fd, addr, addr_len))) {
	5595	.SUCCESS => return,
	5596	.INTR => continue,
	5597	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	5598	.FAULT => \|err\| return errnoBug(err),
	5599	.INVAL => \|err\| return errnoBug(err), // invalid parameters
	5600	.NOTSOCK => \|err\| return errnoBug(err), // always a race condition
	5601	.NOBUFS => return error.SystemResources,
	5602	else => \|err\| return unexpectedErrno(err),
	5603	}
	5604	}
	5605	}
	5606
	5607	fn linkat(
	5608	ev: *Evented,
	5609	cancel_region: *CancelRegion,
	5610	old_dir: fd_t,
	5611	old_path: [*:0]const u8,
	5612	new_dir: fd_t,
	5613	new_path: [*:0]const u8,
	5614	flags: u32,
	5615	) File.HardLinkError!void {
	5616	while (true) {
	5617	const thread = try cancel_region.awaitIoUring();
	5618	thread.enqueue().* = .{
	5619	.opcode = .LINKAT,
	5620	.flags = 0,
	5621	.ioprio = 0,
	5622	.fd = old_dir,
	5623	.off = @intFromPtr(new_path),
	5624	.addr = @intFromPtr(old_path),
	5625	.len = @bitCast(new_dir),
	5626	.rw_flags = flags,
	5627	.user_data = @intFromPtr(cancel_region.fiber),
	5628	.buf_index = 0,
	5629	.personality = 0,
	5630	.splice_fd_in = 0,
	5631	.addr3 = 0,
	5632	.resv = 0,
	5633	};
	5634	ev.yield(null, .nothing);
	5635	switch (cancel_region.errno()) {
	5636	.SUCCESS => return,
	5637	.INTR, .CANCELED => continue,
	5638	.ACCES => return error.AccessDenied,
	5639	.DQUOT => return error.DiskQuota,
	5640	.EXIST => return error.PathAlreadyExists,
	5641	.IO => return error.HardwareFailure,
	5642	.LOOP => return error.SymLinkLoop,
	5643	.MLINK => return error.LinkQuotaExceeded,
	5644	.NAMETOOLONG => return error.NameTooLong,
	5645	.NOENT => return error.FileNotFound,
	5646	.NOMEM => return error.SystemResources,
	5647	.NOSPC => return error.NoSpaceLeft,
	5648	.NOTDIR => return error.NotDir,
	5649	.PERM => return error.PermissionDenied,
	5650	.ROFS => return error.ReadOnlyFileSystem,
	5651	.XDEV => return error.CrossDevice,
	5652	.ILSEQ => return error.BadPathName,
	5653	.FAULT => \|err\| return errnoBug(err),
	5654	.INVAL => \|err\| return errnoBug(err),
	5655	else => \|err\| return unexpectedErrno(err),
	5656	}
	5657	}
	5658	}
	5659
	5660	fn lseek(
	5661	ev: *Evented,
	5662	sync: *CancelRegion.Sync,
	5663	fd: fd_t,
	5664	offset: u64,
	5665	whence: u32,
	5666	) File.SeekError!void {
	5667	_ = ev;
	5668	while (true) {
	5669	try sync.cancel_region.await(.nothing);
	5670	var result: u64 = undefined;
	5671	switch (linux.errno(switch (@sizeOf(usize)) {
	5672	else => comptime unreachable,
	5673	4 => linux.llseek(fd, offset, &result, whence),
	5674	8 => linux.lseek(fd, @bitCast(offset), whence),
	5675	})) {
	5676	.SUCCESS => return,
	5677	.INTR => continue,
	5678	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	5679	.INVAL => return error.Unseekable,
	5680	.OVERFLOW => return error.Unseekable,
	5681	.SPIPE => return error.Unseekable,
	5682	.NXIO => return error.Unseekable,
	5683	else => \|err\| return unexpectedErrno(err),
	5684	}
	5685	}
	5686	}
	5687
	5688	fn openat(
	5689	ev: *Evented,
	5690	cancel_region: *CancelRegion,
	5691	dir: fd_t,
	5692	path: [*:0]const u8,
	5693	flags: linux.O,
	5694	mode: linux.mode_t,
	5695	) File.OpenError!fd_t {
	5696	var mut_flags = flags;
	5697	if (@hasField(linux.O, "LARGEFILE")) mut_flags.LARGEFILE = true;
	5698	while (true) {
	5699	const thread = try cancel_region.awaitIoUring();
	5700	thread.enqueue().* = .{
	5701	.opcode = .OPENAT,
	5702	.flags = 0,
	5703	.ioprio = 0,
	5704	.fd = dir,
	5705	.off = 0,
	5706	.addr = @intFromPtr(path),
	5707	.len = mode,
	5708	.rw_flags = @bitCast(mut_flags),
	5709	.user_data = @intFromPtr(cancel_region.fiber),
	5710	.buf_index = 0,
	5711	.personality = 0,
	5712	.splice_fd_in = 0,
	5713	.addr3 = 0,
	5714	.resv = 0,
	5715	};
	5716	ev.yield(null, .nothing);
	5717	const completion = cancel_region.completion();
	5718	switch (completion.errno()) {
	5719	.SUCCESS => return completion.result,
	5720	.INTR, .CANCELED => continue,
	5721	.FAULT => \|err\| return errnoBug(err),
	5722	.INVAL => return error.BadPathName,
	5723	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	5724	.ACCES => return error.AccessDenied,
	5725	.FBIG => return error.FileTooBig,
	5726	.OVERFLOW => return error.FileTooBig,
	5727	.ISDIR => return error.IsDir,
	5728	.LOOP => return error.SymLinkLoop,
	5729	.MFILE => return error.ProcessFdQuotaExceeded,
	5730	.NAMETOOLONG => return error.NameTooLong,
	5731	.NFILE => return error.SystemFdQuotaExceeded,
	5732	.NODEV => return error.NoDevice,
	5733	.NOENT => return error.FileNotFound,
	5734	.SRCH => return error.FileNotFound, // Linux when opening procfs files.
	5735	.NOMEM => return error.SystemResources,
	5736	.NOSPC => return error.NoSpaceLeft,
	5737	.NOTDIR => return error.NotDir,
	5738	.PERM => return error.PermissionDenied,
	5739	.EXIST => return error.PathAlreadyExists,
	5740	.BUSY => return error.DeviceBusy,
	5741	.OPNOTSUPP => return error.FileLocksUnsupported,
	5742	.AGAIN => return error.WouldBlock,
	5743	.TXTBSY => return error.FileBusy,
	5744	.NXIO => return error.NoDevice,
	5745	.ILSEQ => return error.BadPathName,
	5746	else => \|err\| return unexpectedErrno(err),
	5747	}
	5748	}
	5749	}
	5750
	5751	fn preadv(
	5752	ev: *Evented,
	5753	cancel_region: *CancelRegion,
	5754	fd: fd_t,
	5755	iov: []const iovec,
	5756	offset: ?u64,
	5757	) File.Reader.Error!usize {
	5758	if (iov.len == 0) return 0;
	5759	const gather = iov.len > 1 or iov[0].len > 0xfffff000;
	5760	while (true) {
	5761	const thread = try cancel_region.awaitIoUring();
	5762	thread.enqueue().* = .{
	5763	.opcode = if (gather) .READV else .READ,
	5764	.flags = 0,
	5765	.ioprio = 0,
	5766	.fd = fd,
	5767	.off = offset orelse std.math.maxInt(u64),
	5768	.addr = if (gather) @intFromPtr(iov.ptr) else @intFromPtr(iov[0].base),
	5769	.len = @intCast(if (gather) iov.len else iov[0].len),
	5770	.rw_flags = 0,
	5771	.user_data = @intFromPtr(cancel_region.fiber),
	5772	.buf_index = 0,
	5773	.personality = 0,
	5774	.splice_fd_in = 0,
	5775	.addr3 = 0,
	5776	.resv = 0,
	5777	};
	5778	ev.yield(null, .nothing);
	5779	const completion = cancel_region.completion();
	5780	switch (completion.errno()) {
	5781	.SUCCESS => return @as(u32, @bitCast(completion.result)),
	5782	.INTR, .CANCELED => continue,
	5783	.INVAL => \|err\| return errnoBug(err),
	5784	.FAULT => \|err\| return errnoBug(err),
	5785	.AGAIN => return error.WouldBlock,
	5786	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed
	5787	.IO => return error.InputOutput,
	5788	.ISDIR => return error.IsDir,
	5789	.NOBUFS => return error.SystemResources,
	5790	.NOMEM => return error.SystemResources,
	5791	.NOTCONN => return error.SocketUnconnected,
	5792	.CONNRESET => return error.ConnectionResetByPeer,
	5793	else => \|err\| return unexpectedErrno(err),
	5794	}
	5795	}
	5796	}
	5797
	5798	fn pwritev(
	5799	ev: *Evented,
	5800	cancel_region: *CancelRegion,
	5801	fd: fd_t,
	5802	iov: []const iovec_const,
	5803	offset: ?u64,
	5804	) File.Writer.Error!usize {
	5805	if (iov.len == 0) return 0;
	5806	const scatter = iov.len > 1 or iov[0].len > 0xfffff000;
	5807	while (true) {
	5808	const thread = try cancel_region.awaitIoUring();
	5809	thread.enqueue().* = .{
	5810	.opcode = if (scatter) .WRITEV else .WRITE,
	5811	.flags = 0,
	5812	.ioprio = 0,
	5813	.fd = fd,
	5814	.off = offset orelse std.math.maxInt(u64),
	5815	.addr = if (scatter) @intFromPtr(iov.ptr) else @intFromPtr(iov[0].base),
	5816	.len = @intCast(if (scatter) iov.len else iov[0].len),
	5817	.rw_flags = 0,
	5818	.user_data = @intFromPtr(cancel_region.fiber),
	5819	.buf_index = 0,
	5820	.personality = 0,
	5821	.splice_fd_in = 0,
	5822	.addr3 = 0,
	5823	.resv = 0,
	5824	};
	5825	ev.yield(null, .nothing);
	5826	const completion = cancel_region.completion();
	5827	switch (completion.errno()) {
	5828	.SUCCESS => return @as(u32, @bitCast(completion.result)),
	5829	.INTR, .CANCELED => continue,
	5830	.INVAL => \|err\| return errnoBug(err),
	5831	.FAULT => \|err\| return errnoBug(err),
	5832	.AGAIN => return error.WouldBlock,
	5833	.BADF => return error.NotOpenForWriting, // Can be a race condition.
	5834	.DESTADDRREQ => \|err\| return errnoBug(err), // `connect` was never called.
	5835	.DQUOT => return error.DiskQuota,
	5836	.FBIG => return error.FileTooBig,
	5837	.IO => return error.InputOutput,
	5838	.NOSPC => return error.NoSpaceLeft,
	5839	.PERM => return error.PermissionDenied,
	5840	.PIPE => return error.BrokenPipe,
	5841	.CONNRESET => \|err\| return errnoBug(err), // Not a socket handle.
	5842	.BUSY => return error.DeviceBusy,
	5843	else => \|err\| return unexpectedErrno(err),
	5844	}
	5845	}
	5846	}
	5847
	5848	fn readAll(
	5849	ev: *Evented,
	5850	cancel_region: *CancelRegion,
	5851	fd: fd_t,
	5852	buffer: []u8,
	5853	) (File.Reader.Error \|\| error{EndOfStream})!void {
	5854	var index: usize = 0;
	5855	while (buffer.len - index != 0) {
	5856	const len = try ev.preadv(cancel_region, fd, &.{
	5857	.{ .base = buffer[index..].ptr, .len = buffer.len - index },
	5858	}, null);
	5859	if (len == 0) return error.EndOfStream;
	5860	index += len;
	5861	}
	5862	}
	5863
	5864	fn realPath(
	5865	ev: *Evented,
	5866	sync: *CancelRegion.Sync,
	5867	fd: fd_t,
	5868	out_buffer: []u8,
	5869	) File.RealPathError!usize {
	5870	_ = ev;
	5871	var procfs_buf: [std.fmt.count("/proc/self/fd/{d}\x00", .{std.math.minInt(fd_t)})]u8 = undefined;
	5872	const proc_path = std.fmt.bufPrintSentinel(&procfs_buf, "/proc/self/fd/{d}", .{fd}, 0) catch
	5873	unreachable;
	5874	while (true) {
	5875	try sync.cancel_region.await(.nothing);
	5876	const rc = linux.readlink(proc_path, out_buffer.ptr, out_buffer.len);
	5877	switch (linux.errno(rc)) {
	5878	.SUCCESS => return rc,
	5879	.INTR => continue,
	5880	.ACCES => return error.AccessDenied,
	5881	.FAULT => \|err\| return errnoBug(err),
	5882	.IO => return error.FileSystem,
	5883	.LOOP => return error.SymLinkLoop,
	5884	.NAMETOOLONG => return error.NameTooLong,
	5885	.NOENT => return error.FileNotFound,
	5886	.NOMEM => return error.SystemResources,
	5887	.NOTDIR => return error.NotDir,
	5888	.ILSEQ => \|err\| return errnoBug(err),
	5889	else => \|err\| return unexpectedErrno(err),
	5890	}
	5891	}
	5892	}
	5893
	5894	fn renameat(
	5895	ev: *Evented,
	5896	cancel_region: *CancelRegion,
	5897	old_dir: fd_t,
	5898	old_path: [*:0]const u8,
	5899	new_dir: fd_t,
	5900	new_path: [*:0]const u8,
	5901	flags: linux.RENAME,
	5902	) Dir.RenameError!void {
	5903	while (true) {
	5904	const thread = try cancel_region.awaitIoUring();
	5905	thread.enqueue().* = .{
	5906	.opcode = .RENAMEAT,
	5907	.flags = 0,
	5908	.ioprio = 0,
	5909	.fd = old_dir,
	5910	.off = @intFromPtr(new_path),
	5911	.addr = @intFromPtr(old_path),
	5912	.len = @bitCast(new_dir),
	5913	.rw_flags = @bitCast(flags),
	5914	.user_data = @intFromPtr(cancel_region.fiber),
	5915	.buf_index = 0,
	5916	.personality = 0,
	5917	.splice_fd_in = 0,
	5918	.addr3 = 0,
	5919	.resv = 0,
	5920	};
	5921	ev.yield(null, .nothing);
	5922	switch (cancel_region.errno()) {
	5923	.SUCCESS => return,
	5924	.INTR, .CANCELED => continue,
	5925	.ACCES => return error.AccessDenied,
	5926	.PERM => return error.PermissionDenied,
	5927	.BUSY => return error.FileBusy,
	5928	.DQUOT => return error.DiskQuota,
	5929	.ISDIR => return error.IsDir,
	5930	.IO => return error.HardwareFailure,
	5931	.LOOP => return error.SymLinkLoop,
	5932	.MLINK => return error.LinkQuotaExceeded,
	5933	.NAMETOOLONG => return error.NameTooLong,
	5934	.NOENT => return error.FileNotFound,
	5935	.NOTDIR => return error.NotDir,
	5936	.NOMEM => return error.SystemResources,
	5937	.NOSPC => return error.NoSpaceLeft,
	5938	.EXIST => return error.DirNotEmpty,
	5939	.NOTEMPTY => return error.DirNotEmpty,
	5940	.ROFS => return error.ReadOnlyFileSystem,
	5941	.XDEV => return error.CrossDevice,
	5942	.ILSEQ => return error.BadPathName,
	5943	.FAULT => \|err\| return errnoBug(err),
	5944	.INVAL => \|err\| return errnoBug(err),
	5945	else => \|err\| return unexpectedErrno(err),
	5946	}
	5947	}
	5948	}
	5949
	5950	fn setsockopt(
	5951	ev: *Evented,
	5952	cancel_region: *CancelRegion,
	5953	fd: fd_t,
	5954	level: i32,
	5955	opt_name: u32,
	5956	option: u32,
	5957	) !void {
	5958	const o: []const u8 = @ptrCast(&option);
	5959	while (true) {
	5960	const off: extern struct {
	5961	cmd_op: linux.IO_URING_SOCKET_OP,
	5962	pad: u32,
	5963	} align(@alignOf(u64)) = .{
	5964	.cmd_op = .SETSOCKOPT,
	5965	.pad = 0,
	5966	};
	5967	const addr: extern struct { level: i32, opt_name: u32 } align(@alignOf(u64)) = .{
	5968	.level = level,
	5969	.opt_name = opt_name,
	5970	};
	5971	const thread = try cancel_region.awaitIoUring();
	5972	thread.enqueue().* = .{
	5973	.opcode = .URING_CMD,
	5974	.flags = 0,
	5975	.ioprio = 0,
	5976	.fd = fd,
	5977	.off = @as(const u64, @ptrCast(&off)).,
	5978	.addr = @as(const u64, @ptrCast(&addr)).,
	5979	.len = 0,
	5980	.rw_flags = 0,
	5981	.user_data = @intFromPtr(cancel_region.fiber),
	5982	.buf_index = 0,
	5983	.personality = 0,
	5984	.splice_fd_in = @intCast(o.len),
	5985	.addr3 = @intFromPtr(o.ptr),
	5986	.resv = 0,
	5987	};
	5988	ev.yield(null, .nothing);
	5989	switch (cancel_region.errno()) {
	5990	.SUCCESS => return,
	5991	.INTR, .CANCELED => continue,
	5992	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	5993	.NOTSOCK => \|err\| return errnoBug(err),
	5994	.INVAL => \|err\| return errnoBug(err),
	5995	.FAULT => \|err\| return errnoBug(err),
	5996	else => \|err\| return unexpectedErrno(err),
	5997	}
	5998	}
	5999	}
	6000
	6001	fn socket(
	6002	ev: *Evented,
	6003	cancel_region: *CancelRegion,
	6004	family: linux.sa_family_t,
	6005	options: net.IpAddress.BindOptions,
	6006	) error{
	6007	AddressFamilyUnsupported,
	6008	ProtocolUnsupportedBySystem,
	6009	ProcessFdQuotaExceeded,
	6010	SystemFdQuotaExceeded,
	6011	SystemResources,
	6012	ProtocolUnsupportedByAddressFamily,
	6013	SocketModeUnsupported,
	6014	OptionUnsupported,
	6015	Unexpected,
	6016	Canceled,
	6017	}!fd_t {
	6018	const mode = posixSocketMode(options.mode);
	6019	const protocol = posixProtocol(options.protocol);
	6020	const socket_fd = while (true) {
	6021	const thread = try cancel_region.awaitIoUring();
	6022	thread.enqueue().* = .{
	6023	.opcode = .SOCKET,
	6024	.flags = 0,
	6025	.ioprio = 0,
	6026	.fd = family,
	6027	.off = mode \| linux.SOCK.CLOEXEC,
	6028	.addr = 0,
	6029	.len = protocol,
	6030	.rw_flags = 0,
	6031	.user_data = @intFromPtr(cancel_region.fiber),
	6032	.buf_index = 0,
	6033	.personality = 0,
	6034	.splice_fd_in = 0,
	6035	.addr3 = 0,
	6036	.resv = 0,
	6037	};
	6038	ev.yield(null, .nothing);
	6039	const completion = cancel_region.completion();
	6040	switch (completion.errno()) {
	6041	.SUCCESS => break completion.result,
	6042	.INTR, .CANCELED => continue,
	6043	.AFNOSUPPORT => return error.AddressFamilyUnsupported,
	6044	.INVAL => return error.ProtocolUnsupportedBySystem,
	6045	.MFILE => return error.ProcessFdQuotaExceeded,
	6046	.NFILE => return error.SystemFdQuotaExceeded,
	6047	.NOBUFS => return error.SystemResources,
	6048	.NOMEM => return error.SystemResources,
	6049	.PROTONOSUPPORT => return error.ProtocolUnsupportedByAddressFamily,
	6050	.PROTOTYPE => return error.SocketModeUnsupported,
	6051	else => \|err\| return unexpectedErrno(err),
	6052	}
	6053	};
	6054	errdefer ev.close(socket_fd);
	6055
	6056	if (options.ip6_only) {
	6057	if (linux.IPV6 == void) return error.OptionUnsupported;
	6058	try ev.setsockopt(cancel_region, socket_fd, linux.IPPROTO.IPV6, linux.IPV6.V6ONLY, 0);
	6059	}
	6060
	6061	return socket_fd;
	6062	}
	6063
	6064	fn stat(ev: Evented, cancel_region: CancelRegion, fd: fd_t) Dir.StatError!Dir.Stat {
	6065	return ev.statx(cancel_region, fd, "", linux.AT.EMPTY_PATH) catch \|err\| switch (err) {
	6066	error.BadPathName, error.NameTooLong => unreachable, // path is empty
	6067	error.AccessDenied => return errnoBug(.ACCES),
	6068	error.SymLinkLoop => return errnoBug(.LOOP),
	6069	error.FileNotFound => return errnoBug(.NOENT),
	6070	error.NotDir => return errnoBug(.NOTDIR),
	6071	else => \|e\| return e,
	6072	};
	6073	}
	6074
	6075	fn statx(
	6076	ev: *Evented,
	6077	cancel_region: *CancelRegion,
	6078	dir: fd_t,
	6079	path: [*:0]const u8,
	6080	flags: u32,
	6081	) (Dir.StatError \|\| Dir.PathNameError \|\| error{ FileNotFound, NotDir, SymLinkLoop })!Dir.Stat {
	6082	while (true) {
	6083	var statx_buf = std.mem.zeroes(linux.Statx);
	6084	const thread = try cancel_region.awaitIoUring();
	6085	thread.enqueue().* = .{
	6086	.opcode = .STATX,
	6087	.flags = 0,
	6088	.ioprio = 0,
	6089	.fd = dir,
	6090	.off = @intFromPtr(&statx_buf),
	6091	.addr = @intFromPtr(path),
	6092	.len = @bitCast(linux_statx_request),
	6093	.rw_flags = flags,
	6094	.user_data = @intFromPtr(cancel_region.fiber),
	6095	.buf_index = 0,
	6096	.personality = 0,
	6097	.splice_fd_in = 0,
	6098	.addr3 = 0,
	6099	.resv = 0,
	6100	};
	6101	ev.yield(null, .nothing);
	6102	switch (cancel_region.errno()) {
	6103	.SUCCESS => return statFromLinux(&statx_buf),
	6104	.INTR, .CANCELED => continue,
	6105	.ACCES => return error.AccessDenied,
	6106	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	6107	.FAULT => \|err\| return errnoBug(err),
	6108	.INVAL => \|err\| return errnoBug(err),
	6109	.LOOP => return error.SymLinkLoop,
	6110	.NAMETOOLONG => \|err\| return errnoBug(err),
	6111	.NOENT => return error.FileNotFound,
	6112	.NOTDIR => return error.NotDir,
	6113	.NOMEM => return error.SystemResources,
	6114	else => \|err\| return unexpectedErrno(err),
	6115	}
	6116	}
	6117	}
	6118
	6119	fn urandomReadAll(
	6120	ev: *Evented,
	6121	cancel_region: *CancelRegion,
	6122	buffer: []u8,
	6123	) (File.OpenError \|\| File.Reader.Error \|\| error{EndOfStream})!void {
	6124	return ev.readAll(cancel_region, try ev.random_fd.open(ev, cancel_region, "/dev/urandom", .{
	6125	.ACCMODE = .RDONLY,
	6126	.CLOEXEC = true,
	6127	}), buffer);
	6128	}
	6129
	6130	fn utimensat(
	6131	ev: *Evented,
	6132	sync: *CancelRegion.Sync,
	6133	dir: fd_t,
	6134	path: [*:0]const u8,
	6135	times: ?*const [2]linux.timespec,
	6136	flags: u32,
	6137	) File.SetTimestampsError!void {
	6138	_ = ev;
	6139	while (true) {
	6140	try sync.cancel_region.await(.nothing);
	6141	switch (linux.errno(linux.utimensat(dir, path, times, flags))) {
	6142	.SUCCESS => return,
	6143	.INTR => continue,
	6144	.BADF => \|err\| return errnoBug(err), // always a race condition
	6145	.FAULT => \|err\| return errnoBug(err),
	6146	.INVAL => \|err\| return errnoBug(err),
	6147	.ACCES => return error.AccessDenied,
	6148	.PERM => return error.PermissionDenied,
	6149	.ROFS => return error.ReadOnlyFileSystem,
	6150	else => \|err\| return unexpectedErrno(err),
	6151	}
	6152	}
	6153	}
	6154
	6155	fn writeAll(
	6156	ev: *Evented,
	6157	cancel_region: *CancelRegion,
	6158	fd: fd_t,
	6159	buffer: []const u8,
	6160	) (File.Writer.Error \|\| error{EndOfStream})!void {
	6161	var index: usize = 0;
	6162	while (buffer.len - index != 0) {
	6163	const len = try ev.pwritev(cancel_region, fd, &.{
	6164	.{ .base = buffer[index..].ptr, .len = buffer.len - index },
	6165	}, null);
	6166	if (len == 0) return error.EndOfStream;
	6167	index += len;
	6168	}
	6169	}
	6170
	6171	test {
	6172	_ = Fiber.CancelProtection;
	6173	}

lib/std/Io/fiber.zig created+201

...	...	@@ -0,0 +1,201 @@
	1	pub const supported = switch (builtin.cpu.arch) {
	2	.aarch64, .x86_64 => true,
	3	else => false,
	4	};
	5
	6	/// Stores the cpu state of an inactive fiber.
	7	pub const Context = switch (builtin.cpu.arch) {
	8	.aarch64 => extern struct {
	9	sp: u64,
	10	fp: u64,
	11	pc: u64,
	12	},
	13	.x86_64 => extern struct {
	14	rsp: u64,
	15	rbp: u64,
	16	rip: u64,
	17	},
	18	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
	19	};
	20
	21	pub const Switch = extern struct { old: Context, new: Context };
	22
	23	/// Fills `s.old` with the current cpu state, and restores the cpu state stored in `s.new`.
	24	pub inline fn contextSwitch(s: const Switch) const Switch {
	25	return switch (builtin.cpu.arch) {
	26	.aarch64 => asm volatile (
	27	\\ ldp x0, x2, [x1]
	28	\\ ldr x3, [x2, #16]
	29	\\ mov x4, sp
	30	\\ stp x4, fp, [x0]
	31	\\ adr x5, 0f
	32	\\ ldp x4, fp, [x2]
	33	\\ str x5, [x0, #16]
	34	\\ mov sp, x4
	35	\\ br x3
	36	\\0:
	37	: [received_message] "={x1}" (-> *const Switch),
	38	: [message_to_send] "{x1}" (s),
	39	: .{
	40	.x0 = true,
	41	.x1 = true,
	42	.x2 = true,
	43	.x3 = true,
	44	.x4 = true,
	45	.x5 = true,
	46	.x6 = true,
	47	.x7 = true,
	48	.x8 = true,
	49	.x9 = true,
	50	.x10 = true,
	51	.x11 = true,
	52	.x12 = true,
	53	.x13 = true,
	54	.x14 = true,
	55	.x15 = true,
	56	.x16 = true,
	57	.x17 = true,
	58	.x19 = true,
	59	.x20 = true,
	60	.x21 = true,
	61	.x22 = true,
	62	.x23 = true,
	63	.x24 = true,
	64	.x25 = true,
	65	.x26 = true,
	66	.x27 = true,
	67	.x28 = true,
	68	.x30 = true,
	69	.z0 = true,
	70	.z1 = true,
	71	.z2 = true,
	72	.z3 = true,
	73	.z4 = true,
	74	.z5 = true,
	75	.z6 = true,
	76	.z7 = true,
	77	.z8 = true,
	78	.z9 = true,
	79	.z10 = true,
	80	.z11 = true,
	81	.z12 = true,
	82	.z13 = true,
	83	.z14 = true,
	84	.z15 = true,
	85	.z16 = true,
	86	.z17 = true,
	87	.z18 = true,
	88	.z19 = true,
	89	.z20 = true,
	90	.z21 = true,
	91	.z22 = true,
	92	.z23 = true,
	93	.z24 = true,
	94	.z25 = true,
	95	.z26 = true,
	96	.z27 = true,
	97	.z28 = true,
	98	.z29 = true,
	99	.z30 = true,
	100	.z31 = true,
	101	.p0 = true,
	102	.p1 = true,
	103	.p2 = true,
	104	.p3 = true,
	105	.p4 = true,
	106	.p5 = true,
	107	.p6 = true,
	108	.p7 = true,
	109	.p8 = true,
	110	.p9 = true,
	111	.p10 = true,
	112	.p11 = true,
	113	.p12 = true,
	114	.p13 = true,
	115	.p14 = true,
	116	.p15 = true,
	117	.fpcr = true,
	118	.fpsr = true,
	119	.ffr = true,
	120	.memory = true,
	121	}),
	122	.x86_64 => asm volatile (
	123	\\ movq 0(%%rsi), %%rax
	124	\\ movq 8(%%rsi), %%rcx
	125	\\ leaq 0f(%%rip), %%rdx
	126	\\ movq %%rsp, 0(%%rax)
	127	\\ movq %%rbp, 8(%%rax)
	128	\\ movq %%rdx, 16(%%rax)
	129	\\ movq 0(%%rcx), %%rsp
	130	\\ movq 8(%%rcx), %%rbp
	131	\\ jmpq *16(%%rcx)
	132	\\0:
	133	: [received_message] "={rsi}" (-> *const Switch),
	134	: [message_to_send] "{rsi}" (s),
	135	: .{
	136	.rax = true,
	137	.rcx = true,
	138	.rdx = true,
	139	.rbx = true,
	140	.rsi = true,
	141	.rdi = true,
	142	.r8 = true,
	143	.r9 = true,
	144	.r10 = true,
	145	.r11 = true,
	146	.r12 = true,
	147	.r13 = true,
	148	.r14 = true,
	149	.r15 = true,
	150	.mm0 = true,
	151	.mm1 = true,
	152	.mm2 = true,
	153	.mm3 = true,
	154	.mm4 = true,
	155	.mm5 = true,
	156	.mm6 = true,
	157	.mm7 = true,
	158	.zmm0 = true,
	159	.zmm1 = true,
	160	.zmm2 = true,
	161	.zmm3 = true,
	162	.zmm4 = true,
	163	.zmm5 = true,
	164	.zmm6 = true,
	165	.zmm7 = true,
	166	.zmm8 = true,
	167	.zmm9 = true,
	168	.zmm10 = true,
	169	.zmm11 = true,
	170	.zmm12 = true,
	171	.zmm13 = true,
	172	.zmm14 = true,
	173	.zmm15 = true,
	174	.zmm16 = true,
	175	.zmm17 = true,
	176	.zmm18 = true,
	177	.zmm19 = true,
	178	.zmm20 = true,
	179	.zmm21 = true,
	180	.zmm22 = true,
	181	.zmm23 = true,
	182	.zmm24 = true,
	183	.zmm25 = true,
	184	.zmm26 = true,
	185	.zmm27 = true,
	186	.zmm28 = true,
	187	.zmm29 = true,
	188	.zmm30 = true,
	189	.zmm31 = true,
	190	.fpsr = true,
	191	.fpcr = true,
	192	.mxcsr = true,
	193	.rflags = true,
	194	.dirflag = true,
	195	.memory = true,
	196	}),
	197	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
	198	};
	199	}
	200
	201	const builtin = @import("builtin");

lib/std/c.zig+3-10

...	...	@@ -10724,6 +10724,7 @@ pub extern "c" fn chmod(path: [*:0]const u8, mode: mode_t) c_int;
10724	10724	pub extern "c" fn fchmod(fd: fd_t, mode: mode_t) c_int;
10725	10725	pub extern "c" fn fchmodat(fd: fd_t, path: [*:0]const u8, mode: mode_t, flags: c_uint) c_int;
10726	10726	pub extern "c" fn fchown(fd: fd_t, owner: uid_t, group: gid_t) c_int;
	10727	pub extern "c" fn fchownat(fd: fd_t, path: [*:0]const u8, owner: uid_t, group: gid_t, flags: c_uint) c_int;
10727	10728	pub extern "c" fn umask(mode: mode_t) mode_t;
10728	10729
10729	10730	pub extern "c" fn rmdir(path: [*:0]const u8) c_int;
...	...	@@ -10864,6 +10865,7 @@ pub const pthread_setname_np = switch (native_os) {
10864	10865
10865	10866	pub extern "c" fn pthread_getname_np(thread: pthread_t, name: [*:0]u8, len: usize) c_int;
10866	10867	pub extern "c" fn pthread_kill(pthread_t, signal: SIG) c_int;
	10868	pub extern "c" fn pthread_exit(ptr: ?*anyopaque) noreturn;
10867	10869
10868	10870	pub const pthread_threadid_np = switch (native_os) {
10869	10871	.driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => private.pthread_threadid_np,
...	...	@@ -11296,16 +11298,7 @@ pub const clock_get_time = darwin.clock_get_time;
11296	11298	pub const clock_serv_t = darwin.clock_serv_t;
11297	11299	pub const clock_res_t = darwin.clock_res_t;
11298	11300	pub const @"close$NOCANCEL" = darwin.@"close$NOCANCEL";
11299		pub const dispatch_function_t = darwin.dispatch_function_t;
11300		pub const dispatch_once_f = darwin.dispatch_once_f;
11301		pub const dispatch_once_t = darwin.dispatch_once_t;
11302		pub const dispatch_release = darwin.dispatch_release;
11303		pub const dispatch_semaphore_create = darwin.dispatch_semaphore_create;
11304		pub const dispatch_semaphore_signal = darwin.dispatch_semaphore_signal;
11305		pub const dispatch_semaphore_t = darwin.dispatch_semaphore_t;
11306		pub const dispatch_semaphore_wait = darwin.dispatch_semaphore_wait;
11307		pub const dispatch_time = darwin.dispatch_time;
11308		pub const dispatch_time_t = darwin.dispatch_time_t;
	11301	pub const dispatch = darwin.dispatch;
11309	11302	pub const fcopyfile = darwin.fcopyfile;
11310	11303	pub const host_t = darwin.host_t;
11311	11304	pub const integer_t = darwin.integer_t;

lib/std/c/darwin.zig+3-25

...	...	@@ -18,6 +18,9 @@ comptime {
18	18	assert(builtin.os.tag.isDarwin()); // Prevent access of std.c symbols on wrong OS.
19	19	}
20	20
	21	// Grand Central Dispatch is exposed by libSystem.
	22	pub const dispatch = @import("darwin/dispatch.zig");
	23
21	24	pub const mach_port_t = c_uint;
22	25
23	26	pub const EXC = enum(exception_type_t) {
...	...	@@ -896,31 +899,6 @@ pub const qos_class_t = enum(c_uint) {
896	899	_,
897	900	};
898	901
899		// Grand Central Dispatch is exposed by libSystem.
900		pub extern "c" fn dispatch_release(object: *anyopaque) void;
901
902		pub const dispatch_semaphore_t = *opaque {};
903		pub extern "c" fn dispatch_semaphore_create(value: isize) ?dispatch_semaphore_t;
904		pub extern "c" fn dispatch_semaphore_wait(dsema: dispatch_semaphore_t, timeout: dispatch_time_t) isize;
905		pub extern "c" fn dispatch_semaphore_signal(dsema: dispatch_semaphore_t) isize;
906
907		pub const DISPATCH_TIME_NOW = @compileError("use dispatch_time_t.NOW");
908		pub const DISPATCH_TIME_FOREVER = @compileError("use dispatch_time_t.FOREVER");
909		pub const dispatch_time_t = enum(u64) {
910		NOW = 0,
911		FOREVER = ~0,
912		_,
913		};
914		pub extern "c" fn dispatch_time(when: dispatch_time_t, delta: i64) dispatch_time_t;
915
916		pub const dispatch_once_t = usize;
917		pub const dispatch_function_t = fn (?*anyopaque) callconv(.c) void;
918		pub extern fn dispatch_once_f(
919		predicate: *dispatch_once_t,
920		context: ?*anyopaque,
921		function: dispatch_function_t,
922		) void;
923
924	902	/// Undocumented futex-like API available on darwin 16+
925	903	/// (macOS 10.12+, iOS 10.0+, tvOS 10.0+, watchOS 3.0+, catalyst 13.0+).
926	904	///

lib/std/c/darwin/dispatch.zig created+308

...	...	@@ -0,0 +1,308 @@
	1	// dispatch/base.h
	2	pub const function_t = const fn (?anyopaque) callconv(.c) void;
	3
	4	// dispatch/object.h
	5	pub const object_t = *_os_object_s;
	6	pub const retain = dispatch_retain;
	7	pub const release = dispatch_release;
	8	pub const get_context = dispatch_get_context;
	9	pub const set_context = dispatch_set_context;
	10	pub const set_finalizer_f = dispatch_set_finalizer_f;
	11	pub const activate = dispatch_activate;
	12	pub const @"suspend" = dispatch_suspend;
	13	pub const @"resume" = dispatch_resume;
	14
	15	const _os_object_s = opaque {
	16	pub const retain = dispatch_retain;
	17	pub const release = dispatch_release;
	18	pub const get_context = dispatch_get_context;
	19	pub const set_context = dispatch_set_context;
	20	pub const set_finalizer = dispatch_set_finalizer_f;
	21	pub const activate = dispatch_activate;
	22	pub const @"suspend" = dispatch_suspend;
	23	pub const @"resume" = dispatch_resume;
	24	pub const set_target_queue = dispatch_set_target_queue;
	25	};
	26	extern "c" fn dispatch_retain(object: object_t) void;
	27	extern "c" fn dispatch_release(object: object_t) void;
	28	extern "c" fn dispatch_get_context(object: object_t) ?*anyopaque;
	29	extern "c" fn dispatch_set_context(object: object_t, context: ?*anyopaque) void;
	30	extern "c" fn dispatch_set_finalizer_f(object: object_t, finalizer: ?function_t) void;
	31	extern "c" fn dispatch_activate(object: object_t) void;
	32	extern "c" fn dispatch_suspend(object: object_t) void;
	33	extern "c" fn dispatch_resume(object: object_t) void;
	34
	35	// dispatch/once.h
	36	pub const once_t = enum(isize) {
	37	init = 0,
	38	done = -1,
	39	_,
	40
	41	pub inline fn once(predicate: once_t, context: ?anyopaque, function: function_t) void {
	42	if (@atomicLoad(once_t, predicate, .unordered) != .done) {
	43	@branchHint(.unlikely);
	44	once_f(predicate, context, function);
	45	} else asm volatile ("" ::: .{ .memory = true });
	46	switch (builtin.mode) {
	47	.Debug, .ReleaseSafe => {},
	48	.ReleaseFast, .ReleaseSmall => if (@atomicLoad(once_t, predicate, .unordered) != .done)
	49	unreachable,
	50	}
	51	}
	52	};
	53	pub const once_f = dispatch_once_f;
	54
	55	extern "c" fn dispatch_once_f(predicate: once_t, context: ?anyopaque, function: function_t) void;
	56
	57	// dispatch/queue.h
	58	pub const queue_t = *queue_s;
	59	pub const queue_global_t = queue_t;
	60	pub const queue_serial_executor_t = queue_t;
	61	pub const queue_serial_t = queue_t;
	62	pub const queue_main_t = queue_serial_t;
	63	pub const queue_concurrent_t = queue_t;
	64	pub const async_f = dispatch_async_f;
	65	pub const sync_f = dispatch_sync_f;
	66	pub const async_and_wait_f = dispatch_async_and_wait_f;
	67	pub const apply_f = dispatch_apply_f;
	68	pub const get_current_queue = dispatch_get_current_queue;
	69	pub inline fn get_main_queue() queue_main_t {
	70	return &_dispatch_main_q;
	71	}
	72	pub const queue_priority_t = enum(c_long) {
	73	HIGH = 2,
	74	DEFAULT = 0,
	75	LOW = -1,
	76	BACKGROUND = std.math.minInt(i16),
	77	_,
	78	};
	79	pub const get_global_queue = dispatch_get_global_queue;
	80	pub const queue_attr_t = ?*queue_attr_s;
	81	pub inline fn QUEUE_SERIAL() queue_attr_t {
	82	return null;
	83	}
	84	pub inline fn QUEUE_INACTIVE() queue_attr_t {
	85	return queue_attr_make_initially_inactive(QUEUE_SERIAL());
	86	}
	87	pub inline fn QUEUE_CONCURRENT() queue_attr_t {
	88	return &_dispatch_queue_attr_concurrent;
	89	}
	90	pub inline fn QUEUE_CONCURRENT_INACTIVE() queue_attr_t {
	91	return queue_attr_make_initially_inactive(QUEUE_CONCURRENT());
	92	}
	93	pub const queue_attr_make_initially_inactive = dispatch_queue_attr_make_initially_inactive;
	94	pub const TARGET_QUEUE_DEFAULT: ?queue_t = null;
	95	pub const queue_create_with_target = dispatch_queue_create_with_target;
	96	pub const queue_create = dispatch_queue_create;
	97	pub const CURRENT_QUEUE_LABEL: ?[*:0]const u8 = null;
	98	pub const queue_get_label = dispatch_queue_get_label;
	99	pub const main = dispatch_main;
	100	pub const after_f = dispatch_after_f;
	101
	102	const queue_s = opaque {
	103	pub inline fn as_object(queue: queue_t) object_t {
	104	return @ptrCast(queue);
	105	}
	106	pub const async = async_f;
	107	pub const sync = sync_f;
	108	pub const async_and_wait = async_and_wait_f;
	109	pub const apply = apply_f;
	110	pub const get_current = get_current_queue;
	111	pub const get_main = get_main_queue;
	112	pub const get_global = get_global_queue;
	113	pub const create_with_target = queue_create_with_target;
	114	pub const create = queue_create;
	115	pub const get_label = queue_get_label;
	116	};
	117	extern "c" fn dispatch_async_f(queue: queue_t, context: ?*anyopaque, work: function_t) void;
	118	extern "c" fn dispatch_sync_f(queue: queue_t, context: ?*anyopaque, work: function_t) void;
	119	extern "c" fn dispatch_async_and_wait_f(queue: queue_t, context: ?*anyopaque, work: function_t) void;
	120	extern "c" fn dispatch_apply_f(iterations: usize, queue: ?queue_t, context: ?anyopaque, work: const fn (context: ?*anyopaque, iteration: usize) callconv(.c) void) void;
	121	extern "c" fn dispatch_get_current_queue() queue_t;
	122	extern "c" var _dispatch_main_q: queue_s;
	123	extern "c" fn dispatch_get_global_queue(identifier: isize, flags: usize) queue_global_t;
	124	const queue_attr_s = opaque {
	125	pub inline fn as_object(queue_attr: queue_attr_t) object_t {
	126	return @ptrCast(queue_attr);
	127	}
	128	pub const SERIAL = QUEUE_SERIAL;
	129	pub const INACTIVE = QUEUE_INACTIVE;
	130	pub const CONCURRENT = QUEUE_CONCURRENT;
	131	pub const CONCURRENT_INACTIVE = QUEUE_CONCURRENT_INACTIVE;
	132	};
	133	extern "c" var _dispatch_queue_attr_concurrent: queue_attr_s;
	134	extern "c" fn dispatch_queue_attr_make_initially_inactive(attr: queue_attr_t) queue_attr_t;
	135	extern "c" fn dispatch_queue_create_with_target(label: ?[*:0]const u8, attr: queue_attr_t, target: ?queue_t) ?queue_t;
	136	extern "c" fn dispatch_queue_create(label: ?[*:0]const u8, attr: queue_attr_t) ?queue_t;
	137	extern "c" fn dispatch_queue_get_label(queue: ?queue_t) [*:0]const u8;
	138	extern "c" fn dispatch_set_target_queue(object: object_t, queue: ?queue_t) void;
	139	extern "c" fn dispatch_main() noreturn;
	140	extern "c" fn dispatch_after_f(when: time_t, queue: queue_t, context: ?*anyopaque, work: function_t) void;
	141
	142	// dispatch/semaphore.h
	143	pub const semaphore_t = *semaphore_s;
	144	pub const semaphore_create = dispatch_semaphore_create;
	145	pub const semaphore_wait = dispatch_semaphore_wait;
	146	pub const semaphore_signal = dispatch_semaphore_signal;
	147
	148	const semaphore_s = opaque {
	149	pub inline fn as_object(semaphore: semaphore_t) object_t {
	150	return @ptrCast(semaphore);
	151	}
	152	pub const create = semaphore_create;
	153	pub const wait = semaphore_wait;
	154	pub const signal = semaphore_signal;
	155	};
	156	extern "c" fn dispatch_semaphore_create(value: isize) ?semaphore_t;
	157	extern "c" fn dispatch_semaphore_wait(dsema: semaphore_t, timeout: time_t) isize;
	158	extern "c" fn dispatch_semaphore_signal(dsema: semaphore_t) isize;
	159
	160	// dispatch/source.h
	161	pub const source_t = *source_s;
	162	pub const source_type_t = *const source_type_s;
	163	pub const SOURCE_TYPE_DATA_ADD = &_dispatch_source_type_data_add;
	164	pub const SOURCE_TYPE_DATA_OR = &_dispatch_source_type_data_or;
	165	pub const SOURCE_TYPE_DATA_REPLACE = &_dispatch_source_type_data_replace;
	166	pub const SOURCE_TYPE_MACH_SEND = &_dispatch_source_type_mach_send;
	167	pub const SOURCE_TYPE_MACH_RECV = &_dispatch_source_type_mach_recv;
	168	pub const SOURCE_TYPE_MEMORYPRESSURE = &_dispatch_source_type_memorypressure;
	169	pub const SOURCE_TYPE_PROC = &_dispatch_source_type_proc;
	170	pub const SOURCE_TYPE_READ = &_dispatch_source_type_read;
	171	pub const SOURCE_TYPE_SIGNAL = &_dispatch_source_type_signal;
	172	pub const SOURCE_TYPE_TIMER = &_dispatch_source_type_timer;
	173	pub const SOURCE_TYPE_VNODE = &_dispatch_source_type_vnode;
	174	pub const SOURCE_TYPE_WRITE = &_dispatch_source_type_write;
	175	pub const source_mach_send_flags_t = packed struct(usize) {
	176	DEAD: bool = false,
	177	unused1: @Int(.unsigned, @bitSizeOf(usize) - 1) = 0,
	178	};
	179	pub const source_mach_recv_flags_t = packed struct(usize) {
	180	unused0: @Int(.unsigned, @bitSizeOf(usize) - 0) = 0,
	181	};
	182	pub const source_memorypressure_flags_t = packed struct(usize) {
	183	NORMAL: bool = false,
	184	WARN: bool = false,
	185	CRITICAL: bool = false,
	186	unused3: @Int(.unsigned, @bitSizeOf(usize) - 3) = 0,
	187	};
	188	pub const source_proc_flags_t = packed struct(usize) {
	189	unused0: u27 = 0,
	190	SIGNAL: bool = false,
	191	unused28: u1 = 0,
	192	EXEC: bool = false,
	193	FORK: bool = false,
	194	EXIT: bool = false,
	195	unused32: @Int(.unsigned, @bitSizeOf(usize) - 32) = 0,
	196	};
	197	pub const source_vnode_flags_t = packed struct(usize) {
	198	DELETE: bool = false,
	199	WRITE: bool = false,
	200	EXTEND: bool = false,
	201	ATTRIB: bool = false,
	202	LINK: bool = false,
	203	RENAME: bool = false,
	204	REVOKE: bool = false,
	205	unused7: u1 = 0,
	206	FUNLOCK: bool = false,
	207	unused9: @Int(.unsigned, @bitSizeOf(usize) - 9) = 0,
	208	};
	209	pub const source_timer_flags_t = packed struct(usize) {
	210	STRICT: bool = false,
	211	unused1: @Int(.unsigned, @bitSizeOf(usize) - 1) = 0,
	212	};
	213	pub const source_flags_t = packed union {
	214	raw: usize,
	215	MACH_SEND: source_mach_send_flags_t,
	216	MACH_RECV: source_mach_recv_flags_t,
	217	MEMORYPRESSURE: source_memorypressure_flags_t,
	218	PROC: source_proc_flags_t,
	219	VNODE: source_vnode_flags_t,
	220	pub const none: source_flags_t = .{ .raw = 0 };
	221	};
	222	pub const source_create = dispatch_source_create;
	223	pub const source_set_event_handler_f = dispatch_source_set_event_handler_f;
	224	pub const source_set_cancel_handler_f = dispatch_source_set_cancel_handler_f;
	225	pub const source_cancel = dispatch_source_cancel;
	226	pub const source_testcancel = dispatch_source_testcancel;
	227	pub const source_get_handle = dispatch_source_get_handle;
	228	pub const source_get_mask = dispatch_source_get_mask;
	229	pub const source_get_data = dispatch_source_get_data;
	230	pub const source_merge_data = dispatch_source_merge_data;
	231	pub const source_set_timer = dispatch_source_set_timer;
	232	pub const source_set_registration_handler_f = dispatch_source_set_registration_handler_f;
	233
	234	const source_s = opaque {
	235	pub inline fn as_object(source: source_t) object_t {
	236	return @ptrCast(source);
	237	}
	238	pub const set_event_handler = source_set_event_handler_f;
	239	pub const set_cancel_handler = source_set_cancel_handler_f;
	240	pub const cancel = source_cancel;
	241	pub const testcancel = source_testcancel;
	242	pub const get_handle = source_get_handle;
	243	pub const get_mask = source_get_mask;
	244	pub const get_data = source_get_data;
	245	pub const merge_data = source_merge_data;
	246	pub const set_timer = source_set_timer;
	247	pub const set_registration_handler = source_set_registration_handler_f;
	248	};
	249	const source_type_s = opaque {
	250	pub const DATA_ADD = SOURCE_TYPE_DATA_ADD;
	251	pub const DATA_OR = SOURCE_TYPE_DATA_OR;
	252	pub const DATA_REPLACE = SOURCE_TYPE_DATA_REPLACE;
	253	pub const MACH_SEND = SOURCE_TYPE_MACH_SEND;
	254	pub const MACH_RECV = SOURCE_TYPE_MACH_RECV;
	255	pub const MEMORYPRESSURE = SOURCE_TYPE_MEMORYPRESSURE;
	256	pub const PROC = SOURCE_TYPE_PROC;
	257	pub const READ = SOURCE_TYPE_READ;
	258	pub const SIGNAL = SOURCE_TYPE_SIGNAL;
	259	pub const TIMER = SOURCE_TYPE_TIMER;
	260	pub const VNODE = SOURCE_TYPE_VNODE;
	261	pub const WRITE = SOURCE_TYPE_WRITE;
	262	};
	263	extern "c" const _dispatch_source_type_data_add: source_type_s;
	264	extern "c" const _dispatch_source_type_data_or: source_type_s;
	265	extern "c" const _dispatch_source_type_data_replace: source_type_s;
	266	extern "c" const _dispatch_source_type_mach_send: source_type_s;
	267	extern "c" const _dispatch_source_type_mach_recv: source_type_s;
	268	extern "c" const _dispatch_source_type_memorypressure: source_type_s;
	269	extern "c" const _dispatch_source_type_proc: source_type_s;
	270	extern "c" const _dispatch_source_type_read: source_type_s;
	271	extern "c" const _dispatch_source_type_signal: source_type_s;
	272	extern "c" const _dispatch_source_type_timer: source_type_s;
	273	extern "c" const _dispatch_source_type_vnode: source_type_s;
	274	extern "c" const _dispatch_source_type_write: source_type_s;
	275	extern "c" fn dispatch_source_create(type: source_type_t, handle: usize, mask: source_flags_t, queue: ?queue_t) ?source_t;
	276	extern "c" fn dispatch_source_set_event_handler_f(source: source_t, handler: ?function_t) void;
	277	extern "c" fn dispatch_source_set_cancel_handler_f(source: source_t, handler: ?function_t) void;
	278	extern "c" fn dispatch_source_cancel(source: source_t) void;
	279	extern "c" fn dispatch_source_testcancel(source: source_t) isize;
	280	extern "c" fn dispatch_source_get_handle(source: source_t) usize;
	281	extern "c" fn dispatch_source_get_mask(source: source_t) source_flags_t;
	282	extern "c" fn dispatch_source_get_data(source: source_t) usize;
	283	extern "c" fn dispatch_source_merge_data(source: source_t, value: usize) void;
	284	extern "c" fn dispatch_source_set_timer(source: source_t, start: time_t, interval: u64, leeway: u64) void;
	285	extern "c" fn dispatch_source_set_registration_handler_f(source: source_t, handler: ?function_t) void;
	286
	287	// dispatch/time.h
	288	pub const time_t = enum(u64) {
	289	WALL_NOW = WALLTIME_NOW,
	290	NOW = TIME_NOW,
	291	FOREVER = TIME_FOREVER,
	292	_,
	293
	294	pub const time = dispatch_time;
	295	pub const walltime = dispatch_walltime;
	296	pub const after = dispatch_after_f;
	297	};
	298	pub const WALLTIME_NOW = ~@as(u64, 1);
	299	pub const TIME_NOW: u64 = 0;
	300	pub const TIME_FOREVER = ~@as(u64, 0);
	301	pub const time = dispatch_time;
	302	pub const walltime = dispatch_walltime;
	303
	304	extern "c" fn dispatch_time(when: time_t, delta: i64) time_t;
	305	extern "c" fn dispatch_walltime(when: ?*const std.c.timespec, delta: i64) time_t;
	306
	307	const builtin = @import("builtin");
	308	const std = @import("std");

lib/std/debug.zig+2-6

...	...	@@ -533,9 +533,7 @@ pub fn defaultPanic(msg: []const u8, first_trace_addr: ?usize) noreturn {
533	533	else => {},
534	534	}
535	535
536		// Don't try to cancel during a panic. No need to re-enable cancelation,
537		// because the panic handler doesn't return.
538		_ = std.Options.debug_io.swapCancelProtection(.blocked);
	536	std.Options.debug_io.vtable.crashHandler(std.Options.debug_io.userdata);
539	537
540	538	if (enable_segfault_handler) {
541	539	// If a segfault happens while panicking, we want it to actually segfault, not trigger
...	...	@@ -1535,9 +1533,7 @@ fn handleSegfault(addr: ?usize, name: []const u8, opt_ctx: ?CpuContextPtr) noret
1535	1533	}
1536	1534
1537	1535	pub fn defaultHandleSegfault(addr: ?usize, name: []const u8, opt_ctx: ?CpuContextPtr) noreturn {
1538		// Don't try to cancel during a segfault. No need to re-enable cancelation,
1539		// because the segfault handler doesn't return.
1540		_ = std.Options.debug_io.swapCancelProtection(.blocked);
	1536	std.Options.debug_io.vtable.crashHandler(std.Options.debug_io.userdata);
1541	1537
1542	1538	// There is very similar logic to the following in `defaultPanic`.
1543	1539	switch (panic_stage) {

lib/std/process/Environ.zig+12

...	...	@@ -39,6 +39,10 @@ pub const GlobalBlock = struct {
39	39	pub const global: GlobalBlock = .{ .use_global = true };
40	40
41	41	pub fn deinit(_: GlobalBlock, _: Allocator) void {}
	42
	43	pub fn isEmpty(block: GlobalBlock) bool {
	44	return !block.use_global;
	45	}
42	46	};
43	47
44	48	pub const PosixBlock = struct {
...	...	@@ -51,6 +55,10 @@ pub const PosixBlock = struct {
51	55	gpa.free(block.slice);
52	56	}
53	57
	58	pub fn isEmpty(block: PosixBlock) bool {
	59	return block.slice.len == 0;
	60	}
	61
54	62	pub const View = struct {
55	63	slice: []const [*:0]const u8,
56	64
...	...	@@ -72,6 +80,10 @@ pub const WindowsBlock = struct {
72	80	gpa.free(block.slice);
73	81	}
74	82
	83	pub fn isEmpty(block: WindowsBlock) bool {
	84	return block.slice[0] == 0;
	85	}
	86
75	87	pub const View = struct {
76	88	ptr: [*:0]const u16,
77	89

src/crash_report.zig+25-5

...	...	@@ -1,13 +1,18 @@
	1	pub const enabled = switch (build_options.io_mode) {
	2	.threaded => build_options.enable_debug_extensions,
	3	.evented => false, // would use threadlocals in a way incompatible with evented
	4	};
	5
1	6	/// We override the panic implementation to our own one, so we can print our own information before
2	7	/// calling the default panic handler. This declaration must be re-exposed from `@import("root")`.
3		pub const panic = std.debug.FullPanic(panicImpl);
	8	pub const panic = std.debug.FullPanic(if (enabled) panicImpl else std.debug.defaultPanic);
4	9
5	10	/// We let std install its segfault handler, but we override the target-agnostic handler it calls,
6	11	/// so we can print our own information before calling the default segfault logic. This declaration
7	12	/// must be re-exposed from `@import("root")`.
8		pub const debug = struct {
	13	pub const debug = if (enabled) struct {
9	14	pub const handleSegfault = handleSegfaultImpl;
10		};
	15	} else struct {};
11	16
12	17	/// Printed in panic messages when suggesting a command to run, allowing copy-pasting the command.
13	18	/// Set by `main` as soon as arguments are known. The value here is a default in case we somehow
...	...	@@ -25,7 +30,7 @@ fn panicImpl(msg: []const u8, first_trace_addr: ?usize) noreturn {
25	30	std.debug.defaultPanic(msg, first_trace_addr orelse @returnAddress());
26	31	}
27	32
28		pub const AnalyzeBody = struct {
	33	pub const AnalyzeBody = if (enabled) struct {
29	34	parent: ?*AnalyzeBody,
30	35	sema: *Sema,
31	36	block: *Sema.Block,
...	...	@@ -52,9 +57,15 @@ pub const AnalyzeBody = struct {
52	57	std.debug.assert(current.? == ab); // `Sema.analyzeBodyInner` did not match push/pop calls
53	58	current = ab.parent;
54	59	}
	60	} else struct {
	61	const current: ?noreturn = null;
	62	// Dummy implementation, with functions marked `inline` to avoid interfering with tail calls.
	63	pub inline fn push(_: AnalyzeBody, _: Sema, _: Sema.Block, _: []const Zir.Inst.Index) void {}
	64	pub inline fn pop(_: AnalyzeBody) void {}
	65	pub inline fn setBodyIndex(_: @This(), _: usize) void {}
55	66	};
56	67
57		pub const CodegenFunc = struct {
	68	pub const CodegenFunc = if (enabled) struct {
58	69	zcu: *const Zcu,
59	70	func_index: InternPool.Index,
60	71	threadlocal var current: ?CodegenFunc = null;
...	...	@@ -66,6 +77,11 @@ pub const CodegenFunc = struct {
66	77	std.debug.assert(current.?.func_index == func_index);
67	78	current = null;
68	79	}
	80	} else struct {
	81	const current: ?noreturn = null;
	82	// Dummy implementation
	83	pub fn start(_: *const Zcu, _: InternPool.Index) void {}
	84	pub fn stop(_: InternPool.Index) void {}
69	85	};
70	86
71	87	fn dumpCrashContext() Io.Writer.Error!void {
...	...	@@ -79,6 +95,8 @@ fn dumpCrashContext() Io.Writer.Error!void {
79	95	if (S.already_dumped) return;
80	96	S.already_dumped = true;
81	97
	98	std.Options.debug_io.vtable.crashHandler(std.Options.debug_io.userdata);
	99
82	100	// TODO: this does mean that a different thread could grab the stderr mutex between the context
83	101	// and the actual panic printing, which would be quite confusing.
84	102	const stderr = std.debug.lockStderr(&.{});
...	...	@@ -170,3 +188,5 @@ const Zcu = @import("Zcu.zig");
170	188	const InternPool = @import("InternPool.zig");
171	189	const dev = @import("dev.zig");
172	190	const print_zir = @import("print_zir.zig");
	191
	192	const build_options = @import("build_options");

src/main.zig+2-6

...	...	@@ -52,12 +52,8 @@ pub const std_options: std.Options = .{
52	52	};
53	53	pub const std_options_cwd = if (native_os == .wasi) wasi_cwd else null;
54	54
55		const crash_report_enabled = switch (build_options.io_mode) {
56		.threaded => build_options.enable_debug_extensions,
57		.evented => false, // would use threadlocals in a way incompatible with evented
58		};
59		pub const panic = if (crash_report_enabled) crash_report.panic else std.debug.FullPanic(std.debug.defaultPanic);
60		pub const debug = if (crash_report_enabled) crash_report.debug else struct {};
	55	pub const panic = crash_report.panic;
	56	pub const debug = crash_report.debug;
61	57
62	58	var preopens: std.process.Preopens = .empty;
63	59	pub fn wasi_cwd() Io.Dir {