Commit 8c5f731afbf2572f98b2840f3307e2c8a607d7a8 | zig

author	Matthew Lugg <mlugg@mlugg.co.uk> 2025-12-21 14:50:58+01:00
committer	Matthew Lugg <mlugg@mlugg.co.uk> 2025-12-21 14:50:58+01:00
log	8c5f731afbf2572f98b2840f3307e2c8a607d7a8
tree	1f11358b2a1fd708738a93df52ff18175a7e2c7c
parent	b27bdd5af0976d0b94819d6dda557d0dbb8f0d98
parent	67eed9955005aaa02344b8d566ba140b9f2e0e18

Merge pull request 'Some std.Io goodies' (#30235) from std.Io-misc into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/30235
Reviewed-by: Andrew Kelley <andrewrk@noreply.codeberg.org>

10 files changed, 1210 insertions(+), 933 deletions(-)

.mailmap
lib
- std
  - Build
    Fuzz.zig
    WebServer.zig
  - Io.zig
  - Io
    Threaded.zig
    net
    HostName.zig
    test.zig
    test.zig
src
- Sema.zig
- codegen
  - x86_64
    CodeGen.zig

.mailmap+3

lib/std/Build/Fuzz.zig+1-1

...	@@ -124,7 +124,7 @@ pub fn init(	...	@@ -124,7 +124,7 @@ pub fn init(
124	.coverage_files = .empty,	124	.coverage_files = .empty,
125	.coverage_mutex = .init,	125	.coverage_mutex = .init,
126	.queue_mutex = .init,	126	.queue_mutex = .init,
127	.queue_cond = .{},	127	.queue_cond = .init,
128	.msg_queue = .empty,	128	.msg_queue = .empty,
129	};	129	};
130	}	130	}

lib/std/Build/WebServer.zig+2-2

...	@@ -122,8 +122,8 @@ pub fn init(opts: Options) WebServer {	...	@@ -122,8 +122,8 @@ pub fn init(opts: Options) WebServer {
122	.update_id = .init(0),	122	.update_id = .init(0),
123		123
124	.runner_request_mutex = .init,	124	.runner_request_mutex = .init,
125	.runner_request_ready_cond = .{},	125	.runner_request_ready_cond = .init,
126	.runner_request_empty_cond = .{},	126	.runner_request_empty_cond = .init,
127	.runner_request = null,	127	.runner_request = null,
128	};	128	};
129	}	129	}

lib/std/Io.zig+604-154

...	@@ -650,17 +650,17 @@ pub const VTable = struct {	...	@@ -650,17 +650,17 @@ pub const VTable = struct {
650	groupWait: const fn (?anyopaque, Group, token: anyopaque) void,	650	groupWait: const fn (?anyopaque, Group, token: anyopaque) void,
651	groupCancel: const fn (?anyopaque, Group, token: anyopaque) void,	651	groupCancel: const fn (?anyopaque, Group, token: anyopaque) void,
652		652
		653	recancel: const fn (?anyopaque) void,
		654	swapCancelProtection: const fn (?anyopaque, new: CancelProtection) CancelProtection,
		655	checkCancel: const fn (?anyopaque) Cancelable!void,
		656
653	/// Blocks until one of the futures from the list has a result ready, such	657	/// Blocks until one of the futures from the list has a result ready, such
654	/// that awaiting it will not block. Returns that index.	658	/// that awaiting it will not block. Returns that index.
655	select: const fn (?anyopaque, futures: []const *AnyFuture) Cancelable!usize,	659	select: const fn (?anyopaque, futures: []const *AnyFuture) Cancelable!usize,
656		660
657	mutexLock: const fn (?anyopaque, prev_state: Mutex.State, mutex: *Mutex) Cancelable!void,	661	futexWait: const fn (?anyopaque, ptr: *const u32, expected: u32, Timeout) Cancelable!void,
658	mutexLockUncancelable: const fn (?anyopaque, prev_state: Mutex.State, mutex: *Mutex) void,	662	futexWaitUncancelable: const fn (?anyopaque, ptr: *const u32, expected: u32) void,
659	mutexUnlock: const fn (?anyopaque, prev_state: Mutex.State, mutex: *Mutex) void,	663	futexWake: const fn (?anyopaque, ptr: *const u32, max_waiters: u32) void,
660
661	conditionWait: const fn (?anyopaque, cond: Condition, mutex: Mutex) Cancelable!void,
662	conditionWaitUncancelable: const fn (?anyopaque, cond: Condition, mutex: Mutex) void,
663	conditionWake: const fn (?anyopaque, cond: *Condition, wake: Condition.Wake) void,
664		664
665	dirMake: const fn (?anyopaque, Dir, sub_path: []const u8, Dir.Mode) Dir.MakeError!void,	665	dirMake: const fn (?anyopaque, Dir, sub_path: []const u8, Dir.Mode) Dir.MakeError!void,
666	dirMakePath: const fn (?anyopaque, Dir, sub_path: []const u8, Dir.Mode) Dir.MakeError!void,	666	dirMakePath: const fn (?anyopaque, Dir, sub_path: []const u8, Dir.Mode) Dir.MakeError!void,
...	@@ -701,7 +701,7 @@ pub const VTable = struct {	...	@@ -701,7 +701,7 @@ pub const VTable = struct {
701	netClose: const fn (?anyopaque, handle: net.Socket.Handle) void,	701	netClose: const fn (?anyopaque, handle: net.Socket.Handle) void,
702	netInterfaceNameResolve: const fn (?anyopaque, *const net.Interface.Name) net.Interface.Name.ResolveError!net.Interface,	702	netInterfaceNameResolve: const fn (?anyopaque, *const net.Interface.Name) net.Interface.Name.ResolveError!net.Interface,
703	netInterfaceName: const fn (?anyopaque, net.Interface) net.Interface.NameError!net.Interface.Name,	703	netInterfaceName: const fn (?anyopaque, net.Interface) net.Interface.NameError!net.Interface.Name,
704	netLookup: const fn (?anyopaque, net.HostName, *Queue(net.HostName.LookupResult), net.HostName.LookupOptions) void,	704	netLookup: const fn (?anyopaque, net.HostName, *Queue(net.HostName.LookupResult), net.HostName.LookupOptions) net.HostName.LookupError!void,
705	};	705	};
706		706
707	pub const Cancelable = error{	707	pub const Cancelable = error{
...	@@ -986,7 +986,14 @@ pub fn Future(Result: type) type {	...	@@ -986,7 +986,14 @@ pub fn Future(Result: type) type {
986	any_future: ?*AnyFuture,	986	any_future: ?*AnyFuture,
987	result: Result,	987	result: Result,
988		988
989	/// Equivalent to `await` but places a cancellation request.	989	/// Equivalent to `await` but places a cancellation request. This causes the task to receive
		990	/// `error.Canceled` from its next "cancelation point" (if any). A cancelation point is a
		991	/// call to a function in `Io` which can return `error.Canceled`.
		992	///
		993	/// After cancelation of a task is requested, only the next cancelation point in that task
		994	/// will return `error.Canceled`: future points will not re-signal the cancelation. As such,
		995	/// it is usually a bug to ignore `error.Canceled`. However, to defer handling cancelation
		996	/// requests, see also `recancel` and `CancelProtection`.
990	///	997	///
991	/// Idempotent. Not threadsafe.	998	/// Idempotent. Not threadsafe.
992	pub fn cancel(f: *@This(), io: Io) Result {	999	pub fn cancel(f: *@This(), io: Io) Result {
...	@@ -1083,6 +1090,8 @@ pub const Group = struct {	...	@@ -1083,6 +1090,8 @@ pub const Group = struct {
1083	/// Equivalent to `wait` but immediately requests cancellation on all	1090	/// Equivalent to `wait` but immediately requests cancellation on all
1084	/// members of the group.	1091	/// members of the group.
1085	///	1092	///
		1093	/// For a description of cancelation and cancelation points, see `Future.cancel`.
		1094	///
1086	/// Idempotent. Not threadsafe.	1095	/// Idempotent. Not threadsafe.
1087	pub fn cancel(g: *Group, io: Io) void {	1096	pub fn cancel(g: *Group, io: Io) void {
1088	const token = g.token orelse return;	1097	const token = g.token orelse return;
...	@@ -1091,6 +1100,61 @@ pub const Group = struct {	...	@@ -1091,6 +1100,61 @@ pub const Group = struct {
1091	}	1100	}
1092	};	1101	};
1093		1102
		1103	/// Asserts that `error.Canceled` was returned from a prior cancelation point, and "re-arms" the
		1104	/// cancelation request, so that `error.Canceled` will be returned again from the next cancelation
		1105	/// point.
		1106	///
		1107	/// For a description of cancelation and cancelation points, see `Future.cancel`.
		1108	pub fn recancel(io: Io) void {
		1109	io.vtable.recancel(io.userdata);
		1110	}
		1111
		1112	/// In rare cases, it is desirable to completely block cancelation notification, so that a region
		1113	/// of code can run uninterrupted before `error.Canceled` is potentially observed. Therefore, every
		1114	/// task has a "cancel protection" state which indicates whether or not `Io` functions can introduce
		1115	/// cancelation points.
		1116	///
		1117	/// To modify a task's cancel protection state, see `swapCancelProtection`.
		1118	///
		1119	/// For a description of cancelation and cancelation points, see `Future.cancel`.
		1120	pub const CancelProtection = enum {
		1121	/// Any call to an `Io` function with `error.Canceled` in its error set is a cancelation point.
		1122	///
		1123	/// This is the default state, which all tasks are created in.
		1124	unblocked,
		1125	/// No `Io` function introduces a cancelation point (`error.Canceled` will never be returned).
		1126	blocked,
		1127	};
		1128	/// Updates the current task's cancel protection state (see `CancelProtection`).
		1129	///
		1130	/// The typical usage for this function is to protect a block of code from cancelation:
		1131	/// ```
		1132	/// const old_cancel_protect = io.swapCancelProtection(.blocked);
		1133	/// defer _ = io.swapCancelProtection(old_cancel_protect);
		1134	/// doSomeWork() catch \|err\| switch (err) {
		1135	/// error.Canceled => unreachable,
		1136	/// };
		1137	/// ```
		1138	///
		1139	/// For a description of cancelation and cancelation points, see `Future.cancel`.
		1140	pub fn swapCancelProtection(io: Io, new: CancelProtection) CancelProtection {
		1141	return io.vtable.swapCancelProtection(io.userdata, new);
		1142	}
		1143
		1144	/// This function acts as a pure cancelation point (subject to protection; see `CancelProtection`)
		1145	/// and does nothing else. In other words, it returns `error.Canceled` if there is an outstanding
		1146	/// non-blocked cancelation request, but otherwise is a no-op.
		1147	///
		1148	/// It is rarely necessary to call this function. The primary use case is in long-running CPU-bound
		1149	/// tasks which may need to respond to cancelation before completing. Short tasks, or those which
		1150	/// perform other `Io` operations (and hence have other cancelation points), will typically already
		1151	/// respond quickly to cancelation requests.
		1152	///
		1153	/// For a description of cancelation and cancelation points, see `Future.cancel`.
		1154	pub fn checkCancel(io: Io) Cancelable!void {
		1155	return io.vtable.checkCancel(io.userdata);
		1156	}
		1157
1094	pub fn Select(comptime U: type) type {	1158	pub fn Select(comptime U: type) type {
1095	return struct {	1159	return struct {
1096	io: Io,	1160	io: Io,
...	@@ -1144,7 +1208,9 @@ pub fn Select(comptime U: type) type {	...	@@ -1144,7 +1208,9 @@ pub fn Select(comptime U: type) type {
1144	const args_casted: *const Args = @ptrCast(@alignCast(context));	1208	const args_casted: *const Args = @ptrCast(@alignCast(context));
1145	const unerased_select: *S = @fieldParentPtr("group", group);	1209	const unerased_select: *S = @fieldParentPtr("group", group);
1146	const elem = @unionInit(U, @tagName(field), @call(.auto, function, args_casted.*));	1210	const elem = @unionInit(U, @tagName(field), @call(.auto, function, args_casted.*));
1147	unerased_select.queue.putOneUncancelable(unerased_select.io, elem);	1211	unerased_select.queue.putOneUncancelable(unerased_select.io, elem) catch \|err\| switch (err) {
		1212	error.Closed => unreachable,
		1213	};
1148	}	1214	}
1149	};	1215	};
1150	_ = @atomicRmw(usize, &s.outstanding, .Add, 1, .monotonic);	1216	_ = @atomicRmw(usize, &s.outstanding, .Add, 1, .monotonic);
...	@@ -1158,12 +1224,17 @@ pub fn Select(comptime U: type) type {	...	@@ -1158,12 +1224,17 @@ pub fn Select(comptime U: type) type {
1158	/// Not threadsafe.	1224	/// Not threadsafe.
1159	pub fn wait(s: *S) Cancelable!U {	1225	pub fn wait(s: *S) Cancelable!U {
1160	s.outstanding -= 1;	1226	s.outstanding -= 1;
1161	return s.queue.getOne(s.io);	1227	return s.queue.getOne(s.io) catch \|err\| switch (err) {
		1228	error.Canceled => \|e\| return e,
		1229	error.Closed => unreachable,
		1230	};
1162	}	1231	}
1163		1232
1164	/// Equivalent to `wait` but requests cancellation on all remaining	1233	/// Equivalent to `wait` but requests cancellation on all remaining
1165	/// tasks owned by the select.	1234	/// tasks owned by the select.
1166	///	1235	///
		1236	/// For a description of cancelation and cancelation points, see `Future.cancel`.
		1237	///
1167	/// It is illegal to call `wait` after this.	1238	/// It is illegal to call `wait` after this.
1168	///	1239	///
1169	/// Idempotent. Not threadsafe.	1240	/// Idempotent. Not threadsafe.
...	@@ -1174,104 +1245,340 @@ pub fn Select(comptime U: type) type {	...	@@ -1174,104 +1245,340 @@ pub fn Select(comptime U: type) type {
1174	};	1245	};
1175	}	1246	}
1176		1247
		1248	/// Atomically checks if the value at `ptr` equals `expected`, and if so, blocks until either:
		1249	///
		1250	/// * a matching (same `ptr` argument) `futexWake` call occurs, or
		1251	/// * a spurious ("random") wakeup occurs.
		1252	///
		1253	/// Typically, `futexWake` should be called immediately after updating the value at `ptr.*`, to
		1254	/// unblock tasks using `futexWait` to wait for the value to change from what it previously was.
		1255	///
		1256	/// The caller is responsible for identifying spurious wakeups if necessary, typically by checking
		1257	/// the value at `ptr.*`.
		1258	///
		1259	/// Asserts that `T` is 4 bytes in length and has a well-defined layout with no padding bits.
		1260	pub fn futexWait(io: Io, comptime T: type, ptr: *align(@alignOf(u32)) const T, expected: T) Cancelable!void {
		1261	return futexWaitTimeout(io, T, ptr, expected, .none);
		1262	}
		1263	/// Same as `futexWait`, except also unblocks if `timeout` expires. As with `futexWait`, spurious
		1264	/// wakeups are possible. It remains the caller's responsibility to differentiate between these
		1265	/// three possible wake-up reasons if necessary.
		1266	pub fn futexWaitTimeout(io: Io, comptime T: type, ptr: *align(@alignOf(u32)) const T, expected: T, timeout: Timeout) Cancelable!void {
		1267	comptime assert(@sizeOf(T) == 4);
		1268	const expected_raw: *align(1) const u32 = @ptrCast(&expected);
		1269	return io.vtable.futexWait(io.userdata, @ptrCast(ptr), expected_raw.*, timeout);
		1270	}
		1271	/// Same as `futexWait`, except does not introduce a cancelation point.
		1272	///
		1273	/// For a description of cancelation and cancelation points, see `Future.cancel`.
		1274	pub fn futexWaitUncancelable(io: Io, comptime T: type, ptr: *align(@alignOf(u32)) const T, expected: T) void {
		1275	comptime assert(@sizeOf(T) == @sizeOf(u32));
		1276	const expected_raw: *align(1) const u32 = @ptrCast(&expected);
		1277	io.vtable.futexWaitUncancelable(io.userdata, @ptrCast(ptr), expected_raw.*);
		1278	}
		1279	/// Unblocks pending futex waits on `ptr`, up to a limit of `max_waiters` calls.
		1280	pub fn futexWake(io: Io, comptime T: type, ptr: *align(@alignOf(u32)) const T, max_waiters: u32) void {
		1281	comptime assert(@sizeOf(T) == @sizeOf(u32));
		1282	if (max_waiters == 0) return;
		1283	return io.vtable.futexWake(io.userdata, @ptrCast(ptr), max_waiters);
		1284	}
		1285
1177	pub const Mutex = struct {	1286	pub const Mutex = struct {
1178	state: State,	1287	state: std.atomic.Value(State),
1179		1288
1180	pub const State = enum(usize) {	1289	pub const init: Mutex = .{ .state = .init(.unlocked) };
1181	locked_once = 0b00,
1182	unlocked = 0b01,
1183	contended = 0b10,
1184	/// contended
1185	_,
1186		1290
1187	pub fn isUnlocked(state: State) bool {	1291	const State = enum(u32) {
1188	return @intFromEnum(state) & @intFromEnum(State.unlocked) == @intFromEnum(State.unlocked);	1292	unlocked,
1189	}	1293	locked_once,
		1294	contended,
1190	};	1295	};
1191		1296
1192	pub const init: Mutex = .{ .state = .unlocked };	1297	pub fn tryLock(m: *Mutex) bool {
1193		1298	switch (m.state.cmpxchgWeak(
1194	pub fn tryLock(mutex: *Mutex) bool {	1299	.unlocked,
1195	const prev_state: State = @enumFromInt(@atomicRmw(	1300	.locked_once,
1196	usize,
1197	@as(*usize, @ptrCast(&mutex.state)),
1198	.And,
1199	~@intFromEnum(State.unlocked),
1200	.acquire,	1301	.acquire,
1201	));	1302	.monotonic,
1202	return prev_state.isUnlocked();	1303	) orelse return true) {
		1304	.unlocked => unreachable,
		1305	.locked_once, .contended => return false,
		1306	}
1203	}	1307	}
1204		1308
1205	pub fn lock(mutex: *Mutex, io: std.Io) Cancelable!void {	1309	pub fn lock(m: *Mutex, io: Io) Cancelable!void {
1206	const prev_state: State = @enumFromInt(@atomicRmw(	1310	const initial_state = m.state.cmpxchgWeak(
1207	usize,	1311	.unlocked,
1208	@as(*usize, @ptrCast(&mutex.state)),	1312	.locked_once,
1209	.And,
1210	~@intFromEnum(State.unlocked),
1211	.acquire,	1313	.acquire,
1212	));	1314	.monotonic,
1213	if (prev_state.isUnlocked()) {	1315	) orelse {
1214	@branchHint(.likely);	1316	@branchHint(.likely);
1215	return;	1317	return;
		1318	};
		1319	if (initial_state == .contended) {
		1320	try io.futexWait(State, &m.state.raw, .contended);
		1321	}
		1322	while (m.state.swap(.contended, .acquire) != .unlocked) {
		1323	try io.futexWait(State, &m.state.raw, .contended);
1216	}	1324	}
1217	return io.vtable.mutexLock(io.userdata, prev_state, mutex);
1218	}	1325	}
1219		1326
1220	/// Same as `lock` but cannot be canceled.	1327	/// Same as `lock`, except does not introduce a cancelation point.
1221	pub fn lockUncancelable(mutex: *Mutex, io: std.Io) void {	1328	///
1222	const prev_state: State = @enumFromInt(@atomicRmw(	1329	/// For a description of cancelation and cancelation points, see `Future.cancel`.
1223	usize,	1330	pub fn lockUncancelable(m: *Mutex, io: Io) void {
1224	@as(*usize, @ptrCast(&mutex.state)),	1331	const initial_state = m.state.cmpxchgWeak(
1225	.And,	1332	.unlocked,
1226	~@intFromEnum(State.unlocked),	1333	.locked_once,
1227	.acquire,	1334	.acquire,
1228	));	1335	.monotonic,
1229	if (prev_state.isUnlocked()) {	1336	) orelse {
1230	@branchHint(.likely);	1337	@branchHint(.likely);
1231	return;	1338	return;
		1339	};
		1340	if (initial_state == .contended) {
		1341	io.futexWaitUncancelable(State, &m.state.raw, .contended);
		1342	}
		1343	while (m.state.swap(.contended, .acquire) != .unlocked) {
		1344	io.futexWaitUncancelable(State, &m.state.raw, .contended);
1232	}	1345	}
1233	return io.vtable.mutexLockUncancelable(io.userdata, prev_state, mutex);
1234	}	1346	}
1235		1347
1236	pub fn unlock(mutex: *Mutex, io: std.Io) void {	1348	pub fn unlock(m: *Mutex, io: Io) void {
1237	const prev_state = @cmpxchgWeak(State, &mutex.state, .locked_once, .unlocked, .release, .acquire) orelse {	1349	switch (m.state.swap(.unlocked, .release)) {
1238	@branchHint(.likely);	1350	.unlocked => unreachable,
1239	return;	1351	.locked_once => {},
1240	};	1352	.contended => {
1241	assert(prev_state != .unlocked); // mutex not locked	1353	@branchHint(.unlikely);
1242	return io.vtable.mutexUnlock(io.userdata, prev_state, mutex);	1354	io.futexWake(State, &m.state.raw, 1);
		1355	},
		1356	}
1243	}	1357	}
1244	};	1358	};
1245		1359
1246	pub const Condition = struct {	1360	pub const Condition = struct {
1247	state: u64 = 0,	1361	state: std.atomic.Value(State),
		1362	/// Incremented whenever the condition is signaled
		1363	epoch: std.atomic.Value(u32),
		1364
		1365	const State = packed struct(u32) {
		1366	waiters: u16,
		1367	signals: u16,
		1368	};
		1369
		1370	pub const init: Condition = .{
		1371	.state = .init(.{ .waiters = 0, .signals = 0 }),
		1372	.epoch = .init(0),
		1373	};
1248		1374
1249	pub fn wait(cond: Condition, io: Io, mutex: Mutex) Cancelable!void {	1375	pub fn wait(cond: Condition, io: Io, mutex: Mutex) Cancelable!void {
1250	return io.vtable.conditionWait(io.userdata, cond, mutex);	1376	try waitInner(cond, io, mutex, false);
1251	}	1377	}
1252		1378
		1379	/// Same as `wait`, except does not introduce a cancelation point.
		1380	///
		1381	/// For a description of cancelation and cancelation points, see `Future.cancel`.
1253	pub fn waitUncancelable(cond: Condition, io: Io, mutex: Mutex) void {	1382	pub fn waitUncancelable(cond: Condition, io: Io, mutex: Mutex) void {
1254	return io.vtable.conditionWaitUncancelable(io.userdata, cond, mutex);	1383	waitInner(cond, io, mutex, true) catch \|err\| switch (err) {
		1384	error.Canceled => unreachable,
		1385	};
		1386	}
		1387
		1388	fn waitInner(cond: Condition, io: Io, mutex: Mutex, uncancelable: bool) Cancelable!void {
		1389	var epoch = cond.epoch.load(.acquire); // `.acquire` to ensure ordered before state load
		1390
		1391	{
		1392	const prev_state = cond.state.fetchAdd(.{ .waiters = 1, .signals = 0 }, .monotonic);
		1393	assert(prev_state.waiters < math.maxInt(u16)); // overflow caused by too many waiters
		1394	}
		1395
		1396	mutex.unlock(io);
		1397	defer mutex.lockUncancelable(io);
		1398
		1399	while (true) {
		1400	const result = if (uncancelable)
		1401	io.futexWaitUncancelable(u32, &cond.epoch.raw, epoch)
		1402	else
		1403	io.futexWait(u32, &cond.epoch.raw, epoch);
		1404
		1405	epoch = cond.epoch.load(.acquire); // `.acquire` to ensure ordered before `state` laod
		1406
		1407	// Even on error, try to consume a pending signal first. Otherwise a race might
		1408	// cause a signal to get stuck in the state with no corresponding waiter.
		1409	{
		1410	var prev_state = cond.state.load(.monotonic);
		1411	while (prev_state.signals > 0) {
		1412	prev_state = cond.state.cmpxchgWeak(prev_state, .{
		1413	.waiters = prev_state.waiters - 1,
		1414	.signals = prev_state.signals - 1,
		1415	}, .acquire, .monotonic) orelse {
		1416	// We successfully consumed a signal.
		1417	return;
		1418	};
		1419	}
		1420	}
		1421
		1422	// There are no more signals available; this was a spurious wakeup or an error. If it
		1423	// was an error, we will remove ourselves as a waiter and return that error. Otherwise,
		1424	// we'll loop back to the futex wait.
		1425	result catch \|err\| {
		1426	const prev_state = cond.state.fetchSub(.{ .waiters = 1, .signals = 0 }, .monotonic);
		1427	assert(prev_state.waiters > 0); // underflow caused by illegal state
		1428	return err;
		1429	};
		1430	}
1255	}	1431	}
1256		1432
1257	pub fn signal(cond: *Condition, io: Io) void {	1433	pub fn signal(cond: *Condition, io: Io) void {
1258	io.vtable.conditionWake(io.userdata, cond, .one);	1434	var prev_state = cond.state.load(.monotonic);
		1435	while (prev_state.waiters > prev_state.signals) {
		1436	@branchHint(.unlikely);
		1437	prev_state = cond.state.cmpxchgWeak(prev_state, .{
		1438	.waiters = prev_state.waiters,
		1439	.signals = prev_state.signals + 1,
		1440	}, .release, .monotonic) orelse {
		1441	// Update the epoch to tell the waiting threads that there are new signals for them.
		1442	// Note that a waiting thread could miss a take if exactly (1<<32)-1 wakes happen
		1443	// between it observing the epoch and sleeping on it, but this is extraordinarily
		1444	// unlikely due to the precise number of calls required.
		1445	_ = cond.epoch.fetchAdd(1, .release); // `.release` to ensure ordered after `state` update
		1446	io.futexWake(u32, &cond.epoch.raw, 1);
		1447	return;
		1448	};
		1449	}
1259	}	1450	}
1260		1451
1261	pub fn broadcast(cond: *Condition, io: Io) void {	1452	pub fn broadcast(cond: *Condition, io: Io) void {
1262	io.vtable.conditionWake(io.userdata, cond, .all);	1453	var prev_state = cond.state.load(.monotonic);
		1454	while (prev_state.waiters > prev_state.signals) {
		1455	@branchHint(.unlikely);
		1456	prev_state = cond.state.cmpxchgWeak(prev_state, .{
		1457	.waiters = prev_state.waiters,
		1458	.signals = prev_state.waiters,
		1459	}, .release, .monotonic) orelse {
		1460	// Update the epoch to tell the waiting threads that there are new signals for them.
		1461	// Note that a waiting thread could miss a take if exactly (1<<32)-1 wakes happen
		1462	// between it observing the epoch and sleeping on it, but this is extraordinarily
		1463	// unlikely due to the precise number of calls required.
		1464	_ = cond.epoch.fetchAdd(1, .release); // `.release` to ensure ordered after `state` update
		1465	io.futexWake(u32, &cond.epoch.raw, prev_state.waiters - prev_state.signals);
		1466	return;
		1467	};
		1468	}
1263	}	1469	}
		1470	};
1264		1471
1265	pub const Wake = enum {	1472	/// Logical boolean flag which can be set and unset and supports a "wait until set" operation.
1266	/// Wake up only one thread.	1473	pub const Event = enum(u32) {
1267	one,	1474	unset,
1268	/// Wake up all threads.	1475	waiting,
1269	all,	1476	is_set,
1270	};	1477
		1478	/// Returns whether the logical boolean is `true`.
		1479	pub fn isSet(event: *const Event) bool {
		1480	return switch (@atomicLoad(Event, event, .acquire)) {
		1481	.unset, .waiting => false,
		1482	.is_set => true,
		1483	};
		1484	}
		1485
		1486	/// Blocks until the logical boolean is `true`.
		1487	pub fn wait(event: *Event, io: Io) Io.Cancelable!void {
		1488	if (@cmpxchgStrong(Event, event, .unset, .waiting, .acquire, .acquire)) \|prev\| switch (prev) {
		1489	.unset => unreachable,
		1490	.waiting => {},
		1491	.is_set => return,
		1492	};
		1493	errdefer {
		1494	// Ideally we would restore the event back to `.unset` instead of `.waiting`, but there
		1495	// might be other threads waiting on the event. In theory we could track the number of
		1496	// waiting threads in the unused bits of the `Event`, but that has its own problem: the
		1497	// waiters would wake up when a new waiter was added. So it's easiest to just leave
		1498	// the state at `.waiting`---at worst it causes one redundant call to `futexWake`.
		1499	}
		1500	while (true) {
		1501	try io.futexWait(Event, event, .waiting);
		1502	switch (@atomicLoad(Event, event, .acquire)) {
		1503	.unset => unreachable, // `reset` called before pending `wait` returned
		1504	.waiting => continue,
		1505	.is_set => return,
		1506	}
		1507	}
		1508	}
		1509
		1510	/// Same as `wait`, except does not introduce a cancelation point.
		1511	///
		1512	/// For a description of cancelation and cancelation points, see `Future.cancel`.
		1513	pub fn waitUncancelable(event: *Event, io: Io) void {
		1514	if (@cmpxchgStrong(Event, event, .unset, .waiting, .acquire, .acquire)) \|prev\| switch (prev) {
		1515	.unset => unreachable,
		1516	.waiting => {},
		1517	.is_set => return,
		1518	};
		1519	while (true) {
		1520	io.futexWaitUncancelable(Event, event, .waiting);
		1521	switch (@atomicLoad(Event, event, .acquire)) {
		1522	.unset => unreachable, // `reset` called before pending `wait` returned
		1523	.waiting => continue,
		1524	.is_set => return,
		1525	}
		1526	}
		1527	}
		1528
		1529	/// Blocks the calling thread until either the logical boolean is set, the timeout expires, or a
		1530	/// spurious wakeup occurs. If the timeout expires or a spurious wakeup occurs, `error.Timeout`
		1531	/// is returned.
		1532	pub fn waitTimeout(event: *Event, io: Io, timeout: Timeout) (error{Timeout} \|\| Cancelable)!void {
		1533	if (@cmpxchgStrong(Event, event, .unset, .waiting, .acquire, .acquire)) \|prev\| switch (prev) {
		1534	.unset => unreachable,
		1535	.waiting => assert(!builtin.single_threaded), // invalid state
		1536	.is_set => return,
		1537	};
		1538	errdefer {
		1539	// Ideally we would restore the event back to `.unset` instead of `.waiting`, but there
		1540	// might be other threads waiting on the event. In theory we could track the number of
		1541	// waiting threads in the unused bits of the `Event`, but that has its own problem: the
		1542	// waiters would wake up when a new waiter was added. So it's easiest to just leave
		1543	// the state at `.waiting`---at worst it causes one redundant call to `futexWake`.
		1544	}
		1545	io.futexWaitTimeout(Event, event, .waiting, timeout);
		1546	switch (@atomicLoad(Event, event, .acquire)) {
		1547	.unset => unreachable, // `reset` called before pending `wait` returned
		1548	.waiting => return error.Timeout,
		1549	.is_set => return,
		1550	}
		1551	}
		1552
		1553	/// Sets the logical boolean to true, and hence unblocks any pending calls to `wait`. The
		1554	/// logical boolean remains true until `reset` is called, so future calls to `set` have no
		1555	/// semantic effect.
		1556	///
		1557	/// Any memory accesses prior to a `set` call are "released", so that if this `set` call causes
		1558	/// `isSet` to return `true` or a wait to finish, those tasks will be able to observe those
		1559	/// memory accesses.
		1560	pub fn set(e: *Event, io: Io) void {
		1561	switch (@atomicRmw(Event, e, .Xchg, .is_set, .release)) {
		1562	.unset, .is_set => {},
		1563	.waiting => io.futexWake(Event, e, std.math.maxInt(u32)),
		1564	}
		1565	}
		1566
		1567	/// Sets the logical boolean to false.
		1568	///
		1569	/// Assumes that there is no pending call to `wait` or `waitUncancelable`.
		1570	///
		1571	/// However, concurrent calls to `isSet`, `set`, and `reset` are allowed.
		1572	pub fn reset(e: *Event) void {
		1573	@atomicStore(Event, e, .unset, .monotonic);
		1574	}
1271	};	1575	};
1272		1576
		1577	pub const QueueClosedError = error{Closed};
		1578
1273	pub const TypeErasedQueue = struct {	1579	pub const TypeErasedQueue = struct {
1274	mutex: Mutex,	1580	mutex: Mutex,
		1581	closed: bool,
1275		1582
1276	/// Ring buffer. This data is logically after queued getters.	1583	/// Ring buffer. This data is logically after queued getters.
1277	buffer: []u8,	1584	buffer: []u8,
...	@@ -1283,12 +1590,14 @@ pub const TypeErasedQueue = struct {	...	@@ -1283,12 +1590,14 @@ pub const TypeErasedQueue = struct {
1283		1590
1284	const Put = struct {	1591	const Put = struct {
1285	remaining: []const u8,	1592	remaining: []const u8,
		1593	needed: usize,
1286	condition: Condition,	1594	condition: Condition,
1287	node: std.DoublyLinkedList.Node,	1595	node: std.DoublyLinkedList.Node,
1288	};	1596	};
1289		1597
1290	const Get = struct {	1598	const Get = struct {
1291	remaining: []u8,	1599	remaining: []u8,
		1600	needed: usize,
1292	condition: Condition,	1601	condition: Condition,
1293	node: std.DoublyLinkedList.Node,	1602	node: std.DoublyLinkedList.Node,
1294	};	1603	};
...	@@ -1296,6 +1605,7 @@ pub const TypeErasedQueue = struct {	...	@@ -1296,6 +1605,7 @@ pub const TypeErasedQueue = struct {
1296	pub fn init(buffer: []u8) TypeErasedQueue {	1605	pub fn init(buffer: []u8) TypeErasedQueue {
1297	return .{	1606	return .{
1298	.mutex = .init,	1607	.mutex = .init,
		1608	.closed = false,
1299	.buffer = buffer,	1609	.buffer = buffer,
1300	.start = 0,	1610	.start = 0,
1301	.len = 0,	1611	.len = 0,
...	@@ -1304,7 +1614,27 @@ pub const TypeErasedQueue = struct {	...	@@ -1304,7 +1614,27 @@ pub const TypeErasedQueue = struct {
1304	};	1614	};
1305	}	1615	}
1306		1616
1307	pub fn put(q: *TypeErasedQueue, io: Io, elements: []const u8, min: usize) Cancelable!usize {	1617	pub fn close(q: *TypeErasedQueue, io: Io) void {
		1618	q.mutex.lockUncancelable(io);
		1619	defer q.mutex.unlock(io);
		1620	q.closed = true;
		1621	{
		1622	var it = q.getters.first;
		1623	while (it) \|node\| : (it = node.next) {
		1624	const getter: *Get = @alignCast(@fieldParentPtr("node", node));
		1625	getter.condition.signal(io);
		1626	}
		1627	}
		1628	{
		1629	var it = q.putters.first;
		1630	while (it) \|node\| : (it = node.next) {
		1631	const putter: *Put = @alignCast(@fieldParentPtr("node", node));
		1632	putter.condition.signal(io);
		1633	}
		1634	}
		1635	}
		1636
		1637	pub fn put(q: *TypeErasedQueue, io: Io, elements: []const u8, min: usize) (QueueClosedError \|\| Cancelable)!usize {
1308	assert(elements.len >= min);	1638	assert(elements.len >= min);
1309	if (elements.len == 0) return 0;	1639	if (elements.len == 0) return 0;
1310	try q.mutex.lock(io);	1640	try q.mutex.lock(io);
...	@@ -1312,14 +1642,17 @@ pub const TypeErasedQueue = struct {	...	@@ -1312,14 +1642,17 @@ pub const TypeErasedQueue = struct {
1312	return q.putLocked(io, elements, min, false);	1642	return q.putLocked(io, elements, min, false);
1313	}	1643	}
1314		1644
1315	/// Same as `put` but cannot be canceled.	1645	/// Same as `put`, except does not introduce a cancelation point.
1316	pub fn putUncancelable(q: *TypeErasedQueue, io: Io, elements: []const u8, min: usize) usize {	1646	///
		1647	/// For a description of cancelation and cancelation points, see `Future.cancel`.
		1648	pub fn putUncancelable(q: *TypeErasedQueue, io: Io, elements: []const u8, min: usize) QueueClosedError!usize {
1317	assert(elements.len >= min);	1649	assert(elements.len >= min);
1318	if (elements.len == 0) return 0;	1650	if (elements.len == 0) return 0;
1319	q.mutex.lockUncancelable(io);	1651	q.mutex.lockUncancelable(io);
1320	defer q.mutex.unlock(io);	1652	defer q.mutex.unlock(io);
1321	return q.putLocked(io, elements, min, true) catch \|err\| switch (err) {	1653	return q.putLocked(io, elements, min, true) catch \|err\| switch (err) {
1322	error.Canceled => unreachable,	1654	error.Canceled => unreachable,
		1655	error.Closed => \|e\| return e,
1323	};	1656	};
1324	}	1657	}
1325		1658
...	@@ -1333,49 +1666,79 @@ pub const TypeErasedQueue = struct {	...	@@ -1333,49 +1666,79 @@ pub const TypeErasedQueue = struct {
1333	return if (slice.len > 0) slice else null;	1666	return if (slice.len > 0) slice else null;
1334	}	1667	}
1335		1668
1336	fn putLocked(q: *TypeErasedQueue, io: Io, elements: []const u8, min: usize, uncancelable: bool) Cancelable!usize {	1669	fn putLocked(q: *TypeErasedQueue, io: Io, elements: []const u8, target: usize, uncancelable: bool) (QueueClosedError \|\| Cancelable)!usize {
		1670	// A closed queue cannot be added to, even if there is space in the buffer.
		1671	if (q.closed) return error.Closed;
		1672
1337	// Getters have first priority on the data, and only when the getters	1673	// Getters have first priority on the data, and only when the getters
1338	// queue is empty do we start populating the buffer.	1674	// queue is empty do we start populating the buffer.
1339		1675
1340	var remaining = elements;	1676	// The number of elements we add immediately, before possibly blocking.
		1677	var n: usize = 0;
		1678
1341	while (q.getters.popFirst()) \|getter_node\| {	1679	while (q.getters.popFirst()) \|getter_node\| {
1342	const getter: *Get = @alignCast(@fieldParentPtr("node", getter_node));	1680	const getter: *Get = @alignCast(@fieldParentPtr("node", getter_node));
1343	const copy_len = @min(getter.remaining.len, remaining.len);	1681	const copy_len = @min(getter.remaining.len, elements.len - n);
1344	assert(copy_len > 0);	1682	assert(copy_len > 0);
1345	@memcpy(getter.remaining[0..copy_len], remaining[0..copy_len]);	1683	@memcpy(getter.remaining[0..copy_len], elements[n..][0..copy_len]);
1346	remaining = remaining[copy_len..];
1347	getter.remaining = getter.remaining[copy_len..];	1684	getter.remaining = getter.remaining[copy_len..];
1348	if (getter.remaining.len == 0) {	1685	getter.needed -\|= copy_len;
		1686	n += copy_len;
		1687	if (getter.needed == 0) {
1349	getter.condition.signal(io);	1688	getter.condition.signal(io);
1350	if (remaining.len > 0) continue;	1689	} else {
1351	} else q.getters.prepend(getter_node);	1690	assert(n == elements.len); // we didn't have enough elements for the getter
1352	assert(remaining.len == 0);	1691	q.getters.prepend(getter_node);
1353	return elements.len;	1692	}
		1693	if (n == elements.len) return elements.len;
1354	}	1694	}
1355		1695
1356	while (q.puttableSlice()) \|slice\| {	1696	while (q.puttableSlice()) \|slice\| {
1357	const copy_len = @min(slice.len, remaining.len);	1697	const copy_len = @min(slice.len, elements.len - n);
1358	assert(copy_len > 0);	1698	assert(copy_len > 0);
1359	@memcpy(slice[0..copy_len], remaining[0..copy_len]);	1699	@memcpy(slice[0..copy_len], elements[n..][0..copy_len]);
1360	q.len += copy_len;	1700	q.len += copy_len;
1361	remaining = remaining[copy_len..];	1701	n += copy_len;
1362	if (remaining.len == 0) return elements.len;	1702	if (n == elements.len) return elements.len;
1363	}	1703	}
1364		1704
1365	const total_filled = elements.len - remaining.len;	1705	// Don't block if we hit the target.
1366	if (total_filled >= min) return total_filled;	1706	if (n >= target) return n;
1367		1707
1368	var pending: Put = .{ .remaining = remaining, .condition = .{}, .node = .{} };	1708	var pending: Put = .{
		1709	.remaining = elements[n..],
		1710	.needed = target - n,
		1711	.condition = .init,
		1712	.node = .{},
		1713	};
1369	q.putters.append(&pending.node);	1714	q.putters.append(&pending.node);
1370	defer if (pending.remaining.len > 0) q.putters.remove(&pending.node);	1715	defer if (pending.needed > 0) q.putters.remove(&pending.node);
1371	while (pending.remaining.len > 0) if (uncancelable)	1716
1372	pending.condition.waitUncancelable(io, &q.mutex)	1717	while (pending.needed > 0 and !q.closed) {
1373	else	1718	if (uncancelable) {
1374	try pending.condition.wait(io, &q.mutex);	1719	pending.condition.waitUncancelable(io, &q.mutex);
1375	return elements.len;	1720	continue;
		1721	}
		1722	pending.condition.wait(io, &q.mutex) catch \|err\| switch (err) {
		1723	error.Canceled => if (pending.remaining.len == elements.len) {
		1724	// Canceled while waiting, and appended no elements.
		1725	return error.Canceled;
		1726	} else {
		1727	// Canceled while waiting, but appended some elements, so report those first.
		1728	io.recancel();
		1729	return elements.len - pending.remaining.len;
		1730	},
		1731	};
		1732	}
		1733	if (pending.remaining.len == elements.len) {
		1734	// The queue was closed while we were waiting. We appended no elements.
		1735	assert(q.closed);
		1736	return error.Closed;
		1737	}
		1738	return elements.len - pending.remaining.len;
1376	}	1739	}
1377		1740
1378	pub fn get(q: *@This(), io: Io, buffer: []u8, min: usize) Cancelable!usize {	1741	pub fn get(q: *TypeErasedQueue, io: Io, buffer: []u8, min: usize) (QueueClosedError \|\| Cancelable)!usize {
1379	assert(buffer.len >= min);	1742	assert(buffer.len >= min);
1380	if (buffer.len == 0) return 0;	1743	if (buffer.len == 0) return 0;
1381	try q.mutex.lock(io);	1744	try q.mutex.lock(io);
...	@@ -1383,13 +1746,17 @@ pub const TypeErasedQueue = struct {	...	@@ -1383,13 +1746,17 @@ pub const TypeErasedQueue = struct {
1383	return q.getLocked(io, buffer, min, false);	1746	return q.getLocked(io, buffer, min, false);
1384	}	1747	}
1385		1748
1386	pub fn getUncancelable(q: *@This(), io: Io, buffer: []u8, min: usize) usize {	1749	/// Same as `get`, except does not introduce a cancelation point.
		1750	///
		1751	/// For a description of cancelation and cancelation points, see `Future.cancel`.
		1752	pub fn getUncancelable(q: *TypeErasedQueue, io: Io, buffer: []u8, min: usize) QueueClosedError!usize {
1387	assert(buffer.len >= min);	1753	assert(buffer.len >= min);
1388	if (buffer.len == 0) return 0;	1754	if (buffer.len == 0) return 0;
1389	q.mutex.lockUncancelable(io);	1755	q.mutex.lockUncancelable(io);
1390	defer q.mutex.unlock(io);	1756	defer q.mutex.unlock(io);
1391	return q.getLocked(io, buffer, min, true) catch \|err\| switch (err) {	1757	return q.getLocked(io, buffer, min, true) catch \|err\| switch (err) {
1392	error.Canceled => unreachable,	1758	error.Canceled => unreachable,
		1759	error.Closed => \|e\| return e,
1393	};	1760	};
1394	}	1761	}
1395		1762
...	@@ -1399,21 +1766,23 @@ pub const TypeErasedQueue = struct {	...	@@ -1399,21 +1766,23 @@ pub const TypeErasedQueue = struct {
1399	return if (slice.len > 0) slice else null;	1766	return if (slice.len > 0) slice else null;
1400	}	1767	}
1401		1768
1402	fn getLocked(q: *@This(), io: Io, buffer: []u8, min: usize, uncancelable: bool) Cancelable!usize {	1769	fn getLocked(q: *TypeErasedQueue, io: Io, buffer: []u8, target: usize, uncancelable: bool) (QueueClosedError \|\| Cancelable)!usize {
1403	// The ring buffer gets first priority, then data should come from any	1770	// The ring buffer gets first priority, then data should come from any
1404	// queued putters, then finally the ring buffer should be filled with	1771	// queued putters, then finally the ring buffer should be filled with
1405	// data from putters so they can be resumed.	1772	// data from putters so they can be resumed.
1406		1773
1407	var remaining = buffer;	1774	// The number of elements we received immediately, before possibly blocking.
		1775	var n: usize = 0;
		1776
1408	while (q.gettableSlice()) \|slice\| {	1777	while (q.gettableSlice()) \|slice\| {
1409	const copy_len = @min(slice.len, remaining.len);	1778	const copy_len = @min(slice.len, buffer.len - n);
1410	assert(copy_len > 0);	1779	assert(copy_len > 0);
1411	@memcpy(remaining[0..copy_len], slice[0..copy_len]);	1780	@memcpy(buffer[n..][0..copy_len], slice[0..copy_len]);
1412	q.start += copy_len;	1781	q.start += copy_len;
1413	if (q.buffer.len - q.start == 0) q.start = 0;	1782	if (q.buffer.len - q.start == 0) q.start = 0;
1414	q.len -= copy_len;	1783	q.len -= copy_len;
1415	remaining = remaining[copy_len..];	1784	n += copy_len;
1416	if (remaining.len == 0) {	1785	if (n == buffer.len) {
1417	q.fillRingBufferFromPutters(io);	1786	q.fillRingBufferFromPutters(io);
1418	return buffer.len;	1787	return buffer.len;
1419	}	1788	}
...	@@ -1422,33 +1791,64 @@ pub const TypeErasedQueue = struct {	...	@@ -1422,33 +1791,64 @@ pub const TypeErasedQueue = struct {
1422	// Copy directly from putters into buffer.	1791	// Copy directly from putters into buffer.
1423	while (q.putters.popFirst()) \|putter_node\| {	1792	while (q.putters.popFirst()) \|putter_node\| {
1424	const putter: *Put = @alignCast(@fieldParentPtr("node", putter_node));	1793	const putter: *Put = @alignCast(@fieldParentPtr("node", putter_node));
1425	const copy_len = @min(putter.remaining.len, remaining.len);	1794	const copy_len = @min(putter.remaining.len, buffer.len - n);
1426	assert(copy_len > 0);	1795	assert(copy_len > 0);
1427	@memcpy(remaining[0..copy_len], putter.remaining[0..copy_len]);	1796	@memcpy(buffer[n..][0..copy_len], putter.remaining[0..copy_len]);
1428	putter.remaining = putter.remaining[copy_len..];	1797	putter.remaining = putter.remaining[copy_len..];
1429	remaining = remaining[copy_len..];	1798	putter.needed -\|= copy_len;
1430	if (putter.remaining.len == 0) {	1799	n += copy_len;
		1800	if (putter.needed == 0) {
1431	putter.condition.signal(io);	1801	putter.condition.signal(io);
1432	if (remaining.len > 0) continue;	1802	} else {
1433	} else q.putters.prepend(putter_node);	1803	assert(n == buffer.len); // we didn't have enough space for the putter
1434	assert(remaining.len == 0);	1804	q.putters.prepend(putter_node);
1435	q.fillRingBufferFromPutters(io);	1805	}
1436	return buffer.len;	1806	if (n == buffer.len) {
		1807	q.fillRingBufferFromPutters(io);
		1808	return buffer.len;
		1809	}
1437	}	1810	}
1438		1811
1439	// Both ring buffer and putters queue is empty.	1812	// No need to call `fillRingBufferFromPutters` from this point onwards,
1440	const total_filled = buffer.len - remaining.len;	1813	// because we emptied the ring buffer and the putter queue!
1441	if (total_filled >= min) return total_filled;	1814
		1815	// Don't block if we hit the target or if the queue is closed. Return how
		1816	// many elements we could get immediately, unless the queue was closed and
		1817	// empty, in which case report `error.Closed`.
		1818	if (n == 0 and q.closed) return error.Closed;
		1819	if (n >= target or q.closed) return n;
1442		1820
1443	var pending: Get = .{ .remaining = remaining, .condition = .{}, .node = .{} };	1821	var pending: Get = .{
		1822	.remaining = buffer[n..],
		1823	.needed = target - n,
		1824	.condition = .init,
		1825	.node = .{},
		1826	};
1444	q.getters.append(&pending.node);	1827	q.getters.append(&pending.node);
1445	defer if (pending.remaining.len > 0) q.getters.remove(&pending.node);	1828	defer if (pending.needed > 0) q.getters.remove(&pending.node);
1446	while (pending.remaining.len > 0) if (uncancelable)	1829
1447	pending.condition.waitUncancelable(io, &q.mutex)	1830	while (pending.needed > 0 and !q.closed) {
1448	else	1831	if (uncancelable) {
1449	try pending.condition.wait(io, &q.mutex);	1832	pending.condition.waitUncancelable(io, &q.mutex);
1450	q.fillRingBufferFromPutters(io);	1833	continue;
1451	return buffer.len;	1834	}
		1835	pending.condition.wait(io, &q.mutex) catch \|err\| switch (err) {
		1836	error.Canceled => if (pending.remaining.len == buffer.len) {
		1837	// Canceled while waiting, and received no elements.
		1838	return error.Canceled;
		1839	} else {
		1840	// Canceled while waiting, but received some elements, so report those first.
		1841	io.recancel();
		1842	return buffer.len - pending.remaining.len;
		1843	},
		1844	};
		1845	}
		1846	if (pending.remaining.len == buffer.len) {
		1847	// The queue was closed while we were waiting. We received no elements.
		1848	assert(q.closed);
		1849	return error.Closed;
		1850	}
		1851	return buffer.len - pending.remaining.len;
1452	}	1852	}
1453		1853
1454	/// Called when there is nonzero space available in the ring buffer and	1854	/// Called when there is nonzero space available in the ring buffer and
...	@@ -1464,7 +1864,8 @@ pub const TypeErasedQueue = struct {	...	@@ -1464,7 +1864,8 @@ pub const TypeErasedQueue = struct {
1464	@memcpy(slice[0..copy_len], putter.remaining[0..copy_len]);	1864	@memcpy(slice[0..copy_len], putter.remaining[0..copy_len]);
1465	q.len += copy_len;	1865	q.len += copy_len;
1466	putter.remaining = putter.remaining[copy_len..];	1866	putter.remaining = putter.remaining[copy_len..];
1467	if (putter.remaining.len == 0) {	1867	putter.needed -\|= copy_len;
		1868	if (putter.needed == 0) {
1468	putter.condition.signal(io);	1869	putter.condition.signal(io);
1469	break;	1870	break;
1470	}	1871	}
...	@@ -1487,59 +1888,112 @@ pub fn Queue(Elem: type) type {	...	@@ -1487,59 +1888,112 @@ pub fn Queue(Elem: type) type {
1487	return .{ .type_erased = .init(@ptrCast(buffer)) };	1888	return .{ .type_erased = .init(@ptrCast(buffer)) };
1488	}	1889	}
1489		1890
1490	/// Appends elements to the end of the queue. The function returns when	1891	pub fn close(q: *@This(), io: Io) void {
1491	/// at least `min` elements have been added to the buffer or sent	1892	q.type_erased.close(io);
1492	/// directly to a consumer.	1893	}
		1894
		1895	/// Appends elements to the end of the queue, potentially blocking if
		1896	/// there is insufficient capacity. Returns when any one of the
		1897	/// following conditions is satisfied:
1493	///	1898	///
1494	/// Returns how many elements have been added to the queue.	1899	/// * At least `target` elements have been added to the queue
		1900	/// * The queue is closed
		1901	/// * The current task is canceled
1495	///	1902	///
1496	/// Asserts that `elements.len >= min`.	1903	/// Returns how many of `elements` have been added to the queue, if any.
1497	pub fn put(q: *@This(), io: Io, elements: []const Elem, min: usize) Cancelable!usize {	1904	/// If an error is returned, no elements have been added.
1498	return @divExact(try q.type_erased.put(io, @ptrCast(elements), min * @sizeOf(Elem)), @sizeOf(Elem));	1905	///
		1906	/// If the queue is closed or the task is canceled, but some items were
		1907	/// already added before the closure or cancelation, then `put` may
		1908	/// return a number lower than `target`, in which case future calls are
		1909	/// guaranteed to return `error.Canceled` or `error.Closed`.
		1910	///
		1911	/// A return value of 0 is only possible if `target` is 0, in which case
		1912	/// the call is guaranteed to queue as many of `elements` as is possible
		1913	/// without blocking.
		1914	///
		1915	/// Asserts that `elements.len >= target`.
		1916	pub fn put(q: *@This(), io: Io, elements: []const Elem, target: usize) (QueueClosedError \|\| Cancelable)!usize {
		1917	return @divExact(try q.type_erased.put(io, @ptrCast(elements), target * @sizeOf(Elem)), @sizeOf(Elem));
1499	}	1918	}
1500		1919
1501	/// Same as `put` but blocks until all elements have been added to the queue.	1920	/// Same as `put` but blocks until all elements have been added to the queue.
1502	pub fn putAll(q: *@This(), io: Io, elements: []const Elem) Cancelable!void {	1921	///
1503	assert(try q.put(io, elements, elements.len) == elements.len);	1922	/// If the queue is closed or canceled, `error.Closed` or `error.Canceled`
		1923	/// is returned, and it is unspecified how many, if any, of `elements` were
		1924	/// added to the queue prior to cancelation or closure.
		1925	pub fn putAll(q: *@This(), io: Io, elements: []const Elem) (QueueClosedError \|\| Cancelable)!void {
		1926	const n = try q.put(io, elements, elements.len);
		1927	if (n != elements.len) {
		1928	_ = try q.put(io, elements[n..], elements.len - n);
		1929	unreachable; // partial `put` implies queue was closed or we were canceled
		1930	}
1504	}	1931	}
1505		1932
1506	/// Same as `put` but cannot be interrupted.	1933	/// Same as `put`, except does not introduce a cancelation point.
1507	pub fn putUncancelable(q: *@This(), io: Io, elements: []const Elem, min: usize) usize {	1934	///
1508	return @divExact(q.type_erased.putUncancelable(io, @ptrCast(elements), min * @sizeOf(Elem)), @sizeOf(Elem));	1935	/// For a description of cancelation and cancelation points, see `Future.cancel`.
		1936	pub fn putUncancelable(q: *@This(), io: Io, elements: []const Elem, min: usize) QueueClosedError!usize {
		1937	return @divExact(try q.type_erased.putUncancelable(io, @ptrCast(elements), min * @sizeOf(Elem)), @sizeOf(Elem));
1509	}	1938	}
1510		1939
1511	pub fn putOne(q: *@This(), io: Io, item: Elem) Cancelable!void {	1940	/// Appends `item` to the end of the queue, blocking if the queue is full.
		1941	pub fn putOne(q: *@This(), io: Io, item: Elem) (QueueClosedError \|\| Cancelable)!void {
1512	assert(try q.put(io, &.{item}, 1) == 1);	1942	assert(try q.put(io, &.{item}, 1) == 1);
1513	}	1943	}
1514		1944
1515	pub fn putOneUncancelable(q: *@This(), io: Io, item: Elem) void {	1945	/// Same as `putOne`, except does not introduce a cancelation point.
1516	assert(q.putUncancelable(io, &.{item}, 1) == 1);	1946	///
		1947	/// For a description of cancelation and cancelation points, see `Future.cancel`.
		1948	pub fn putOneUncancelable(q: *@This(), io: Io, item: Elem) QueueClosedError!void {
		1949	assert(try q.putUncancelable(io, &.{item}, 1) == 1);
1517	}	1950	}
1518		1951
1519	/// Receives elements from the beginning of the queue. The function	1952	/// Receives elements from the beginning of the queue, potentially blocking
1520	/// returns when at least `min` elements have been populated inside	1953	/// if there are insufficient elements currently in the queue. Returns when
1521	/// `buffer`.	1954	/// any one of the following conditions is satisfied:
		1955	///
		1956	/// * At least `target` elements have been received from the queue
		1957	/// * The queue is closed and contains no buffered elements
		1958	/// * The current task is canceled
		1959	///
		1960	/// Returns how many elements of `buffer` have been populated, if any.
		1961	/// If an error is returned, no elements have been populated.
		1962	///
		1963	/// If the queue is closed or the task is canceled, but some items were
		1964	/// already received before the closure or cancelation, then `get` may
		1965	/// return a number lower than `target`, in which case future calls are
		1966	/// guaranteed to return `error.Canceled` or `error.Closed`.
1522	///	1967	///
1523	/// Returns how many elements of `buffer` have been populated.	1968	/// A return value of 0 is only possible if `target` is 0, in which case
		1969	/// the call is guaranteed to fill as much of `buffer` as is possible
		1970	/// without blocking.
1524	///	1971	///
1525	/// Asserts that `buffer.len >= min`.	1972	/// Asserts that `buffer.len >= target`.
1526	pub fn get(q: *@This(), io: Io, buffer: []Elem, min: usize) Cancelable!usize {	1973	pub fn get(q: *@This(), io: Io, buffer: []Elem, target: usize) (QueueClosedError \|\| Cancelable)!usize {
1527	return @divExact(try q.type_erased.get(io, @ptrCast(buffer), min * @sizeOf(Elem)), @sizeOf(Elem));	1974	return @divExact(try q.type_erased.get(io, @ptrCast(buffer), target * @sizeOf(Elem)), @sizeOf(Elem));
1528	}	1975	}
1529		1976
1530	pub fn getUncancelable(q: *@This(), io: Io, buffer: []Elem, min: usize) usize {	1977	/// Same as `get`, except does not introduce a cancelation point.
1531	return @divExact(q.type_erased.getUncancelable(io, @ptrCast(buffer), min * @sizeOf(Elem)), @sizeOf(Elem));	1978	///
		1979	/// For a description of cancelation and cancelation points, see `Future.cancel`.
		1980	pub fn getUncancelable(q: *@This(), io: Io, buffer: []Elem, min: usize) QueueClosedError!usize {
		1981	return @divExact(try q.type_erased.getUncancelable(io, @ptrCast(buffer), min * @sizeOf(Elem)), @sizeOf(Elem));
1532	}	1982	}
1533		1983
1534	pub fn getOne(q: *@This(), io: Io) Cancelable!Elem {	1984	/// Receives one element from the beginning of the queue, blocking if the queue is empty.
		1985	pub fn getOne(q: *@This(), io: Io) (QueueClosedError \|\| Cancelable)!Elem {
1535	var buf: [1]Elem = undefined;	1986	var buf: [1]Elem = undefined;
1536	assert(try q.get(io, &buf, 1) == 1);	1987	assert(try q.get(io, &buf, 1) == 1);
1537	return buf[0];	1988	return buf[0];
1538	}	1989	}
1539		1990
1540	pub fn getOneUncancelable(q: *@This(), io: Io) Elem {	1991	/// Same as `getOne`, except does not introduce a cancelation point.
		1992	///
		1993	/// For a description of cancelation and cancelation points, see `Future.cancel`.
		1994	pub fn getOneUncancelable(q: *@This(), io: Io) QueueClosedError!Elem {
1541	var buf: [1]Elem = undefined;	1995	var buf: [1]Elem = undefined;
1542	assert(q.getUncancelable(io, &buf, 1) == 1);	1996	assert(try q.getUncancelable(io, &buf, 1) == 1);
1543	return buf[0];	1997	return buf[0];
1544	}	1998	}
1545		1999
...	@@ -1627,10 +2081,6 @@ pub fn concurrent(	...	@@ -1627,10 +2081,6 @@ pub fn concurrent(
1627	return future;	2081	return future;
1628	}	2082	}
1629		2083
1630	pub fn cancelRequested(io: Io) bool {
1631	return io.vtable.cancelRequested(io.userdata);
1632	}
1633
1634	pub const SleepError = error{UnsupportedClock} \|\| UnexpectedError \|\| Cancelable;	2084	pub const SleepError = error{UnsupportedClock} \|\| UnexpectedError \|\| Cancelable;
1635		2085
1636	pub fn sleep(io: Io, duration: Duration, clock: Clock) SleepError!void {	2086	pub fn sleep(io: Io, duration: Duration, clock: Clock) SleepError!void {

lib/std/Io/Threaded.zig+351-675

...	@@ -86,7 +86,9 @@ const Thread = struct {	...	@@ -86,7 +86,9 @@ const Thread = struct {
86	/// The value that needs to be passed to pthread_kill or tgkill in order to	86	/// The value that needs to be passed to pthread_kill or tgkill in order to
87	/// send a signal.	87	/// send a signal.
88	signal_id: SignaleeId,	88	signal_id: SignaleeId,
89	current_closure: ?*Closure = null,	89	current_closure: ?*Closure,
		90	/// Only populated if `current_closure != null`. Indicates the current cancel protection mode.
		91	cancel_protection: Io.CancelProtection,
90		92
91	const SignaleeId = if (std.Thread.use_pthreads) std.c.pthread_t else std.Thread.Id;	93	const SignaleeId = if (std.Thread.use_pthreads) std.c.pthread_t else std.Thread.Id;
92		94
...	@@ -98,6 +100,12 @@ const Thread = struct {	...	@@ -98,6 +100,12 @@ const Thread = struct {
98		100
99	fn checkCancel(thread: *Thread) error{Canceled}!void {	101	fn checkCancel(thread: *Thread) error{Canceled}!void {
100	const closure = thread.current_closure orelse return;	102	const closure = thread.current_closure orelse return;
		103
		104	switch (thread.cancel_protection) {
		105	.unblocked => {},
		106	.blocked => return,
		107	}
		108
101	switch (@cmpxchgStrong(	109	switch (@cmpxchgStrong(
102	CancelStatus,	110	CancelStatus,
103	&closure.cancel_status,	111	&closure.cancel_status,
...	@@ -115,6 +123,11 @@ const Thread = struct {	...	@@ -115,6 +123,11 @@ const Thread = struct {
115	fn beginSyscall(thread: *Thread) error{Canceled}!void {	123	fn beginSyscall(thread: *Thread) error{Canceled}!void {
116	const closure = thread.current_closure orelse return;	124	const closure = thread.current_closure orelse return;
117		125
		126	switch (thread.cancel_protection) {
		127	.unblocked => {},
		128	.blocked => return,
		129	}
		130
118	switch (@cmpxchgStrong(	131	switch (@cmpxchgStrong(
119	CancelStatus,	132	CancelStatus,
120	&closure.cancel_status,	133	&closure.cancel_status,
...	@@ -135,6 +148,12 @@ const Thread = struct {	...	@@ -135,6 +148,12 @@ const Thread = struct {
135		148
136	fn endSyscall(thread: *Thread) void {	149	fn endSyscall(thread: *Thread) void {
137	const closure = thread.current_closure orelse return;	150	const closure = thread.current_closure orelse return;
		151
		152	switch (thread.cancel_protection) {
		153	.unblocked => {},
		154	.blocked => return,
		155	}
		156
138	_ = @cmpxchgStrong(	157	_ = @cmpxchgStrong(
139	CancelStatus,	158	CancelStatus,
140	&closure.cancel_status,	159	&closure.cancel_status,
...	@@ -155,6 +174,220 @@ const Thread = struct {	...	@@ -155,6 +174,220 @@ const Thread = struct {
155	fn currentSignalId() SignaleeId {	174	fn currentSignalId() SignaleeId {
156	return if (std.Thread.use_pthreads) std.c.pthread_self() else std.Thread.getCurrentId();	175	return if (std.Thread.use_pthreads) std.c.pthread_self() else std.Thread.getCurrentId();
157	}	176	}
		177
		178	fn futexWaitUncancelable(ptr: *const u32, expect: u32) void {
		179	return Thread.futexWaitTimed(null, ptr, expect, null) catch unreachable;
		180	}
		181
		182	fn futexWait(thread: Thread, ptr: const u32, expect: u32) Io.Cancelable!void {
		183	return Thread.futexWaitTimed(thread, ptr, expect, null) catch \|err\| switch (err) {
		184	error.Canceled => return error.Canceled,
		185	error.Timeout => unreachable,
		186	};
		187	}
		188
		189	fn futexWaitTimed(thread: ?Thread, ptr: const u32, expect: u32, timeout_ns: ?u64) Io.Cancelable!void {
		190	@branchHint(.cold);
		191
		192	if (builtin.single_threaded) unreachable; // nobody would ever wake us
		193
		194	if (builtin.cpu.arch.isWasm()) {
		195	comptime assert(builtin.cpu.has(.wasm, .atomics));
		196	if (thread) \|t\| try t.checkCancel();
		197	const to: i64 = if (timeout_ns) \|ns\| ns else -1;
		198	const signed_expect: i32 = @bitCast(expect);
		199	const result = asm volatile (
		200	\\local.get %[ptr]
		201	\\local.get %[expected]
		202	\\local.get %[timeout]
		203	\\memory.atomic.wait32 0
		204	\\local.set %[ret]
		205	: [ret] "=r" (-> u32),
		206	: [ptr] "r" (ptr),
		207	[expected] "r" (signed_expect),
		208	[timeout] "r" (to),
		209	);
		210	switch (result) {
		211	0 => {}, // ok
		212	1 => {}, // expected != loaded
		213	2 => {}, // timeout
		214	else => assert(!is_debug),
		215	}
		216	} else switch (native_os) {
		217	.linux => {
		218	const linux = std.os.linux;
		219	var ts_buffer: linux.timespec = undefined;
		220	const ts: ?*linux.timespec = if (timeout_ns) \|ns\| ts: {
		221	ts_buffer = timestampToPosix(ns);
		222	break :ts &ts_buffer;
		223	} else null;
		224	if (thread) \|t\| try t.beginSyscall();
		225	const rc = linux.futex_4arg(ptr, .{ .cmd = .WAIT, .private = true }, expect, ts);
		226	if (thread) \|t\| t.endSyscall();
		227	switch (linux.errno(rc)) {
		228	.SUCCESS => {}, // notified by `wake()`
		229	.INTR => {}, // caller's responsibility to retry
		230	.AGAIN => {}, // ptr.* != expect
		231	.INVAL => {}, // possibly timeout overflow
		232	.TIMEDOUT => {}, // timeout
		233	.FAULT => recoverableOsBugDetected(), // ptr was invalid
		234	else => recoverableOsBugDetected(),
		235	}
		236	},
		237	.driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => {
		238	const c = std.c;
		239	const flags: c.UL = .{
		240	.op = .COMPARE_AND_WAIT,
		241	.NO_ERRNO = true,
		242	};
		243	if (thread) \|t\| try t.beginSyscall();
		244	const status = switch (darwin_supports_ulock_wait2) {
		245	true => c.__ulock_wait2(flags, ptr, expect, ns: {
		246	const ns = timeout_ns orelse break :ns 0;
		247	if (ns == 0) break :ns 1;
		248	break :ns ns;
		249	}, 0),
		250	false => c.__ulock_wait(flags, ptr, expect, us: {
		251	const ns = timeout_ns orelse break :us 0;
		252	const us = std.math.lossyCast(u32, ns / std.time.ns_per_us);
		253	if (us == 0) break :us 1;
		254	break :us us;
		255	}),
		256	};
		257	if (thread) \|t\| t.endSyscall();
		258	if (status >= 0) return;
		259	switch (@as(c.E, @enumFromInt(-status))) {
		260	.INTR => {}, // spurious wake
		261	// Address of the futex was paged out. This is unlikely, but possible in theory, and
		262	// pthread/libdispatch on darwin bother to handle it. In this case we'll return
		263	// without waiting, but the caller should retry anyway.
		264	.FAULT => {},
		265	.TIMEDOUT => {}, // timeout
		266	else => recoverableOsBugDetected(),
		267	}
		268	},
		269	.windows => {
		270	var timeout_value: windows.LARGE_INTEGER = undefined;
		271	var timeout_ptr: ?*const windows.LARGE_INTEGER = null;
		272	// NTDLL functions work with time in units of 100 nanoseconds.
		273	// Positive values are absolute deadlines while negative values are relative durations.
		274	if (timeout_ns) \|delay\| {
		275	timeout_value = @as(windows.LARGE_INTEGER, @intCast(delay / 100));
		276	timeout_value = -timeout_value;
		277	timeout_ptr = &timeout_value;
		278	}
		279	if (thread) \|t\| try t.checkCancel();
		280	switch (windows.ntdll.RtlWaitOnAddress(ptr, &expect, @sizeOf(@TypeOf(expect)), timeout_ptr)) {
		281	.SUCCESS => {},
		282	.CANCELLED => {},
		283	.TIMEOUT => {}, // timeout
		284	else => recoverableOsBugDetected(),
		285	}
		286	},
		287	.freebsd => {
		288	const flags = @intFromEnum(std.c.UMTX_OP.WAIT_UINT_PRIVATE);
		289	var tm_size: usize = 0;
		290	var tm: std.c._umtx_time = undefined;
		291	var tm_ptr: ?*const std.c._umtx_time = null;
		292	if (timeout_ns) \|ns\| {
		293	tm_ptr = &tm;
		294	tm_size = @sizeOf(@TypeOf(tm));
		295	tm.flags = 0; // use relative time not UMTX_ABSTIME
		296	tm.clockid = .MONOTONIC;
		297	tm.timeout = timestampToPosix(ns);
		298	}
		299	if (thread) \|t\| try t.beginSyscall();
		300	const rc = std.c._umtx_op(@intFromPtr(ptr), flags, @as(c_ulong, expect), tm_size, @intFromPtr(tm_ptr));
		301	if (thread) \|t\| t.endSyscall();
		302	if (is_debug) switch (posix.errno(rc)) {
		303	.SUCCESS => {},
		304	.FAULT => unreachable, // one of the args points to invalid memory
		305	.INVAL => unreachable, // arguments should be correct
		306	.TIMEDOUT => {}, // timeout
		307	.INTR => {}, // spurious wake
		308	else => unreachable,
		309	};
		310	},
		311	else => @compileError("unimplemented: futexWait"),
		312	}
		313	}
		314
		315	fn futexWake(ptr: *const u32, max_waiters: u32) void {
		316	@branchHint(.cold);
		317
		318	if (builtin.single_threaded) return; // nothing to wake up
		319
		320	if (builtin.cpu.arch.isWasm()) {
		321	comptime assert(builtin.cpu.has(.wasm, .atomics));
		322	assert(max_waiters != 0);
		323	const woken_count = asm volatile (
		324	\\local.get %[ptr]
		325	\\local.get %[waiters]
		326	\\memory.atomic.notify 0
		327	\\local.set %[ret]
		328	: [ret] "=r" (-> u32),
		329	: [ptr] "r" (ptr),
		330	[waiters] "r" (max_waiters),
		331	);
		332	_ = woken_count; // can be 0 when linker flag 'shared-memory' is not enabled
		333	} else switch (native_os) {
		334	.linux => {
		335	const linux = std.os.linux;
		336	switch (linux.errno(linux.futex_3arg(
		337	ptr,
		338	.{ .cmd = .WAKE, .private = true },
		339	@min(max_waiters, std.math.maxInt(i32)),
		340	))) {
		341	.SUCCESS => return, // successful wake up
		342	.INVAL => return, // invalid futex_wait() on ptr done elsewhere
		343	.FAULT => return, // pointer became invalid while doing the wake
		344	else => return recoverableOsBugDetected(), // deadlock due to operating system bug
		345	}
		346	},
		347	.driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => {
		348	const c = std.c;
		349	const flags: c.UL = .{
		350	.op = .COMPARE_AND_WAIT,
		351	.NO_ERRNO = true,
		352	.WAKE_ALL = max_waiters > 1,
		353	};
		354	while (true) {
		355	const status = c.__ulock_wake(flags, ptr, 0);
		356	if (status >= 0) return;
		357	switch (@as(c.E, @enumFromInt(-status))) {
		358	.INTR, .CANCELED => continue, // spurious wake()
		359	.FAULT => unreachable, // __ulock_wake doesn't generate EFAULT according to darwin pthread_cond_t
		360	.NOENT => return, // nothing was woken up
		361	.ALREADY => unreachable, // only for UL.Op.WAKE_THREAD
		362	else => unreachable, // deadlock due to operating system bug
		363	}
		364	}
		365	},
		366	.windows => {
		367	assert(max_waiters != 0);
		368	switch (max_waiters) {
		369	1 => windows.ntdll.RtlWakeAddressSingle(ptr),
		370	else => windows.ntdll.RtlWakeAddressAll(ptr),
		371	}
		372	},
		373	.freebsd => {
		374	const rc = std.c._umtx_op(
		375	@intFromPtr(ptr),
		376	@intFromEnum(std.c.UMTX_OP.WAKE_PRIVATE),
		377	@as(c_ulong, max_waiters),
		378	0, // there is no timeout struct
		379	0, // there is no timeout struct pointer
		380	);
		381	switch (posix.errno(rc)) {
		382	.SUCCESS => {},
		383	.FAULT => {}, // it's ok if the ptr doesn't point to valid memory
		384	.INVAL => unreachable, // arguments should be correct
		385	else => unreachable, // deadlock due to operating system bug
		386	}
		387	},
		388	else => @compileError("unimplemented: futexWake"),
		389	}
		390	}
158	};	391	};
159		392
160	const max_iovecs_len = 8;	393	const max_iovecs_len = 8;
...	@@ -298,6 +531,8 @@ pub fn init(	...	@@ -298,6 +531,8 @@ pub fn init(
298	.have_signal_handler = false,	531	.have_signal_handler = false,
299	.main_thread = .{	532	.main_thread = .{
300	.signal_id = Thread.currentSignalId(),	533	.signal_id = Thread.currentSignalId(),
		534	.current_closure = null,
		535	.cancel_protection = undefined,
301	},	536	},
302	};	537	};
303		538
...	@@ -332,7 +567,11 @@ pub const init_single_threaded: Threaded = .{	...	@@ -332,7 +567,11 @@ pub const init_single_threaded: Threaded = .{
332	.old_sig_io = undefined,	567	.old_sig_io = undefined,
333	.old_sig_pipe = undefined,	568	.old_sig_pipe = undefined,
334	.have_signal_handler = false,	569	.have_signal_handler = false,
335	.main_thread = .{ .signal_id = undefined },	570	.main_thread = .{
		571	.signal_id = undefined,
		572	.current_closure = null,
		573	.cancel_protection = undefined,
		574	},
336	};	575	};
337		576
338	pub fn setAsyncLimit(t: *Threaded, new_limit: Io.Limit) void {	577	pub fn setAsyncLimit(t: *Threaded, new_limit: Io.Limit) void {
...	@@ -367,6 +606,8 @@ fn join(t: *Threaded) void {	...	@@ -367,6 +606,8 @@ fn join(t: *Threaded) void {
367	fn worker(t: *Threaded) void {	606	fn worker(t: *Threaded) void {
368	var thread: Thread = .{	607	var thread: Thread = .{
369	.signal_id = Thread.currentSignalId(),	608	.signal_id = Thread.currentSignalId(),
		609	.current_closure = null,
		610	.cancel_protection = undefined,
370	};	611	};
371	Thread.current = &thread;	612	Thread.current = &thread;
372		613
...	@@ -403,13 +644,13 @@ pub fn io(t: *Threaded) Io {	...	@@ -403,13 +644,13 @@ pub fn io(t: *Threaded) Io {
403	.groupWait = groupWait,	644	.groupWait = groupWait,
404	.groupCancel = groupCancel,	645	.groupCancel = groupCancel,
405		646
406	.mutexLock = mutexLock,	647	.recancel = recancel,
407	.mutexLockUncancelable = mutexLockUncancelable,	648	.swapCancelProtection = swapCancelProtection,
408	.mutexUnlock = mutexUnlock,	649	.checkCancel = checkCancel,
409		650
410	.conditionWait = conditionWait,	651	.futexWait = futexWait,
411	.conditionWaitUncancelable = conditionWaitUncancelable,	652	.futexWaitUncancelable = futexWaitUncancelable,
412	.conditionWake = conditionWake,	653	.futexWake = futexWake,
413		654
414	.dirMake = dirMake,	655	.dirMake = dirMake,
415	.dirMakePath = dirMakePath,	656	.dirMakePath = dirMakePath,
...	@@ -499,13 +740,13 @@ pub fn ioBasic(t: *Threaded) Io {	...	@@ -499,13 +740,13 @@ pub fn ioBasic(t: *Threaded) Io {
499	.groupWait = groupWait,	740	.groupWait = groupWait,
500	.groupCancel = groupCancel,	741	.groupCancel = groupCancel,
501		742
502	.mutexLock = mutexLock,	743	.recancel = recancel,
503	.mutexLockUncancelable = mutexLockUncancelable,	744	.swapCancelProtection = swapCancelProtection,
504	.mutexUnlock = mutexUnlock,	745	.checkCancel = checkCancel,
505		746
506	.conditionWait = conditionWait,	747	.futexWait = futexWait,
507	.conditionWaitUncancelable = conditionWaitUncancelable,	748	.futexWaitUncancelable = futexWaitUncancelable,
508	.conditionWake = conditionWake,	749	.futexWake = futexWake,
509		750
510	.dirMake = dirMake,	751	.dirMake = dirMake,
511	.dirMakePath = dirMakePath,	752	.dirMakePath = dirMakePath,
...	@@ -577,26 +818,31 @@ const preadv_sym = if (posix.lfs64_abi) posix.system.preadv64 else posix.system.	...	@@ -577,26 +818,31 @@ const preadv_sym = if (posix.lfs64_abi) posix.system.preadv64 else posix.system.
577	const AsyncClosure = struct {	818	const AsyncClosure = struct {
578	closure: Closure,	819	closure: Closure,
579	func: const fn (context: anyopaque, result: *anyopaque) void,	820	func: const fn (context: anyopaque, result: *anyopaque) void,
580	reset_event: ResetEvent,	821	event: Io.Event,
581	select_condition: ?*ResetEvent,	822	select_condition: ?*Io.Event,
582	context_alignment: Alignment,	823	context_alignment: Alignment,
583	result_offset: usize,	824	result_offset: usize,
584	alloc_len: usize,	825	alloc_len: usize,
585		826
586	const done_reset_event: *ResetEvent = @ptrFromInt(@alignOf(ResetEvent));	827	const done_event: *Io.Event = @ptrFromInt(@alignOf(Io.Event));
587		828
588	fn start(closure: Closure, t: Threaded) void {	829	fn start(closure: Closure, t: Threaded) void {
589	const ac: *AsyncClosure = @alignCast(@fieldParentPtr("closure", closure));	830	const ac: *AsyncClosure = @alignCast(@fieldParentPtr("closure", closure));
590	const current_thread = Thread.getCurrent(t);	831	const current_thread = Thread.getCurrent(t);
		832
591	current_thread.current_closure = closure;	833	current_thread.current_closure = closure;
		834	current_thread.cancel_protection = .unblocked;
		835
592	ac.func(ac.contextPointer(), ac.resultPointer());	836	ac.func(ac.contextPointer(), ac.resultPointer());
		837
593	current_thread.current_closure = null;	838	current_thread.current_closure = null;
		839	current_thread.cancel_protection = undefined;
594		840
595	if (@atomicRmw(?*ResetEvent, &ac.select_condition, .Xchg, done_reset_event, .release)) \|select_reset\| {	841	if (@atomicRmw(?*Io.Event, &ac.select_condition, .Xchg, done_event, .release)) \|select_event\| {
596	assert(select_reset != done_reset_event);	842	assert(select_event != done_event);
597	select_reset.set();	843	select_event.set(ioBasic(t));
598	}	844	}
599	ac.reset_event.set();	845	ac.event.set(ioBasic(t));
600	}	846	}
601		847
602	fn resultPointer(ac: AsyncClosure) []u8 {	848	fn resultPointer(ac: AsyncClosure) []u8 {
...	@@ -638,7 +884,7 @@ const AsyncClosure = struct {	...	@@ -638,7 +884,7 @@ const AsyncClosure = struct {
638	.context_alignment = context_alignment,	884	.context_alignment = context_alignment,
639	.result_offset = actual_result_offset,	885	.result_offset = actual_result_offset,
640	.alloc_len = alloc_len,	886	.alloc_len = alloc_len,
641	.reset_event = .unset,	887	.event = .unset,
642	.select_condition = null,	888	.select_condition = null,
643	};	889	};
644	@memcpy(ac.contextPointer()[0..context.len], context);	890	@memcpy(ac.contextPointer()[0..context.len], context);
...	@@ -646,10 +892,10 @@ const AsyncClosure = struct {	...	@@ -646,10 +892,10 @@ const AsyncClosure = struct {
646	}	892	}
647		893
648	fn waitAndDeinit(ac: AsyncClosure, t: Threaded, result: []u8) void {	894	fn waitAndDeinit(ac: AsyncClosure, t: Threaded, result: []u8) void {
649	ac.reset_event.wait(t) catch \|err\| switch (err) {	895	ac.event.wait(ioBasic(t)) catch \|err\| switch (err) {
650	error.Canceled => {	896	error.Canceled => {
651	ac.closure.requestCancel(t);	897	ac.closure.requestCancel(t);
652	ac.reset_event.waitUncancelable();	898	ac.event.waitUncancelable(ioBasic(t));
653	},	899	},
654	};	900	};
655	@memcpy(result, ac.resultPointer()[0..result.len]);	901	@memcpy(result, ac.resultPointer()[0..result.len]);
...	@@ -771,14 +1017,19 @@ const GroupClosure = struct {	...	@@ -771,14 +1017,19 @@ const GroupClosure = struct {
771	const current_thread = Thread.getCurrent(t);	1017	const current_thread = Thread.getCurrent(t);
772	const group = gc.group;	1018	const group = gc.group;
773	const group_state: *std.atomic.Value(usize) = @ptrCast(&group.state);	1019	const group_state: *std.atomic.Value(usize) = @ptrCast(&group.state);
774	const reset_event: *ResetEvent = @ptrCast(&group.context);	1020	const event: *Io.Event = @ptrCast(&group.context);
		1021
775	current_thread.current_closure = closure;	1022	current_thread.current_closure = closure;
		1023	current_thread.cancel_protection = .unblocked;
		1024
776	gc.func(group, gc.contextPointer());	1025	gc.func(group, gc.contextPointer());
		1026
777	current_thread.current_closure = null;	1027	current_thread.current_closure = null;
		1028	current_thread.cancel_protection = undefined;
778		1029
779	const prev_state = group_state.fetchSub(sync_one_pending, .acq_rel);	1030	const prev_state = group_state.fetchSub(sync_one_pending, .acq_rel);
780	assert((prev_state / sync_one_pending) > 0);	1031	assert((prev_state / sync_one_pending) > 0);
781	if (prev_state == (sync_one_pending \| sync_is_waiting)) reset_event.set();	1032	if (prev_state == (sync_one_pending \| sync_is_waiting)) event.set(ioBasic(t));
782	}	1033	}
783		1034
784	fn contextPointer(gc: GroupClosure) []u8 {	1035	fn contextPointer(gc: GroupClosure) []u8 {
...	@@ -939,10 +1190,10 @@ fn groupWait(userdata: ?anyopaque, group: Io.Group, token: *anyopaque) void {	...	@@ -939,10 +1190,10 @@ fn groupWait(userdata: ?anyopaque, group: Io.Group, token: *anyopaque) void {
939	if (builtin.single_threaded) return;	1190	if (builtin.single_threaded) return;
940		1191
941	const group_state: *std.atomic.Value(usize) = @ptrCast(&group.state);	1192	const group_state: *std.atomic.Value(usize) = @ptrCast(&group.state);
942	const reset_event: *ResetEvent = @ptrCast(&group.context);	1193	const event: *Io.Event = @ptrCast(&group.context);
943	const prev_state = group_state.fetchAdd(GroupClosure.sync_is_waiting, .acquire);	1194	const prev_state = group_state.fetchAdd(GroupClosure.sync_is_waiting, .acquire);
944	assert(prev_state & GroupClosure.sync_is_waiting == 0);	1195	assert(prev_state & GroupClosure.sync_is_waiting == 0);
945	if ((prev_state / GroupClosure.sync_one_pending) > 0) reset_event.wait(t) catch \|err\| switch (err) {	1196	if ((prev_state / GroupClosure.sync_one_pending) > 0) event.wait(ioBasic(t)) catch \|err\| switch (err) {
946	error.Canceled => {	1197	error.Canceled => {
947	var node: *std.SinglyLinkedList.Node = @ptrCast(@alignCast(token));	1198	var node: *std.SinglyLinkedList.Node = @ptrCast(@alignCast(token));
948	while (true) {	1199	while (true) {
...	@@ -950,7 +1201,7 @@ fn groupWait(userdata: ?anyopaque, group: Io.Group, token: *anyopaque) void {	...	@@ -950,7 +1201,7 @@ fn groupWait(userdata: ?anyopaque, group: Io.Group, token: *anyopaque) void {
950	gc.closure.requestCancel(t);	1201	gc.closure.requestCancel(t);
951	node = node.next orelse break;	1202	node = node.next orelse break;
952	}	1203	}
953	reset_event.waitUncancelable();	1204	event.waitUncancelable(ioBasic(t));
954	},	1205	},
955	};	1206	};
956		1207
...	@@ -979,10 +1230,10 @@ fn groupCancel(userdata: ?anyopaque, group: Io.Group, token: *anyopaque) void	...	@@ -979,10 +1230,10 @@ fn groupCancel(userdata: ?anyopaque, group: Io.Group, token: *anyopaque) void
979	}	1230	}
980		1231
981	const group_state: *std.atomic.Value(usize) = @ptrCast(&group.state);	1232	const group_state: *std.atomic.Value(usize) = @ptrCast(&group.state);
982	const reset_event: *ResetEvent = @ptrCast(&group.context);	1233	const event: *Io.Event = @ptrCast(&group.context);
983	const prev_state = group_state.fetchAdd(GroupClosure.sync_is_waiting, .acquire);	1234	const prev_state = group_state.fetchAdd(GroupClosure.sync_is_waiting, .acquire);
984	assert(prev_state & GroupClosure.sync_is_waiting == 0);	1235	assert(prev_state & GroupClosure.sync_is_waiting == 0);
985	if ((prev_state / GroupClosure.sync_one_pending) > 0) reset_event.waitUncancelable();	1236	if ((prev_state / GroupClosure.sync_one_pending) > 0) event.waitUncancelable(ioBasic(t));
986		1237
987	{	1238	{
988	var node: *std.SinglyLinkedList.Node = @ptrCast(@alignCast(token));	1239	var node: *std.SinglyLinkedList.Node = @ptrCast(@alignCast(token));
...	@@ -995,6 +1246,32 @@ fn groupCancel(userdata: ?anyopaque, group: Io.Group, token: *anyopaque) void	...	@@ -995,6 +1246,32 @@ fn groupCancel(userdata: ?anyopaque, group: Io.Group, token: *anyopaque) void
995	}	1246	}
996	}	1247	}
997		1248
		1249	fn recancel(userdata: ?*anyopaque) void {
		1250	const t: *Threaded = @ptrCast(@alignCast(userdata));
		1251	const current_thread: *Thread = .getCurrent(t);
		1252	const cancel_status = &current_thread.current_closure.?.cancel_status;
		1253	switch (@atomicLoad(CancelStatus, cancel_status, .monotonic)) {
		1254	.none => unreachable, // called `recancel` when not canceled
		1255	.requested => unreachable, // called `recancel` when cancelation was already outstanding
		1256	.acknowledged => {},
		1257	_ => unreachable, // invalid state: not in a syscall
		1258	}
		1259	@atomicStore(CancelStatus, cancel_status, .requested, .monotonic);
		1260	}
		1261
		1262	fn swapCancelProtection(userdata: ?*anyopaque, new: Io.CancelProtection) Io.CancelProtection {
		1263	const t: *Threaded = @ptrCast(@alignCast(userdata));
		1264	const current_thread: *Thread = .getCurrent(t);
		1265	const old = current_thread.cancel_protection;
		1266	current_thread.cancel_protection = new;
		1267	return old;
		1268	}
		1269
		1270	fn checkCancel(userdata: ?*anyopaque) Io.Cancelable!void {
		1271	const t: *Threaded = @ptrCast(@alignCast(userdata));
		1272	return Thread.getCurrent(t).checkCancel();
		1273	}
		1274
998	fn await(	1275	fn await(
999	userdata: ?*anyopaque,	1276	userdata: ?*anyopaque,
1000	any_future: *Io.AnyFuture,	1277	any_future: *Io.AnyFuture,
...	@@ -1020,187 +1297,35 @@ fn cancel(	...	@@ -1020,187 +1297,35 @@ fn cancel(
1020	ac.waitAndDeinit(t, result);	1297	ac.waitAndDeinit(t, result);
1021	}	1298	}
1022		1299
1023	fn mutexLock(userdata: ?anyopaque, prev_state: Io.Mutex.State, mutex: Io.Mutex) Io.Cancelable!void {	1300	fn futexWait(userdata: ?anyopaque, ptr: const u32, expected: u32, timeout: Io.Timeout) Io.Cancelable!void {
1024	if (builtin.single_threaded) unreachable; // Interface should have prevented this.
1025	if (native_os == .netbsd) @panic("TODO");
1026	if (native_os == .openbsd) @panic("TODO");
1027	const t: *Threaded = @ptrCast(@alignCast(userdata));	1301	const t: *Threaded = @ptrCast(@alignCast(userdata));
1028	const current_thread = Thread.getCurrent(t);	1302	const current_thread = Thread.getCurrent(t);
1029	if (prev_state == .contended) {
1030	try futexWait(current_thread, @ptrCast(&mutex.state), @intFromEnum(Io.Mutex.State.contended));
1031	}
1032	while (@atomicRmw(Io.Mutex.State, &mutex.state, .Xchg, .contended, .acquire) != .unlocked) {
1033	try futexWait(current_thread, @ptrCast(&mutex.state), @intFromEnum(Io.Mutex.State.contended));
1034	}
1035	}
1036
1037	fn mutexLockUncancelable(userdata: ?anyopaque, prev_state: Io.Mutex.State, mutex: Io.Mutex) void {
1038	if (builtin.single_threaded) unreachable; // Interface should have prevented this.
1039	if (native_os == .netbsd) @panic("TODO");
1040	if (native_os == .openbsd) @panic("TODO");
1041	_ = userdata;
1042	if (prev_state == .contended) {
1043	futexWaitUncancelable(@ptrCast(&mutex.state), @intFromEnum(Io.Mutex.State.contended));
1044	}
1045	while (@atomicRmw(Io.Mutex.State, &mutex.state, .Xchg, .contended, .acquire) != .unlocked) {
1046	futexWaitUncancelable(@ptrCast(&mutex.state), @intFromEnum(Io.Mutex.State.contended));
1047	}
1048	}
1049
1050	fn mutexUnlock(userdata: ?anyopaque, prev_state: Io.Mutex.State, mutex: Io.Mutex) void {
1051	if (builtin.single_threaded) unreachable; // Interface should have prevented this.
1052	if (native_os == .netbsd) @panic("TODO");
1053	if (native_os == .openbsd) @panic("TODO");
1054	_ = userdata;
1055	_ = prev_state;
1056	if (@atomicRmw(Io.Mutex.State, &mutex.state, .Xchg, .unlocked, .release) == .contended) {
1057	futexWake(@ptrCast(&mutex.state), 1);
1058	}
1059	}
1060
1061	fn conditionWaitUncancelable(userdata: ?anyopaque, cond: Io.Condition, mutex: *Io.Mutex) void {
1062	if (builtin.single_threaded) unreachable; // Deadlock.
1063	if (native_os == .netbsd) @panic("TODO");
1064	if (native_os == .openbsd) @panic("TODO");
1065	const t: *Threaded = @ptrCast(@alignCast(userdata));
1066	const t_io = ioBasic(t);	1303	const t_io = ioBasic(t);
1067	comptime assert(@TypeOf(cond.state) == u64);	1304	const timeout_ns: ?u64 = ns: {
1068	const ints: *[2]std.atomic.Value(u32) = @ptrCast(&cond.state);	1305	const d = (timeout.toDurationFromNow(t_io) catch break :ns 10) orelse break :ns null;
1069	const cond_state = &ints[0];	1306	break :ns std.math.lossyCast(u64, d.raw.toNanoseconds());
1070	const cond_epoch = &ints[1];	1307	};
1071	const one_waiter = 1;	1308	switch (native_os) {
1072	const waiter_mask = 0xffff;	1309	.illumos, .netbsd, .openbsd => @panic("TODO"),
1073	const one_signal = 1 << 16;	1310	else => try current_thread.futexWaitTimed(ptr, expected, timeout_ns),
1074	const signal_mask = 0xffff << 16;
1075	var epoch = cond_epoch.load(.acquire);
1076	var state = cond_state.fetchAdd(one_waiter, .monotonic);
1077	assert(state & waiter_mask != waiter_mask);
1078	state += one_waiter;
1079
1080	mutex.unlock(t_io);
1081	defer mutex.lockUncancelable(t_io);
1082
1083	while (true) {
1084	futexWaitUncancelable(cond_epoch, epoch);
1085	epoch = cond_epoch.load(.acquire);
1086	state = cond_state.load(.monotonic);
1087	while (state & signal_mask != 0) {
1088	const new_state = state - one_waiter - one_signal;
1089	state = cond_state.cmpxchgWeak(state, new_state, .acquire, .monotonic) orelse return;
1090	}
1091	}	1311	}
1092	}	1312	}
1093		1313
1094	fn conditionWait(userdata: ?anyopaque, cond: Io.Condition, mutex: *Io.Mutex) Io.Cancelable!void {	1314	fn futexWaitUncancelable(userdata: ?anyopaque, ptr: const u32, expected: u32) void {
1095	if (builtin.single_threaded) unreachable; // Deadlock.
1096	if (native_os == .netbsd) @panic("TODO");
1097	if (native_os == .openbsd) @panic("TODO");
1098	const t: *Threaded = @ptrCast(@alignCast(userdata));	1315	const t: *Threaded = @ptrCast(@alignCast(userdata));
1099	const current_thread = Thread.getCurrent(t);	1316	_ = t;
1100	const t_io = ioBasic(t);	1317	switch (native_os) {
1101	comptime assert(@TypeOf(cond.state) == u64);	1318	.illumos, .netbsd, .openbsd => @panic("TODO"),
1102	const ints: *[2]std.atomic.Value(u32) = @ptrCast(&cond.state);	1319	else => Thread.futexWaitUncancelable(ptr, expected),
1103	const cond_state = &ints[0];
1104	const cond_epoch = &ints[1];
1105	const one_waiter = 1;
1106	const waiter_mask = 0xffff;
1107	const one_signal = 1 << 16;
1108	const signal_mask = 0xffff << 16;
1109	// Observe the epoch, then check the state again to see if we should wake up.
1110	// The epoch must be observed before we check the state or we could potentially miss a wake() and deadlock:
1111	//
1112	// - T1: s = LOAD(&state)
1113	// - T2: UPDATE(&s, signal)
1114	// - T2: UPDATE(&epoch, 1) + FUTEX_WAKE(&epoch)
1115	// - T1: e = LOAD(&epoch) (was reordered after the state load)
1116	// - T1: s & signals == 0 -> FUTEX_WAIT(&epoch, e) (missed the state update + the epoch change)
1117	//
1118	// Acquire barrier to ensure the epoch load happens before the state load.
1119	var epoch = cond_epoch.load(.acquire);
1120	var state = cond_state.fetchAdd(one_waiter, .monotonic);
1121	assert(state & waiter_mask != waiter_mask);
1122	state += one_waiter;
1123
1124	mutex.unlock(t_io);
1125	defer mutex.lockUncancelable(t_io);
1126
1127	while (true) {
1128	try futexWait(current_thread, cond_epoch, epoch);
1129
1130	epoch = cond_epoch.load(.acquire);
1131	state = cond_state.load(.monotonic);
1132
1133	// Try to wake up by consuming a signal and decremented the waiter we
1134	// added previously. Acquire barrier ensures code before the wake()
1135	// which added the signal happens before we decrement it and return.
1136	while (state & signal_mask != 0) {
1137	const new_state = state - one_waiter - one_signal;
1138	state = cond_state.cmpxchgWeak(state, new_state, .acquire, .monotonic) orelse return;
1139	}
1140	}	1320	}
1141	}	1321	}
1142		1322
1143	fn conditionWake(userdata: ?anyopaque, cond: Io.Condition, wake: Io.Condition.Wake) void {	1323	fn futexWake(userdata: ?anyopaque, ptr: const u32, max_waiters: u32) void {
1144	if (builtin.single_threaded) unreachable; // Nothing to wake up.
1145	const t: *Threaded = @ptrCast(@alignCast(userdata));	1324	const t: *Threaded = @ptrCast(@alignCast(userdata));
1146	_ = t;	1325	_ = t;
1147	comptime assert(@TypeOf(cond.state) == u64);	1326	switch (native_os) {
1148	const ints: *[2]std.atomic.Value(u32) = @ptrCast(&cond.state);	1327	.illumos, .netbsd, .openbsd => @panic("TODO"),
1149	const cond_state = &ints[0];	1328	else => Thread.futexWake(ptr, max_waiters),
1150	const cond_epoch = &ints[1];
1151	const one_waiter = 1;
1152	const waiter_mask = 0xffff;
1153	const one_signal = 1 << 16;
1154	const signal_mask = 0xffff << 16;
1155	var state = cond_state.load(.monotonic);
1156	while (true) {
1157	const waiters = (state & waiter_mask) / one_waiter;
1158	const signals = (state & signal_mask) / one_signal;
1159
1160	// Reserves which waiters to wake up by incrementing the signals count.
1161	// Therefore, the signals count is always less than or equal to the
1162	// waiters count. We don't need to Futex.wake if there's nothing to
1163	// wake up or if other wake() threads have reserved to wake up the
1164	// current waiters.
1165	const wakeable = waiters - signals;
1166	if (wakeable == 0) {
1167	return;
1168	}
1169
1170	const to_wake = switch (wake) {
1171	.one => 1,
1172	.all => wakeable,
1173	};
1174
1175	// Reserve the amount of waiters to wake by incrementing the signals
1176	// count. Release barrier ensures code before the wake() happens before
1177	// the signal it posted and consumed by the wait() threads.
1178	const new_state = state + (one_signal * to_wake);
1179	state = cond_state.cmpxchgWeak(state, new_state, .release, .monotonic) orelse {
1180	// Wake up the waiting threads we reserved above by changing the epoch value.
1181	//
1182	// A waiting thread could miss a wake up if exactly ((1<<32)-1)
1183	// wake()s happen between it observing the epoch and sleeping on
1184	// it. This is very unlikely due to how many precise amount of
1185	// Futex.wake() calls that would be between the waiting thread's
1186	// potential preemption.
1187	//
1188	// Release barrier ensures the signal being added to the state
1189	// happens before the epoch is changed. If not, the waiting thread
1190	// could potentially deadlock from missing both the state and epoch
1191	// change:
1192	//
1193	// - T2: UPDATE(&epoch, 1) (reordered before the state change)
1194	// - T1: e = LOAD(&epoch)
1195	// - T1: s = LOAD(&state)
1196	// - T2: UPDATE(&state, signal) + FUTEX_WAKE(&epoch)
1197	// - T1: s & signals == 0 -> FUTEX_WAIT(&epoch, e) (missed both epoch change and state change)
1198	_ = cond_epoch.fetchAdd(1, .release);
1199	if (native_os == .netbsd) @panic("TODO");
1200	if (native_os == .openbsd) @panic("TODO");
1201	futexWake(cond_epoch, to_wake);
1202	return;
1203	};
1204	}	1329	}
1205	}	1330	}
1206		1331
...	@@ -3630,28 +3755,28 @@ fn sleepPosix(userdata: ?*anyopaque, timeout: Io.Timeout) Io.SleepError!void {	...	@@ -3630,28 +3755,28 @@ fn sleepPosix(userdata: ?*anyopaque, timeout: Io.Timeout) Io.SleepError!void {
3630	fn select(userdata: ?anyopaque, futures: []const Io.AnyFuture) Io.Cancelable!usize {	3755	fn select(userdata: ?anyopaque, futures: []const Io.AnyFuture) Io.Cancelable!usize {
3631	const t: *Threaded = @ptrCast(@alignCast(userdata));	3756	const t: *Threaded = @ptrCast(@alignCast(userdata));
3632		3757
3633	var reset_event: ResetEvent = .unset;	3758	var event: Io.Event = .unset;
3634		3759
3635	for (futures, 0..) \|future, i\| {	3760	for (futures, 0..) \|future, i\| {
3636	const closure: *AsyncClosure = @ptrCast(@alignCast(future));	3761	const closure: *AsyncClosure = @ptrCast(@alignCast(future));
3637	if (@atomicRmw(?*ResetEvent, &closure.select_condition, .Xchg, &reset_event, .seq_cst) == AsyncClosure.done_reset_event) {	3762	if (@atomicRmw(?*Io.Event, &closure.select_condition, .Xchg, &event, .seq_cst) == AsyncClosure.done_event) {
3638	for (futures[0..i]) \|cleanup_future\| {	3763	for (futures[0..i]) \|cleanup_future\| {
3639	const cleanup_closure: *AsyncClosure = @ptrCast(@alignCast(cleanup_future));	3764	const cleanup_closure: *AsyncClosure = @ptrCast(@alignCast(cleanup_future));
3640	if (@atomicRmw(?*ResetEvent, &cleanup_closure.select_condition, .Xchg, null, .seq_cst) == AsyncClosure.done_reset_event) {	3765	if (@atomicRmw(?*Io.Event, &cleanup_closure.select_condition, .Xchg, null, .seq_cst) == AsyncClosure.done_event) {
3641	cleanup_closure.reset_event.waitUncancelable(); // Ensure no reference to our stack-allocated reset_event.	3766	cleanup_closure.event.waitUncancelable(ioBasic(t)); // Ensure no reference to our stack-allocated event.
3642	}	3767	}
3643	}	3768	}
3644	return i;	3769	return i;
3645	}	3770	}
3646	}	3771	}
3647		3772
3648	try reset_event.wait(t);	3773	try event.wait(ioBasic(t));
3649		3774
3650	var result: ?usize = null;	3775	var result: ?usize = null;
3651	for (futures, 0..) \|future, i\| {	3776	for (futures, 0..) \|future, i\| {
3652	const closure: *AsyncClosure = @ptrCast(@alignCast(future));	3777	const closure: *AsyncClosure = @ptrCast(@alignCast(future));
3653	if (@atomicRmw(?*ResetEvent, &closure.select_condition, .Xchg, null, .seq_cst) == AsyncClosure.done_reset_event) {	3778	if (@atomicRmw(?*Io.Event, &closure.select_condition, .Xchg, null, .seq_cst) == AsyncClosure.done_event) {
3654	closure.reset_event.waitUncancelable(); // Ensure no reference to our stack-allocated reset_event.	3779	closure.event.waitUncancelable(ioBasic(t)); // Ensure no reference to our stack-allocated event.
3655	if (result == null) result = i; // In case multiple are ready, return first.	3780	if (result == null) result = i; // In case multiple are ready, return first.
3656	}	3781	}
3657	}	3782	}
...	@@ -5670,11 +5795,13 @@ fn netLookup(	...	@@ -5670,11 +5795,13 @@ fn netLookup(
5670	host_name: HostName,	5795	host_name: HostName,
5671	resolved: *Io.Queue(HostName.LookupResult),	5796	resolved: *Io.Queue(HostName.LookupResult),
5672	options: HostName.LookupOptions,	5797	options: HostName.LookupOptions,
5673	) void {	5798	) net.HostName.LookupError!void {
5674	const t: *Threaded = @ptrCast(@alignCast(userdata));	5799	const t: *Threaded = @ptrCast(@alignCast(userdata));
5675	const current_thread = Thread.getCurrent(t);	5800	defer resolved.close(io(t));
5676	const t_io = io(t);	5801	netLookupFallible(t, host_name, resolved, options) catch \|err\| switch (err) {
5677	resolved.putOneUncancelable(t_io, .{ .end = netLookupFallible(t, current_thread, host_name, resolved, options) });	5802	error.Closed => unreachable, // `resolved` must not be closed until `netLookup` returns
		5803	else => \|e\| return e,
		5804	};
5678	}	5805	}
5679		5806
5680	fn netLookupUnavailable(	5807	fn netLookupUnavailable(
...	@@ -5682,22 +5809,23 @@ fn netLookupUnavailable(	...	@@ -5682,22 +5809,23 @@ fn netLookupUnavailable(
5682	host_name: HostName,	5809	host_name: HostName,
5683	resolved: *Io.Queue(HostName.LookupResult),	5810	resolved: *Io.Queue(HostName.LookupResult),
5684	options: HostName.LookupOptions,	5811	options: HostName.LookupOptions,
5685	) void {	5812	) net.HostName.LookupError!void {
5686	_ = host_name;	5813	_ = host_name;
5687	_ = options;	5814	_ = options;
5688	const t: *Threaded = @ptrCast(@alignCast(userdata));	5815	const t: *Threaded = @ptrCast(@alignCast(userdata));
5689	const t_io = ioBasic(t);	5816	resolved.close(ioBasic(t));
5690	resolved.putOneUncancelable(t_io, .{ .end = error.NetworkDown });	5817	return error.NetworkDown;
5691	}	5818	}
5692		5819
5693	fn netLookupFallible(	5820	fn netLookupFallible(
5694	t: *Threaded,	5821	t: *Threaded,
5695	current_thread: *Thread,
5696	host_name: HostName,	5822	host_name: HostName,
5697	resolved: *Io.Queue(HostName.LookupResult),	5823	resolved: *Io.Queue(HostName.LookupResult),
5698	options: HostName.LookupOptions,	5824	options: HostName.LookupOptions,
5699	) !void {	5825	) (net.HostName.LookupError \|\| Io.QueueClosedError)!void {
5700	if (!have_networking) return error.NetworkDown;	5826	if (!have_networking) return error.NetworkDown;
		5827
		5828	const current_thread: *Thread = .getCurrent(t);
5701	const t_io = io(t);	5829	const t_io = io(t);
5702	const name = host_name.bytes;	5830	const name = host_name.bytes;
5703	assert(name.len <= HostName.max_len);	5831	assert(name.len <= HostName.max_len);
...	@@ -6238,7 +6366,7 @@ fn lookupDnsSearch(	...	@@ -6238,7 +6366,7 @@ fn lookupDnsSearch(
6238	host_name: HostName,	6366	host_name: HostName,
6239	resolved: *Io.Queue(HostName.LookupResult),	6367	resolved: *Io.Queue(HostName.LookupResult),
6240	options: HostName.LookupOptions,	6368	options: HostName.LookupOptions,
6241	) HostName.LookupError!void {	6369	) (HostName.LookupError \|\| Io.QueueClosedError)!void {
6242	const t_io = io(t);	6370	const t_io = io(t);
6243	const rc = HostName.ResolvConf.init(t_io) catch return error.ResolvConfParseFailed;	6371	const rc = HostName.ResolvConf.init(t_io) catch return error.ResolvConfParseFailed;
6244		6372
...	@@ -6282,7 +6410,7 @@ fn lookupDns(	...	@@ -6282,7 +6410,7 @@ fn lookupDns(
6282	rc: *const HostName.ResolvConf,	6410	rc: *const HostName.ResolvConf,
6283	resolved: *Io.Queue(HostName.LookupResult),	6411	resolved: *Io.Queue(HostName.LookupResult),
6284	options: HostName.LookupOptions,	6412	options: HostName.LookupOptions,
6285	) HostName.LookupError!void {	6413	) (HostName.LookupError \|\| Io.QueueClosedError)!void {
6286	const t_io = io(t);	6414	const t_io = io(t);
6287	const family_records: [2]struct { af: IpAddress.Family, rr: HostName.DnsRecord } = .{	6415	const family_records: [2]struct { af: IpAddress.Family, rr: HostName.DnsRecord } = .{
6288	.{ .af = .ip6, .rr = .A },	6416	.{ .af = .ip6, .rr = .A },
...	@@ -6496,8 +6624,10 @@ fn lookupHosts(	...	@@ -6496,8 +6624,10 @@ fn lookupHosts(
6496	return error.DetectingNetworkConfigurationFailed;	6624	return error.DetectingNetworkConfigurationFailed;
6497	},	6625	},
6498	},	6626	},
6499	error.Canceled => \|e\| return e,	6627	error.Canceled,
6500	error.UnknownHostName => \|e\| return e,	6628	error.Closed,
		6629	error.UnknownHostName,
		6630	=> \|e\| return e,
6501	};	6631	};
6502	}	6632	}
6503		6633
...	@@ -6507,7 +6637,7 @@ fn lookupHostsReader(	...	@@ -6507,7 +6637,7 @@ fn lookupHostsReader(
6507	resolved: *Io.Queue(HostName.LookupResult),	6637	resolved: *Io.Queue(HostName.LookupResult),
6508	options: HostName.LookupOptions,	6638	options: HostName.LookupOptions,
6509	reader: *Io.Reader,	6639	reader: *Io.Reader,
6510	) error{ ReadFailed, Canceled, UnknownHostName }!void {	6640	) error{ ReadFailed, Canceled, UnknownHostName, Closed }!void {
6511	const t_io = io(t);	6641	const t_io = io(t);
6512	var addresses_len: usize = 0;	6642	var addresses_len: usize = 0;
6513	var canonical_name: ?HostName = null;	6643	var canonical_name: ?HostName = null;
...	@@ -6612,460 +6742,6 @@ fn copyCanon(canonical_name_buffer: *[HostName.max_len]u8, name: []const u8) Hos	...	@@ -6612,460 +6742,6 @@ fn copyCanon(canonical_name_buffer: *[HostName.max_len]u8, name: []const u8) Hos
6612	/// ulock_wait2() uses 64-bit nano-second timeouts (with the same convention)	6742	/// ulock_wait2() uses 64-bit nano-second timeouts (with the same convention)
6613	const darwin_supports_ulock_wait2 = builtin.os.version_range.semver.min.major >= 11;	6743	const darwin_supports_ulock_wait2 = builtin.os.version_range.semver.min.major >= 11;
6614		6744
6615	fn futexWait(current_thread: Thread, ptr: const std.atomic.Value(u32), expect: u32) Io.Cancelable!void {
6616	@branchHint(.cold);
6617
6618	if (builtin.cpu.arch.isWasm()) {
6619	comptime assert(builtin.cpu.has(.wasm, .atomics));
6620	try current_thread.checkCancel();
6621	const timeout: i64 = -1;
6622	const signed_expect: i32 = @bitCast(expect);
6623	const result = asm volatile (
6624	\\local.get %[ptr]
6625	\\local.get %[expected]
6626	\\local.get %[timeout]
6627	\\memory.atomic.wait32 0
6628	\\local.set %[ret]
6629	: [ret] "=r" (-> u32),
6630	: [ptr] "r" (&ptr.raw),
6631	[expected] "r" (signed_expect),
6632	[timeout] "r" (timeout),
6633	);
6634	switch (result) {
6635	0 => {}, // ok
6636	1 => {}, // expected != loaded
6637	2 => assert(!is_debug), // timeout
6638	else => assert(!is_debug),
6639	}
6640	} else switch (native_os) {
6641	.linux => {
6642	const linux = std.os.linux;
6643	try current_thread.beginSyscall();
6644	const rc = linux.futex_4arg(ptr, .{ .cmd = .WAIT, .private = true }, expect, null);
6645	current_thread.endSyscall();
6646	switch (linux.errno(rc)) {
6647	.SUCCESS => {}, // notified by `wake()`
6648	.INTR => {}, // caller's responsibility to retry
6649	.AGAIN => {}, // ptr.* != expect
6650	.INVAL => {}, // possibly timeout overflow
6651	.TIMEDOUT => recoverableOsBugDetected(),
6652	.FAULT => recoverableOsBugDetected(), // ptr was invalid
6653	else => recoverableOsBugDetected(),
6654	}
6655	},
6656	.driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => {
6657	const c = std.c;
6658	const flags: c.UL = .{
6659	.op = .COMPARE_AND_WAIT,
6660	.NO_ERRNO = true,
6661	};
6662	try current_thread.beginSyscall();
6663	const status = if (darwin_supports_ulock_wait2)
6664	c.__ulock_wait2(flags, ptr, expect, 0, 0)
6665	else
6666	c.__ulock_wait(flags, ptr, expect, 0);
6667	current_thread.endSyscall();
6668
6669	if (status >= 0) return;
6670
6671	if (is_debug) switch (@as(c.E, @enumFromInt(-status))) {
6672	.INTR => {}, // spurious wake
6673	// Address of the futex was paged out. This is unlikely, but possible in theory, and
6674	// pthread/libdispatch on darwin bother to handle it. In this case we'll return
6675	// without waiting, but the caller should retry anyway.
6676	.FAULT => {},
6677	.TIMEDOUT => unreachable,
6678	else => unreachable,
6679	};
6680	},
6681	.windows => {
6682	try current_thread.checkCancel();
6683	switch (windows.ntdll.RtlWaitOnAddress(ptr, &expect, @sizeOf(@TypeOf(expect)), null)) {
6684	.SUCCESS => {},
6685	.CANCELLED => return error.Canceled,
6686	else => recoverableOsBugDetected(),
6687	}
6688	},
6689	.freebsd => {
6690	const flags = @intFromEnum(std.c.UMTX_OP.WAIT_UINT_PRIVATE);
6691	try current_thread.beginSyscall();
6692	const rc = std.c._umtx_op(@intFromPtr(&ptr.raw), flags, @as(c_ulong, expect), 0, 0);
6693	current_thread.endSyscall();
6694	if (is_debug) switch (posix.errno(rc)) {
6695	.SUCCESS => {},
6696	.FAULT => unreachable, // one of the args points to invalid memory
6697	.INVAL => unreachable, // arguments should be correct
6698	.TIMEDOUT => unreachable, // no timeout provided
6699	.INTR => {}, // spurious wake
6700	else => unreachable,
6701	};
6702	},
6703	else => @compileError("unimplemented: futexWait"),
6704	}
6705	}
6706
6707	pub fn futexWaitUncancelable(ptr: *const std.atomic.Value(u32), expect: u32) void {
6708	@branchHint(.cold);
6709
6710	if (builtin.cpu.arch.isWasm()) {
6711	comptime assert(builtin.cpu.has(.wasm, .atomics));
6712	const timeout: i64 = -1;
6713	const signed_expect: i32 = @bitCast(expect);
6714	const result = asm volatile (
6715	\\local.get %[ptr]
6716	\\local.get %[expected]
6717	\\local.get %[timeout]
6718	\\memory.atomic.wait32 0
6719	\\local.set %[ret]
6720	: [ret] "=r" (-> u32),
6721	: [ptr] "r" (&ptr.raw),
6722	[expected] "r" (signed_expect),
6723	[timeout] "r" (timeout),
6724	);
6725	switch (result) {
6726	0 => {}, // ok
6727	1 => {}, // expected != loaded
6728	2 => recoverableOsBugDetected(), // timeout
6729	else => recoverableOsBugDetected(),
6730	}
6731	} else switch (native_os) {
6732	.linux => {
6733	const linux = std.os.linux;
6734	const rc = linux.futex_4arg(ptr, .{ .cmd = .WAIT, .private = true }, expect, null);
6735	switch (linux.errno(rc)) {
6736	.SUCCESS => {}, // notified by `wake()`
6737	.INTR => {}, // caller's responsibility to repeat
6738	.AGAIN => {}, // ptr.* != expect
6739	.INVAL => {}, // possibly timeout overflow
6740	.TIMEDOUT => recoverableOsBugDetected(),
6741	.FAULT => recoverableOsBugDetected(), // ptr was invalid
6742	else => recoverableOsBugDetected(),
6743	}
6744	},
6745	.driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => {
6746	const c = std.c;
6747	const flags: c.UL = .{
6748	.op = .COMPARE_AND_WAIT,
6749	.NO_ERRNO = true,
6750	};
6751	const status = if (darwin_supports_ulock_wait2)
6752	c.__ulock_wait2(flags, ptr, expect, 0, 0)
6753	else
6754	c.__ulock_wait(flags, ptr, expect, 0);
6755
6756	if (status >= 0) return;
6757
6758	switch (@as(c.E, @enumFromInt(-status))) {
6759	// Wait was interrupted by the OS or other spurious signalling.
6760	.INTR => {},
6761	// Address of the futex was paged out. This is unlikely, but possible in theory, and
6762	// pthread/libdispatch on darwin bother to handle it. In this case we'll return
6763	// without waiting, but the caller should retry anyway.
6764	.FAULT => {},
6765	.TIMEDOUT => recoverableOsBugDetected(),
6766	else => recoverableOsBugDetected(),
6767	}
6768	},
6769	.windows => {
6770	switch (windows.ntdll.RtlWaitOnAddress(ptr, &expect, @sizeOf(@TypeOf(expect)), null)) {
6771	.SUCCESS, .CANCELLED => {},
6772	else => recoverableOsBugDetected(),
6773	}
6774	},
6775	.freebsd => {
6776	const flags = @intFromEnum(std.c.UMTX_OP.WAIT_UINT_PRIVATE);
6777	const rc = std.c._umtx_op(@intFromPtr(&ptr.raw), flags, @as(c_ulong, expect), 0, 0);
6778	switch (posix.errno(rc)) {
6779	.SUCCESS => {},
6780	.INTR => {}, // spurious wake
6781	.FAULT => recoverableOsBugDetected(), // one of the args points to invalid memory
6782	.INVAL => recoverableOsBugDetected(), // arguments should be correct
6783	.TIMEDOUT => recoverableOsBugDetected(), // no timeout provided
6784	else => recoverableOsBugDetected(),
6785	}
6786	},
6787	else => @compileError("unimplemented: futexWaitUncancelable"),
6788	}
6789	}
6790
6791	pub fn futexWake(ptr: *const std.atomic.Value(u32), max_waiters: u32) void {
6792	@branchHint(.cold);
6793
6794	if (builtin.cpu.arch.isWasm()) {
6795	comptime assert(builtin.cpu.has(.wasm, .atomics));
6796	assert(max_waiters != 0);
6797	const woken_count = asm volatile (
6798	\\local.get %[ptr]
6799	\\local.get %[waiters]
6800	\\memory.atomic.notify 0
6801	\\local.set %[ret]
6802	: [ret] "=r" (-> u32),
6803	: [ptr] "r" (&ptr.raw),
6804	[waiters] "r" (max_waiters),
6805	);
6806	_ = woken_count; // can be 0 when linker flag 'shared-memory' is not enabled
6807	} else switch (native_os) {
6808	.linux => {
6809	const linux = std.os.linux;
6810	switch (linux.errno(linux.futex_3arg(
6811	&ptr.raw,
6812	.{ .cmd = .WAKE, .private = true },
6813	@min(max_waiters, std.math.maxInt(i32)),
6814	))) {
6815	.SUCCESS => return, // successful wake up
6816	.INVAL => return, // invalid futex_wait() on ptr done elsewhere
6817	.FAULT => return, // pointer became invalid while doing the wake
6818	else => return recoverableOsBugDetected(), // deadlock due to operating system bug
6819	}
6820	},
6821	.driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => {
6822	const c = std.c;
6823	const flags: c.UL = .{
6824	.op = .COMPARE_AND_WAIT,
6825	.NO_ERRNO = true,
6826	.WAKE_ALL = max_waiters > 1,
6827	};
6828	while (true) {
6829	const status = c.__ulock_wake(flags, ptr, 0);
6830	if (status >= 0) return;
6831	switch (@as(c.E, @enumFromInt(-status))) {
6832	.INTR, .CANCELED => continue, // spurious wake()
6833	.FAULT => unreachable, // __ulock_wake doesn't generate EFAULT according to darwin pthread_cond_t
6834	.NOENT => return, // nothing was woken up
6835	.ALREADY => unreachable, // only for UL.Op.WAKE_THREAD
6836	else => unreachable, // deadlock due to operating system bug
6837	}
6838	}
6839	},
6840	.windows => {
6841	assert(max_waiters != 0);
6842	switch (max_waiters) {
6843	1 => windows.ntdll.RtlWakeAddressSingle(ptr),
6844	else => windows.ntdll.RtlWakeAddressAll(ptr),
6845	}
6846	},
6847	.freebsd => {
6848	const rc = std.c._umtx_op(
6849	@intFromPtr(&ptr.raw),
6850	@intFromEnum(std.c.UMTX_OP.WAKE_PRIVATE),
6851	@as(c_ulong, max_waiters),
6852	0, // there is no timeout struct
6853	0, // there is no timeout struct pointer
6854	);
6855	switch (posix.errno(rc)) {
6856	.SUCCESS => {},
6857	.FAULT => {}, // it's ok if the ptr doesn't point to valid memory
6858	.INVAL => unreachable, // arguments should be correct
6859	else => unreachable, // deadlock due to operating system bug
6860	}
6861	},
6862	else => @compileError("unimplemented: futexWake"),
6863	}
6864	}
6865
6866	/// A thread-safe logical boolean value which can be `set` and `unset`.
6867	///
6868	/// It can also block threads until the value is set with cancelation via timed
6869	/// waits. Statically initializable; four bytes on all targets.
6870	pub const ResetEvent = switch (native_os) {
6871	.illumos, .netbsd, .openbsd => ResetEventPosix,
6872	else => ResetEventFutex,
6873	};
6874
6875	/// A `ResetEvent` implementation based on futexes.
6876	const ResetEventFutex = enum(u32) {
6877	unset = 0,
6878	waiting = 1,
6879	is_set = 2,
6880
6881	/// Returns whether the logical boolean is `set`.
6882	///
6883	/// Once `reset` is called, this returns false until the next `set`.
6884	///
6885	/// The memory accesses before the `set` can be said to happen before
6886	/// `isSet` returns true.
6887	pub fn isSet(ref: *const ResetEventFutex) bool {
6888	if (builtin.single_threaded) return switch (ref.*) {
6889	.unset => false,
6890	.waiting => unreachable,
6891	.is_set => true,
6892	};
6893	// Acquire barrier ensures memory accesses before `set` happen before
6894	// returning true.
6895	return @atomicLoad(ResetEventFutex, ref, .acquire) == .is_set;
6896	}
6897
6898	/// Blocks the calling thread until `set` is called.
6899	///
6900	/// This is effectively a more efficient version of `while (!isSet()) {}`.
6901	///
6902	/// The memory accesses before the `set` can be said to happen before `wait` returns.
6903	pub fn wait(ref: ResetEventFutex, t: Threaded) Io.Cancelable!void {
6904	if (builtin.single_threaded) switch (ref.*) {
6905	.unset => unreachable, // Deadlock, no other threads to wake us up.
6906	.waiting => unreachable, // Invalid state.
6907	.is_set => return,
6908	};
6909	// Try to set the state from `unset` to `waiting` to indicate to the
6910	// `set` thread that others are blocked on the ResetEventFutex. Avoid using
6911	// any strict barriers until we know the ResetEventFutex is set.
6912	var state = @atomicLoad(ResetEventFutex, ref, .acquire);
6913	if (state == .is_set) {
6914	@branchHint(.likely);
6915	return;
6916	}
6917	if (state == .unset) {
6918	state = @cmpxchgStrong(ResetEventFutex, ref, state, .waiting, .acquire, .acquire) orelse .waiting;
6919	}
6920	const current_thread = Thread.getCurrent(t);
6921	while (state == .waiting) {
6922	try futexWait(current_thread, @ptrCast(ref), @intFromEnum(ResetEventFutex.waiting));
6923	state = @atomicLoad(ResetEventFutex, ref, .acquire);
6924	}
6925	assert(state == .is_set);
6926	}
6927
6928	/// Same as `wait` except uninterruptible.
6929	pub fn waitUncancelable(ref: *ResetEventFutex) void {
6930	if (builtin.single_threaded) switch (ref.*) {
6931	.unset => unreachable, // Deadlock, no other threads to wake us up.
6932	.waiting => unreachable, // Invalid state.
6933	.is_set => return,
6934	};
6935	// Try to set the state from `unset` to `waiting` to indicate to the
6936	// `set` thread that others are blocked on the ResetEventFutex. Avoid using
6937	// any strict barriers until we know the ResetEventFutex is set.
6938	var state = @atomicLoad(ResetEventFutex, ref, .acquire);
6939	if (state == .is_set) {
6940	@branchHint(.likely);
6941	return;
6942	}
6943	if (state == .unset) {
6944	state = @cmpxchgStrong(ResetEventFutex, ref, state, .waiting, .acquire, .acquire) orelse .waiting;
6945	}
6946	while (state == .waiting) {
6947	futexWaitUncancelable(@ptrCast(ref), @intFromEnum(ResetEventFutex.waiting));
6948	state = @atomicLoad(ResetEventFutex, ref, .acquire);
6949	}
6950	assert(state == .is_set);
6951	}
6952
6953	/// Marks the logical boolean as `set` and unblocks any threads in `wait`
6954	/// or `timedWait` to observe the new state.
6955	///
6956	/// The logical boolean stays `set` until `reset` is called, making future
6957	/// `set` calls do nothing semantically.
6958	///
6959	/// The memory accesses before `set` can be said to happen before `isSet`
6960	/// returns true or `wait`/`timedWait` return successfully.
6961	pub fn set(ref: *ResetEventFutex) void {
6962	if (builtin.single_threaded) {
6963	ref.* = .is_set;
6964	return;
6965	}
6966	if (@atomicRmw(ResetEventFutex, ref, .Xchg, .is_set, .release) == .waiting) {
6967	futexWake(@ptrCast(ref), std.math.maxInt(u32));
6968	}
6969	}
6970
6971	/// Unmarks the ResetEventFutex as if `set` was never called.
6972	///
6973	/// Assumes no threads are blocked in `wait` or `timedWait`. Concurrent
6974	/// calls to `set`, `isSet` and `reset` are allowed.
6975	pub fn reset(ref: *ResetEventFutex) void {
6976	if (builtin.single_threaded) {
6977	ref.* = .unset;
6978	return;
6979	}
6980	@atomicStore(ResetEventFutex, ref, .unset, .monotonic);
6981	}
6982	};
6983
6984	/// A `ResetEvent` implementation based on pthreads API.
6985	const ResetEventPosix = struct {
6986	cond: std.c.pthread_cond_t,
6987	mutex: std.c.pthread_mutex_t,
6988	state: ResetEventFutex,
6989
6990	pub const unset: ResetEventPosix = .{
6991	.cond = std.c.PTHREAD_COND_INITIALIZER,
6992	.mutex = std.c.PTHREAD_MUTEX_INITIALIZER,
6993	.state = .unset,
6994	};
6995
6996	pub fn isSet(rep: *const ResetEventPosix) bool {
6997	if (builtin.single_threaded) return switch (rep.state) {
6998	.unset => false,
6999	.waiting => unreachable,
7000	.is_set => true,
7001	};
7002	return @atomicLoad(ResetEventFutex, &rep.state, .acquire) == .is_set;
7003	}
7004
7005	pub fn wait(rep: ResetEventPosix, t: Threaded) Io.Cancelable!void {
7006	if (builtin.single_threaded) switch (rep.*) {
7007	.unset => unreachable, // Deadlock, no other threads to wake us up.
7008	.waiting => unreachable, // Invalid state.
7009	.is_set => return,
7010	};
7011	const current_thread = Thread.getCurrent(t);
7012	assert(std.c.pthread_mutex_lock(&rep.mutex) == .SUCCESS);
7013	defer assert(std.c.pthread_mutex_unlock(&rep.mutex) == .SUCCESS);
7014	sw: switch (rep.state) {
7015	.unset => {
7016	rep.state = .waiting;
7017	continue :sw .waiting;
7018	},
7019	.waiting => {
7020	try current_thread.beginSyscall();
7021	assert(std.c.pthread_cond_wait(&rep.cond, &rep.mutex) == .SUCCESS);
7022	current_thread.endSyscall();
7023	continue :sw rep.state;
7024	},
7025	.is_set => return,
7026	}
7027	}
7028
7029	pub fn waitUncancelable(rep: *ResetEventPosix) void {
7030	if (builtin.single_threaded) switch (rep.*) {
7031	.unset => unreachable, // Deadlock, no other threads to wake us up.
7032	.waiting => unreachable, // Invalid state.
7033	.is_set => return,
7034	};
7035	assert(std.c.pthread_mutex_lock(&rep.mutex) == .SUCCESS);
7036	defer assert(std.c.pthread_mutex_unlock(&rep.mutex) == .SUCCESS);
7037	sw: switch (rep.state) {
7038	.unset => {
7039	rep.state = .waiting;
7040	continue :sw .waiting;
7041	},
7042	.waiting => {
7043	assert(std.c.pthread_cond_wait(&rep.cond, &rep.mutex) == .SUCCESS);
7044	continue :sw rep.state;
7045	},
7046	.is_set => return,
7047	}
7048	}
7049
7050	pub fn set(rep: *ResetEventPosix) void {
7051	if (builtin.single_threaded) {
7052	rep.* = .is_set;
7053	return;
7054	}
7055	if (@atomicRmw(ResetEventFutex, &rep.state, .Xchg, .is_set, .release) == .waiting) {
7056	assert(std.c.pthread_cond_broadcast(&rep.cond) == .SUCCESS);
7057	}
7058	}
7059
7060	pub fn reset(rep: *ResetEventPosix) void {
7061	if (builtin.single_threaded) {
7062	rep.* = .unset;
7063	return;
7064	}
7065	@atomicStore(ResetEventFutex, &rep.state, .unset, .monotonic);
7066	}
7067	};
7068
7069	fn closeSocketWindows(s: ws2_32.SOCKET) void {	6745	fn closeSocketWindows(s: ws2_32.SOCKET) void {
7070	const rc = ws2_32.closesocket(s);	6746	const rc = ws2_32.closesocket(s);
7071	if (is_debug) switch (rc) {	6747	if (is_debug) switch (rc) {

lib/std/Io/net/HostName.zig+63-44

 @@ -82,19 +82,22 @@ pub const LookupError = error{
 pub const LookupResult = union(enum) {
     address: IpAddress,
     canonical_name: HostName,
-    end: LookupError!void,
 };
-/// Adds any number of `IpAddress` into resolved, exactly one canonical_name,
+/// Adds any number of `LookupResult.address` into `resolved`, and exactly one
-/// and then always finishes by adding one `LookupResult.end` entry.
+/// `LookupResult.canonical_name`.
 ///
 /// Guaranteed not to block if provided queue has capacity at least 16.
+///
+/// Closes `resolved` before return, even on error.
+///
+/// Asserts `resolved` is not closed until this call returns.
 pub fn lookup(
     host_name: HostName,
     io: Io,
     resolved: *Io.Queue(LookupResult),
     options: LookupOptions,
-) void {
+) LookupError!void {
     return io.vtable.netLookup(io.userdata, host_name, resolved, options);
 }
 @@ -211,23 +214,25 @@ pub fn connect(
     port: u16,
     options: IpAddress.ConnectOptions,
 ) ConnectError!Stream {
-    var connect_many_buffer: [32]ConnectManyResult = undefined;
+    var connect_many_buffer: [32]IpAddress.ConnectError!Stream = undefined;
-    var connect_many_queue: Io.Queue(ConnectManyResult) = .init(&connect_many_buffer);
+    var connect_many_queue: Io.Queue(IpAddress.ConnectError!Stream) = .init(&connect_many_buffer);
     var connect_many = io.async(connectMany, .{ host_name, io, port, &connect_many_queue, options });
-    var saw_end = false;
     defer {
-        connect_many.cancel(io);
+        connect_many.cancel(io) catch {};
-        if (!saw_end) while (true) switch (connect_many_queue.getOneUncancelable(io)) {
+        while (connect_many_queue.getOneUncancelable(io)) |loser| {
-            .connection => |loser| if (loser) |s| s.close(io) else |_| continue,
+            if (loser) |s| s.close(io) else |_| {}
-            .end => break,
+        } else |err| switch (err) {
-        };
+            error.Closed => {},
+        }
     }
-    var aggregate_error: ConnectError = error.UnknownHostName;
+    var ip_connect_error: ?IpAddress.ConnectError = null;
-    while (connect_many_queue.getOne(io)) |result| switch (result) {
+    while (connect_many_queue.getOne(io)) |result| {
-        .connection => |connection| if (connection) |stream| return stream else |err| switch (err) {
+        if (result) |stream| {
+            return stream;
+        } else |err| switch (err) {
             error.SystemResources,
             error.OptionUnsupported,
             error.ProcessFdQuotaExceeded,
 @@ -237,66 +242,80 @@ pub fn connect(
             error.WouldBlock => return error.Unexpected,
-            else => |e| aggregate_error = e,
+            else => |e| ip_connect_error = e,
-        },
+        }
-        .end => |end| {
-            saw_end = true;
-            try end;
-            return aggregate_error;
-        },
     } else |err| switch (err) {
         error.Canceled => |e| return e,
+        error.Closed => {
+            // There was no successful connection attempt. If there was a lookup error, return that.
+            try connect_many.await(io);
+            // Otherwise, return the error from a failed IP connection attempt.
+            return ip_connect_error orelse
+                return error.UnknownHostName;
+        },
     }
 }
-pub const ConnectManyResult = union(enum) {
-    connection: IpAddress.ConnectError!Stream,
-    end: ConnectError!void,
-};
 /// Asynchronously establishes a connection to all IP addresses associated with
 /// a host name, adding them to a results queue upon completion.
+///
+/// Closes `results` before return, even on error.
+///
+/// Asserts `results` is not closed until this call returns.
 pub fn connectMany(
     host_name: HostName,
     io: Io,
     port: u16,
-    results: *Io.Queue(ConnectManyResult),
+    results: *Io.Queue(IpAddress.ConnectError!Stream),
     options: IpAddress.ConnectOptions,
-) void {
+) LookupError!void {
+    defer results.close(io);
     var canonical_name_buffer: [max_len]u8 = undefined;
     var lookup_buffer: [32]HostName.LookupResult = undefined;
     var lookup_queue: Io.Queue(LookupResult) = .init(&lookup_buffer);
-    var group: Io.Group = .init;
+    var lookup_future = io.async(lookup, .{ host_name, io, &lookup_queue, .{
-    defer group.cancel(io);
-    group.async(io, lookup, .{ host_name, io, &lookup_queue, .{
         .port = port,
         .canonical_name_buffer = &canonical_name_buffer,
     } });
+    defer lookup_future.cancel(io) catch {};
+    var group: Io.Group = .init;
+    defer group.cancel(io);
     while (lookup_queue.getOne(io)) |dns_result| switch (dns_result) {
         .address => |address| group.async(io, enqueueConnection, .{ address, io, results, options }),
         .canonical_name => continue,
-        .end => |lookup_result| {
-            group.wait(io);
-            results.putOneUncancelable(io, .{ .end = lookup_result });
-            return;
-        },
     } else |err| switch (err) {
-        error.Canceled => |e| {
+        error.Canceled => |e| return e,
-            group.cancel(io);
+        error.Closed => {
-            results.putOneUncancelable(io, .{ .end = e });
+            group.wait(io);
+            return lookup_future.await(io);
         },
     }
 }
 fn enqueueConnection(
     address: IpAddress,
     io: Io,
-    queue: *Io.Queue(ConnectManyResult),
+    queue: *Io.Queue(IpAddress.ConnectError!Stream),
     options: IpAddress.ConnectOptions,
 ) void {
-    queue.putOneUncancelable(io, .{ .connection = address.connect(io, options) });
+    enqueueConnectionFallible(address, io, queue, options) catch |err| switch (err) {
+        error.Canceled => {},
+    };
+}
+fn enqueueConnectionFallible(
+    address: IpAddress,
+    io: Io,
+    queue: *Io.Queue(IpAddress.ConnectError!Stream),
+    options: IpAddress.ConnectOptions,
+) Io.Cancelable!void {
+    const result = address.connect(io, options);
+    errdefer if (result) |s| s.close(io) else |_| {};
+    queue.putOne(io, result) catch |err| switch (err) {
+        error.Closed => unreachable, // `queue` must not be closed
+        error.Canceled => |e| return e,
+    };
 }
 pub const ResolvConf = struct {

lib/std/Io/net/test.zig+14-16

 @@ -129,7 +129,7 @@ test "resolve DNS" {
         var results_buffer: [32]net.HostName.LookupResult = undefined;
         var results: Io.Queue(net.HostName.LookupResult) = .init(&results_buffer);
-        net.HostName.lookup(try .init("localhost"), io, &results, .{
+        try net.HostName.lookup(try .init("localhost"), io, &results, .{
             .port = 80,
             .canonical_name_buffer = &canonical_name_buffer,
         });
 @@ -142,11 +142,10 @@ test "resolve DNS" {
                     addresses_found += 1;
             },
             .canonical_name => |canonical_name| try testing.expectEqualStrings("localhost", canonical_name.bytes),
-            .end => |end| {
+        } else |err| switch (err) {
-                try end;
+            error.Closed => {},
-                break;
+            error.Canceled => |e| return e,
-            },
+        }
-        } else |err| return err;
         try testing.expect(addresses_found != 0);
     }
 @@ -161,20 +160,19 @@ test "resolve DNS" {
         net.HostName.lookup(try .init("example.com"), io, &results, .{
             .port = 80,
             .canonical_name_buffer = &canonical_name_buffer,
-        });
+        }) catch |err| switch (err) {
+            error.UnknownHostName => return error.SkipZigTest,
+            error.NameServerFailure => return error.SkipZigTest,
+            else => |e| return e,
+        };
         while (results.getOne(io)) |result| switch (result) {
             .address => {},
             .canonical_name => {},
-            .end => |end| {
+        } else |err| switch (err) {
-                end catch |err| switch (err) {
+            error.Closed => {},
-                    error.UnknownHostName => return error.SkipZigTest,
+            error.Canceled => |e| return e,
-                    error.NameServerFailure => return error.SkipZigTest,
+        }
-                    else => return err,
-                };
-                break;
-            },
-        } else |err| return err;
     }
 }

lib/std/Io/test.zig+164-8

...	@@ -209,10 +209,10 @@ test "select" {	...	@@ -209,10 +209,10 @@ test "select" {
209	return;	209	return;
210	},	210	},
211	};	211	};
212	defer if (get_a.cancel(io)) \|_\| {} else \|_\| @panic("fail");	212	defer _ = get_a.cancel(io) catch {};
213		213
214	var get_b = try io.concurrent(Io.Queue(u8).getOne, .{ &queue, io });	214	var get_b = try io.concurrent(Io.Queue(u8).getOne, .{ &queue, io });
215	defer if (get_b.cancel(io)) \|_\| {} else \|_\| @panic("fail");	215	defer _ = get_b.cancel(io) catch {};
216		216
217	var timeout = io.async(Io.sleep, .{ io, .fromMilliseconds(1), .awake });	217	var timeout = io.async(Io.sleep, .{ io, .fromMilliseconds(1), .awake });
218	defer timeout.cancel(io) catch {};	218	defer timeout.cancel(io) catch {};
...	@@ -225,12 +225,9 @@ test "select" {	...	@@ -225,12 +225,9 @@ test "select" {
225	.get_a => return error.TestFailure,	225	.get_a => return error.TestFailure,
226	.get_b => return error.TestFailure,	226	.get_b => return error.TestFailure,
227	.timeout => {	227	.timeout => {
228	// Unblock the queues to avoid making this unit test depend on	228	queue.close(io);
229	// cancellation.	229	try testing.expectError(error.Closed, get_a.await(io));
230	queue.putOneUncancelable(io, 1);	230	try testing.expectError(error.Closed, get_b.await(io));
231	queue.putOneUncancelable(io, 1);
232	try testing.expectEqual(1, try get_a.await(io));
233	try testing.expectEqual(1, try get_b.await(io));
234	},	231	},
235	}	232	}
236	}	233	}
...	@@ -255,3 +252,162 @@ test "Queue" {	...	@@ -255,3 +252,162 @@ test "Queue" {
255	try testQueue(4);	252	try testQueue(4);
256	try testQueue(5);	253	try testQueue(5);
257	}	254	}
		255
		256	test "Queue.close single-threaded" {
		257	const io = std.testing.io;
		258
		259	var buf: [10]u8 = undefined;
		260	var queue: Io.Queue(u8) = .init(&buf);
		261
		262	try queue.putAll(io, &.{ 0, 1, 2, 3, 4, 5, 6 });
		263	try expectEqual(3, try queue.put(io, &.{ 7, 8, 9, 10 }, 0)); // there is capacity for 3 more items
		264
		265	var get_buf: [4]u8 = undefined;
		266
		267	// Receive some elements before closing
		268	try expectEqual(4, try queue.get(io, &get_buf, 0));
		269	try expectEqual(0, get_buf[0]);
		270	try expectEqual(1, get_buf[1]);
		271	try expectEqual(2, get_buf[2]);
		272	try expectEqual(3, get_buf[3]);
		273	try expectEqual(4, try queue.getOne(io));
		274
		275	// ...and add a couple more now there's space
		276	try queue.putAll(io, &.{ 20, 21 });
		277
		278	queue.close(io);
		279
		280	// Receive more elements after closing
		281	try expectEqual(4, try queue.get(io, &get_buf, 0));
		282	try expectEqual(5, get_buf[0]);
		283	try expectEqual(6, get_buf[1]);
		284	try expectEqual(7, get_buf[2]);
		285	try expectEqual(8, get_buf[3]);
		286	try expectEqual(9, try queue.getOne(io));
		287
		288	// Cannot put anything while closed, even if the buffer has space
		289	try expectError(error.Closed, queue.putOne(io, 100));
		290	try expectError(error.Closed, queue.putAll(io, &.{ 101, 102 }));
		291	try expectError(error.Closed, queue.putUncancelable(io, &.{ 103, 104 }, 0));
		292
		293	// Even if we ask for 3 items, the queue is closed, so we only get the last 2
		294	try expectEqual(2, try queue.get(io, &get_buf, 4));
		295	try expectEqual(20, get_buf[0]);
		296	try expectEqual(21, get_buf[1]);
		297
		298	// The queue is now empty, so `get` should return `error.Closed` too
		299	try expectError(error.Closed, queue.getOne(io));
		300	try expectError(error.Closed, queue.get(io, &get_buf, 0));
		301	try expectError(error.Closed, queue.putUncancelable(io, &get_buf, 2));
		302	}
		303
		304	test "Event" {
		305	const global = struct {
		306	fn waitAndRead(io: Io, event: Io.Event, ptr: const u32) Io.Cancelable!u32 {
		307	try event.wait(io);
		308	return ptr.*;
		309	}
		310	};
		311
		312	const io = std.testing.io;
		313
		314	var event: Io.Event = .unset;
		315	var buffer: u32 = undefined;
		316
		317	{
		318	var future = io.concurrent(global.waitAndRead, .{ io, &event, &buffer }) catch \|err\| switch (err) {
		319	error.ConcurrencyUnavailable => return error.SkipZigTest,
		320	};
		321
		322	buffer = 123;
		323	event.set(io);
		324
		325	const result = try future.await(io);
		326
		327	try std.testing.expectEqual(123, result);
		328	}
		329
		330	event.reset();
		331
		332	{
		333	var future = io.concurrent(global.waitAndRead, .{ io, &event, &buffer }) catch \|err\| switch (err) {
		334	error.ConcurrencyUnavailable => return error.SkipZigTest,
		335	};
		336	try std.testing.expectError(error.Canceled, future.cancel(io));
		337	}
		338	}
		339
		340	test "recancel" {
		341	const global = struct {
		342	fn worker(io: Io) Io.Cancelable!void {
		343	var dummy_event: Io.Event = .unset;
		344
		345	if (dummy_event.wait(io)) {
		346	return;
		347	} else \|err\| switch (err) {
		348	error.Canceled => io.recancel(),
		349	}
		350
		351	// Now we expect to see `error.Canceled` again.
		352	return dummy_event.wait(io);
		353	}
		354	};
		355
		356	const io = std.testing.io;
		357	var future = io.concurrent(global.worker, .{io}) catch \|err\| switch (err) {
		358	error.ConcurrencyUnavailable => return error.SkipZigTest,
		359	};
		360	if (future.cancel(io)) {
		361	return error.UnexpectedSuccess; // both `wait` calls should have returned `error.Canceled`
		362	} else \|err\| switch (err) {
		363	error.Canceled => {},
		364	}
		365	}
		366
		367	test "swapCancelProtection" {
		368	const global = struct {
		369	fn waitTwice(
		370	io: Io,
		371	event: *Io.Event,
		372	) error{ Canceled, CanceledWhileProtected }!void {
		373	// Wait for `event` while protected from cancelation.
		374	{
		375	const old_prot = io.swapCancelProtection(.blocked);
		376	defer _ = io.swapCancelProtection(old_prot);
		377	event.wait(io) catch \|err\| switch (err) {
		378	error.Canceled => return error.CanceledWhileProtected,
		379	};
		380	}
		381	// Reset the event (it will never be set again), and this time wait for it without protection.
		382	event.reset();
		383	_ = try event.wait(io);
		384	}
		385	fn sleepThenSet(io: Io, event: *Io.Event) !void {
		386	// Give `waitTwice` a chance to get canceled.
		387	try io.sleep(.fromMilliseconds(200), .awake);
		388	event.set(io);
		389	}
		390	};
		391
		392	const io = std.testing.io;
		393
		394	var event: Io.Event = .unset;
		395
		396	var wait_future = io.concurrent(global.waitTwice, .{ io, &event }) catch \|err\| switch (err) {
		397	error.ConcurrencyUnavailable => return error.SkipZigTest,
		398	};
		399	defer wait_future.cancel(io) catch {};
		400
		401	var set_future = try io.concurrent(global.sleepThenSet, .{ io, &event });
		402	defer set_future.cancel(io) catch {};
		403
		404	if (wait_future.cancel(io)) {
		405	return error.UnexpectedSuccess; // there was no `set` call to unblock the second `wait`
		406	} else \|err\| switch (err) {
		407	error.Canceled => {},
		408	error.CanceledWhileProtected => \|e\| return e,
		409	}
		410
		411	// Because it reached the `set`, it should be too late for `sleepThenSet` to see `error.Canceled`.
		412	try set_future.cancel(io);
		413	}

src/Sema.zig-27

...	@@ -29023,33 +29023,6 @@ fn coerceExtra(	...	@@ -29023,33 +29023,6 @@ fn coerceExtra(
29023	else => {},	29023	else => {},
29024	},	29024	},
29025	.error_union => switch (inst_ty.zigTypeTag(zcu)) {	29025	.error_union => switch (inst_ty.zigTypeTag(zcu)) {
29026	.error_union => eu: {
29027	if (maybe_inst_val) \|inst_val\| {
29028	switch (inst_val.toIntern()) {
29029	.undef => return pt.undefRef(dest_ty),
29030	else => switch (zcu.intern_pool.indexToKey(inst_val.toIntern())) {
29031	.error_union => \|error_union\| switch (error_union.val) {
29032	.err_name => \|err_name\| {
29033	const error_set_ty = inst_ty.errorUnionSet(zcu);
29034	const error_set_val = Air.internedToRef((try pt.intern(.{ .err = .{
29035	.ty = error_set_ty.toIntern(),
29036	.name = err_name,
29037	} })));
29038	return sema.wrapErrorUnionSet(block, dest_ty, error_set_val, inst_src);
29039	},
29040	.payload => \|payload\| {
29041	const payload_val = Air.internedToRef(payload);
29042	return sema.wrapErrorUnionPayload(block, dest_ty, payload_val, inst_src) catch \|err\| switch (err) {
29043	error.NotCoercible => break :eu,
29044	else => \|e\| return e,
29045	};
29046	},
29047	},
29048	else => unreachable,
29049	},
29050	}
29051	}
29052	},
29053	.error_set => {	29026	.error_set => {
29054	// E to E!T	29027	// E to E!T
29055	return sema.wrapErrorUnionSet(block, dest_ty, inst, inst_src);	29028	return sema.wrapErrorUnionSet(block, dest_ty, inst, inst_src);

src/codegen/x86_64/CodeGen.zig+8-6

 @@ -171444,12 +171444,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
                             const elem_dies = bt.feed();
                             if (tuple_type.values.get(ip)[field_index] != .none) continue;
                             const field_type = Type.fromInterned(tuple_type.types.get(ip)[field_index]);
-                            elem_disp = @intCast(field_type.abiAlignment(zcu).forward(elem_disp));
+                            if (!hack_around_sema_opv_bugs or field_type.hasRuntimeBitsIgnoreComptime(zcu)) {
-                            var elem = try cg.tempFromOperand(elem_ref, elem_dies);
+                                elem_disp = @intCast(field_type.abiAlignment(zcu).forward(elem_disp));
-                            try res.write(&elem, .{ .disp = elem_disp }, cg);
+                                var elem = try cg.tempFromOperand(elem_ref, elem_dies);
-                            try elem.die(cg);
+                                try res.write(&elem, .{ .disp = elem_disp }, cg);
-                            try cg.resetTemps(reset_index);
+                                try elem.die(cg);
-                            elem_disp += @intCast(field_type.abiSize(zcu));
+                                try cg.resetTemps(reset_index);
+                                elem_disp += @intCast(field_type.abiSize(zcu));
+                            }
                         }
                     },
                     else => return cg.fail("failed to select {s} {f}", .{

...	@@ -2,6 +2,7 @@ Adam Goertz <adambgoertz@gmail.com>	...	@@ -2,6 +2,7 @@ Adam Goertz <adambgoertz@gmail.com>
2	Ali Chraghi <alichraghi@proton.me> <alichraghi@pm.me>	2	Ali Chraghi <alichraghi@proton.me> <alichraghi@pm.me>
3	Andrea Orru <andrea@orru.io> <andreaorru1991@gmail.com>	3	Andrea Orru <andrea@orru.io> <andreaorru1991@gmail.com>
4	Andrew Kelley <andrew@ziglang.org> <superjoe30@gmail.com>	4	Andrew Kelley <andrew@ziglang.org> <superjoe30@gmail.com>
		5	Andrew Kelley <andrew@ziglang.org> <andrewrk@noreply.codeberg.org>
5	Bogdan Romanyuk <wrongnull@gmail.com> <65823030+wrongnull@users.noreply.github.com>	6	Bogdan Romanyuk <wrongnull@gmail.com> <65823030+wrongnull@users.noreply.github.com>
6	Casey Banner <kcbanner@gmail.com>	7	Casey Banner <kcbanner@gmail.com>
7	Dacheng Gao <successgdc@gmail.com>	8	Dacheng Gao <successgdc@gmail.com>
...	@@ -22,6 +23,7 @@ Felix "xq" Queißner <xq@random-projects.net> <git@masterq32.de>	...	@@ -22,6 +23,7 @@ Felix "xq" Queißner <xq@random-projects.net> <git@masterq32.de>
22	Felix "xq" Queißner <xq@random-projects.net> <git@mq32.de>	23	Felix "xq" Queißner <xq@random-projects.net> <git@mq32.de>
23	Felix "xq" Queißner <xq@random-projects.net> <git@random-projects.net>	24	Felix "xq" Queißner <xq@random-projects.net> <git@random-projects.net>
24	Frank Denis <124872+jedisct1@users.noreply.github.com> <github@pureftpd.org>	25	Frank Denis <124872+jedisct1@users.noreply.github.com> <github@pureftpd.org>
		26	Frank Denis <124872+jedisct1@users.noreply.github.com> <jedisct1@noreply.codeberg.org>
25	Garrett Beck <garrettlennoxbeck@gmail.com> <138411610+garrettlennoxbeck@users.noreply.github.com>	27	Garrett Beck <garrettlennoxbeck@gmail.com> <138411610+garrettlennoxbeck@users.noreply.github.com>
26	Gaëtan S <blaxoujunior@gmail.com>	28	Gaëtan S <blaxoujunior@gmail.com>
27	GalaxyShard <dominic.adragna@byteroach.com>	29	GalaxyShard <dominic.adragna@byteroach.com>
...	@@ -55,6 +57,7 @@ Marc Tiehuis <marc@tiehu.is> <marctiehuis@gmail.com>	...	@@ -55,6 +57,7 @@ Marc Tiehuis <marc@tiehu.is> <marctiehuis@gmail.com>
55	Mason Remaley <mason@anthropicstudios.com>	57	Mason Remaley <mason@anthropicstudios.com>
56	Mason Remaley <mason@anthropicstudios.com> <MasonRemaley@users.noreply.github.com>	58	Mason Remaley <mason@anthropicstudios.com> <MasonRemaley@users.noreply.github.com>
57	Matthew Lugg <mlugg@mlugg.co.uk>	59	Matthew Lugg <mlugg@mlugg.co.uk>
		60	Matthew Lugg <mlugg@mlugg.co.uk> <mlugg@noreply.codeberg.org>
58	Meghan Denny <hello@nektro.net>	61	Meghan Denny <hello@nektro.net>
59	Meghan Denny <hello@nektro.net> <meghan@bun.sh>	62	Meghan Denny <hello@nektro.net> <meghan@bun.sh>
60	Michael Bartnett <michael.bartnett@gmail.com> <michaelbartnett@users.noreply.github.com>	63	Michael Bartnett <michael.bartnett@gmail.com> <michaelbartnett@users.noreply.github.com>